From 852c1dc2866fc59b6250288aad6c8d7c988588a8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 23 Dec 2014 16:35:54 -0800 Subject: Print out fetch, build, and total time for builds. --- lib/spack/spack/package.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fa91dbbbea..4ab7ff23cf 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -35,6 +35,7 @@ README. """ import os import re +import time import inspect import subprocess import platform as py_platform @@ -390,6 +391,10 @@ class Package(object): if not hasattr(self, 'list_depth'): self.list_depth = 1 + # Set up some internal variables for timing. + self._fetch_time = 0.0 + self._total_time = 0.0 + @property def version(self): @@ -606,6 +611,7 @@ class Package(object): if not self.spec.concrete: raise ValueError("Can only fetch concrete packages.") + start_time = time.time() if spack.do_checksum and not self.version in self.versions: tty.warn("There is no checksum on file to fetch %s safely." % self.spec.format('$_$@')) @@ -624,6 +630,7 @@ class Package(object): "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) self.stage.fetch() + self._fetch_time = time.time() - start_time if spack.do_checksum and self.version in self.versions: self.stage.check() @@ -720,6 +727,7 @@ class Package(object): if not ignore_deps: self.do_install_dependencies() + start_time = time.time() if not fake_install: self.do_patch() @@ -765,7 +773,13 @@ class Package(object): if not keep_stage: self.stage.destroy() - tty.msg("Successfully installed %s" % self.name) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time + + tty.msg("Successfully installed %s." % self.name, + "Fetch: %.2f sec. Build: %.2f sec. Total: %.2f sec." + % (self._fetch_time, build_time, self._total_time)) print_pkg(self.prefix) # Use os._exit here to avoid raising a SystemExit exception, -- cgit v1.2.3-70-g09d2 From b3042db75523b465c088f9aff8efb73049a9c81c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 25 Dec 2014 16:05:45 -0800 Subject: Add patch function to Package, so that packages can define custom patch functions. --- lib/spack/spack/build_environment.py | 7 +++++++ lib/spack/spack/package.py | 16 +++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a2fcff1f10..87cfa772ca 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -199,3 +199,10 @@ def set_module_variables_for_package(pkg): # Useful directories within the prefix are encapsulated in # a Prefix object. m.prefix = pkg.prefix + + +def setup_package(pkg): + """Execute all environment setup routines.""" + set_compiler_environment_variables(pkg) + set_build_environment_variables(pkg) + set_module_variables_for_package(pkg) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4ab7ff23cf..1a797e88b1 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -662,8 +662,11 @@ class Package(object): # Kick off the stage first. self.do_stage() + # Package can add its own patch function. + has_patch_fun = hasattr(self, 'patch') and callable(self.patch) + # If there are no patches, note it. - if not self.patches: + if not self.patches and not has_patch_fun: tty.msg("No patches needed for %s." % self.name) return @@ -686,7 +689,7 @@ class Package(object): tty.msg("Already patched %s" % self.name) return - # Apply all the patches for specs that match this on + # Apply all the patches for specs that match this one for spec, patch_list in self.patches.items(): if self.spec.satisfies(spec): for patch in patch_list: @@ -704,6 +707,11 @@ class Package(object): os.remove(bad_file) touch(good_file) + if has_patch_fun: + self.patch() + + tty.msg("Patched %s" % self.name) + def do_install(self, **kwargs): """This class should call this version of the install method. @@ -750,9 +758,7 @@ class Package(object): spack.install_layout.make_path_for_spec(self.spec) # Set up process's build environment before running install. - build_env.set_compiler_environment_variables(self) - build_env.set_build_environment_variables(self) - build_env.set_module_variables_for_package(self) + build_env.setup_package(self) if fake_install: mkdirp(self.prefix.bin) -- cgit v1.2.3-70-g09d2 From 7b71e6fb5a5c8c73c71814ceaa756681afc73c61 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 25 Dec 2014 16:06:30 -0800 Subject: spack env command spack env allows regular commands to be run with a spack build environment. It also displays the spack build environment for a package. --- lib/spack/spack/cmd/env.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 lib/spack/spack/cmd/env.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py new file mode 100644 index 0000000000..bde76b5daf --- /dev/null +++ b/lib/spack/spack/cmd/env.py @@ -0,0 +1,69 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +from external import argparse +import llnl.util.tty as tty +import spack.cmd +import spack.build_environment as build_env + +description = "Run a command with the environment for a particular spec's install." + +def setup_parser(subparser): + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help="specs of package environment to emulate.") + + +def env(parser, args): + if not args.spec: + tty.die("spack env requires a spec.") + + # Specs may have spaces in them, so if they do, require that the + # caller put a '--' between the spec and the command to be + # executed. If there is no '--', assume that the spec is the + # first argument. + sep = '--' + if sep in args.spec: + s = args.spec.index(sep) + spec = args.spec[:s] + cmd = args.spec[s+1:] + else: + spec = args.spec[0] + cmd = args.spec[1:] + + specs = spack.cmd.parse_specs(spec, concretize=True) + if len(specs) > 1: + tty.die("spack env only takes one spec.") + spec = specs[0] + + build_env.setup_package(spec.package) + + if not cmd: + # If no command act like the "env" command and print out env vars. + for key, val in os.environ.items(): + print "%s=%s" % (key, val) + + else: + # Otherwise execute the command with the new environment + os.execvp(cmd[0], cmd) -- cgit v1.2.3-70-g09d2 From 20388ece86c91c7f2db112b6da38cfb2b8853196 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 25 Dec 2014 16:07:39 -0800 Subject: Clearer code in filter_file. --- lib/spack/llnl/util/filesystem.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6a04d98a18..24c77a80db 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -63,8 +63,11 @@ def filter_file(regex, repl, *filenames, **kwargs): # Allow strings to use \1, \2, etc. for replacement, like sed if not callable(repl): unescaped = repl.replace(r'\\', '\\') - repl = lambda m: re.sub( - r'\\([0-9])', lambda x: m.group(int(x.group(1))), unescaped) + def replace_groups_with_groupid(m): + def groupid_to_group(x): + return m.group(int(x.group(1))) + return re.sub(r'\\([1-9])', groupid_to_group, unescaped) + repl = replace_groups_with_groupid if string: regex = re.escape(regex) -- cgit v1.2.3-70-g09d2 From 0bc861db6e8edf470c91be81e60546b0619216de Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 25 Dec 2014 17:55:19 -0800 Subject: Fix up bzip2 install --- lib/spack/llnl/util/filesystem.py | 8 ++++++++ lib/spack/spack/build_environment.py | 1 + var/spack/packages/bzip2/package.py | 17 ++++++++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24c77a80db..9f08832598 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -30,6 +30,7 @@ import os import sys import re import shutil +import stat import errno import getpass from contextlib import contextmanager, closing @@ -145,6 +146,13 @@ def install(src, dest): shutil.copy(src, dest) set_install_permissions(dest) + src_mode = os.stat(src).st_mode + dest_mode = os.stat(dest).st_mode + if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + os.chmod(dest, dest_mode) + def expand_user(path): """Find instances of '%u' in a path and replace with the current user's diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87cfa772ca..45353ec640 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -190,6 +190,7 @@ def set_module_variables_for_package(pkg): m.makedirs = os.makedirs m.remove = os.remove m.removedirs = os.removedirs + m.symlink = os.symlink m.mkdirp = mkdirp m.install = install diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py index 83ae88e564..fbd5869a53 100644 --- a/var/spack/packages/bzip2/package.py +++ b/var/spack/packages/bzip2/package.py @@ -1,4 +1,5 @@ from spack import * +from glob import glob class Bzip2(Package): """bzip2 is a freely available, patent free high-quality data @@ -15,5 +16,19 @@ class Bzip2(Package): # No configure system -- have to filter the makefile for this package. filter_file(r'CC=gcc', 'CC=cc', 'Makefile', string=True) - make() + make('-f', 'Makefile-libbz2_so') + make('clean') make("install", "PREFIX=%s" % prefix) + + bzip2_exe = join_path(prefix.bin, 'bzip2') + install('bzip2-shared', bzip2_exe) + for libfile in glob('libbz2.so*'): + install(libfile, prefix.lib) + + bunzip2 = join_path(prefix.bin, 'bunzip2') + remove(bunzip2) + symlink(bzip2_exe, bunzip2) + + bzcat = join_path(prefix.bin, 'bzcat') + remove(bzcat) + symlink(bzip2_exe, bzcat) -- cgit v1.2.3-70-g09d2