diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 2 | ||||
| -rw-r--r-- | lib/spack/docs/getting_started.rst | 2 | ||||
| -rwxr-xr-x | lib/spack/env/cc | 2 | ||||
| -rw-r--r-- | lib/spack/llnl/util/filesystem.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/checksum.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/repo.py | 48 | ||||
| -rw-r--r-- | lib/spack/spack/directives.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/directory_layout.py | 14 | ||||
| -rw-r--r-- | lib/spack/spack/modules.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/multimethod.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 57 | ||||
| -rw-r--r-- | lib/spack/spack/repository.py | 99 | ||||
| -rw-r--r-- | lib/spack/spack/resource.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/test/namespace_trie.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/test/tally_plugin.py | 8 | ||||
| -rw-r--r-- | lib/spack/spack/util/environment.py | 7 | ||||
| -rw-r--r-- | lib/spack/spack/util/web.py | 6 |
17 files changed, 218 insertions, 72 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db..f94ac3d2ba 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -896,7 +896,7 @@ Or, similarly with modules, you could type: $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use a package, you can type unload or unuse similarly: .. code-block:: sh diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 67ca18e71a..2c5b68ea65 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_: +<http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_: .. code-block:: sh diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b3..a323c48124 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -130,7 +130,7 @@ if [ -z "$mode" ]; then done fi -# Dump the version and exist if we're in testing mode. +# Dump the version and exit if we're in testing mode. if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then echo "$mode" exit diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 015eeb9aa1..366237ef8f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -152,15 +152,20 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) + + # Expand dsst to its eventual full path if it is a directory. + if os.path.isdir(dest): + dest = join_path(dest, os.path.basename(src)) + shutil.copy(src, dest) set_install_permissions(dest) copy_mode(src, dest) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index b1ad89dbb8..c451993233 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs): tty.msg("Downloading...") hashes = [] - for i, (url, version) in enumerate(zip(urls, versions)): + i = 0 + for url, version in zip(urls, versions): stage = Stage(url) try: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) - hashes.append( - spack.util.crypto.checksum(hashlib.md5, stage.archive_file)) + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) except FailedDownloadError, e: tty.msg("Failed to fetch %s" % url) continue + except Exception, e: + tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) + continue finally: if not keep_stage: stage.destroy() + i += 1 - return zip(versions, hashes) + return hashes diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 34c755fb67..c2e352786d 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -74,51 +74,7 @@ def setup_parser(subparser): def repo_create(args): """Create a new package repository.""" - root = canonicalize_path(args.directory) - namespace = args.namespace - - if not args.namespace: - namespace = os.path.basename(root) - - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("'%s' is not a valid namespace." % namespace) - - existed = False - if os.path.exists(root): - if os.path.isfile(root): - tty.die('File %s already exists and is not a directory' % root) - elif os.path.isdir(root): - if not os.access(root, os.R_OK | os.W_OK): - tty.die('Cannot create new repo in %s: cannot access directory.' % root) - if os.listdir(root): - tty.die('Cannot create new repo in %s: directory is not empty.' % root) - existed = True - - full_path = os.path.realpath(root) - parent = os.path.dirname(full_path) - if not os.access(parent, os.R_OK | os.W_OK): - tty.die("Cannot create repository in %s: can't access parent!" % root) - - try: - config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) - - mkdirp(packages_path) - with open(config_path, 'w') as config: - config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) - - except (IOError, OSError) as e: - tty.die('Failed to create new repository in %s.' % root, - "Caused by %s: %s" % (type(e), e)) - - # try to clean up. - if existed: - shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(packages_path, ignore_errors=True) - else: - shutil.rmtree(root, ignore_errors=True) - + full_path, namespace = create_repo(args.directory, args.namespace) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", 'spack repo add %s' % full_path) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 5745adce63..c8542f55f0 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs): def _depends_on(pkg, spec, when=None): - if when is None: + # If when is False do nothing + if when is False: + return + # If when is None or True make sure the condition is always satisfied + if when is None or when is True: when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 3e416a6a1f..29d87b65b3 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -173,7 +173,9 @@ class YamlDirectoryLayout(DirectoryLayout): self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' - self.build_log_name = 'build.out' # TODO: use config file. + self.build_log_name = 'build.out' # build log. + self.build_env_name = 'build.env' # build environment + self.packages_dir = 'repos' # archive of package.py files # Cache of already written/read extension maps. self._extension_maps = {} @@ -231,6 +233,16 @@ class YamlDirectoryLayout(DirectoryLayout): self.build_log_name) + def build_env_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.build_env_name) + + + def build_packages_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.packages_dir) + + def create_install_directory(self, spec): _check_concrete(spec) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index c834763564..c27043db8c 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -33,6 +33,7 @@ number of directories to be appended to paths in the user's environment: * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH + * /include directories for CPATH * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH @@ -121,6 +122,7 @@ class EnvModule(object): ('LIBRARY_PATH', self.spec.prefix.lib64), ('LD_LIBRARY_PATH', self.spec.prefix.lib), ('LD_LIBRARY_PATH', self.spec.prefix.lib64), + ('CPATH', self.spec.prefix.include), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index df9b9b2ab1..51c6a8e89d 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -193,10 +193,11 @@ class when(object): platform-specific versions. There's not much we can do to get around this because of the way decorators work. """ -class when(object): def __init__(self, spec): pkg = get_calling_module_name() - self.spec = parse_anonymous_spec(spec, pkg) + if spec is True: + spec = pkg + self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None def __call__(self, method): # Get the first definition of the method in the calling scope @@ -207,7 +208,9 @@ class when(object): if not type(original_method) == SpecMultiMethod: original_method = SpecMultiMethod(original_method) - original_method.register(self.spec, method) + if self.spec is not None: + original_method.register(self.spec, method) + return original_method diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0214dcd771..106c546d5c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -58,6 +58,7 @@ import spack.compilers import spack.mirror import spack.hooks import spack.directives +import spack.repository import spack.build_environment import spack.url import spack.util.web @@ -66,6 +67,7 @@ from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError +from spack.util.environment import dump_environment """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -884,10 +886,14 @@ class Package(object): # Do the real install in the source directory. self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + # This redirects I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) self.install(self.spec, self.prefix) # Ensure that something was actually installed. @@ -896,7 +902,12 @@ class Package(object): # Move build log into install directory on success if not fake: log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) install(log_path, log_install_path) + install(env_path, env_install_path) + + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) # On successful install, remove the stage. if not keep_stage: @@ -1205,6 +1216,52 @@ def validate_package_url(url_string): tty.die("Invalid file type in URL: '%s'" % url_string) +def dump_packages(spec, path): + """Dump all package information for a spec and its dependencies. + + This creates a package repository within path for every + namespace in the spec DAG, and fills the repos wtih package + files and patch files for every node in the DAG. + """ + mkdirp(path) + + # Copy in package.py files from any dependencies. + # Note that we copy them in as they are in the *install* directory + # NOT as they are in the repository, because we want a snapshot of + # how *this* particular build was done. + for node in spec.traverse(): + if node is not spec: + # Locate the dependency package in the install tree and find + # its provenance information. + source = spack.install_layout.build_packages_path(node) + source_repo_root = join_path(source, node.namespace) + + # There's no provenance installed for the source package. Skip it. + # User can always get something current from the builtin repo. + if not os.path.isdir(source_repo_root): + continue + + # Create a source repo and get the pkg directory out of it. + try: + source_repo = spack.repository.Repo(source_repo_root) + source_pkg_dir = source_repo.dirname_for_package_name(node.name) + except RepoError as e: + tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + + # Create a destination repository + dest_repo_root = join_path(path, node.namespace) + if not os.path.exists(dest_repo_root): + spack.repository.create_repo(dest_repo_root) + repo = spack.repository.Repo(dest_repo_root) + + # Get the location of the package in the dest repo. + dest_pkg_dir = repo.dirname_for_package_name(node.name) + if node is not spec: + install_tree(source_pkg_dir, dest_pkg_dir) + else: + spack.repo.dump_provenance(node, dest_pkg_dir) + + def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index f58cd52125..8d06fefe7f 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -33,7 +33,7 @@ from bisect import bisect_left from external import yaml import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import * import spack.error import spack.config @@ -316,6 +316,16 @@ class RepoPath(object): return self.repo_for_pkg(spec).get(spec) + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).dump_provenance(spec, path) + + def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) @@ -552,6 +562,35 @@ class Repo(object): return self._instances[key] + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + # Some preliminary checks. + if spec.virtual: + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) + + # Install any patch files needed by packages. + mkdirp(path) + for spec, patches in spec.package.patches.items(): + for patch in patches: + if patch.path: + if os.path.exists(patch.path): + install(patch.path, path) + else: + tty.warn("Patch file did not exist: %s" % patch.path) + + # Install the package.py file itself. + install(self.filename_for_package_name(spec), path) + + def purge(self): """Clear entire package instance cache.""" self._instances.clear() @@ -705,6 +744,58 @@ class Repo(object): return self.exists(pkg_name) +def create_repo(root, namespace=None): + """Create a new repository in root with the specified namespace. + + If the namespace is not provided, use basename of root. + Return the canonicalized path and the namespace of the created repository. + """ + root = canonicalize_path(root) + if not namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace) + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + raise BadRepoError('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + raise BadRepoError("Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + raise BadRepoError('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + return full_path, namespace + + class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" @@ -713,6 +804,10 @@ class NoRepoConfiguredError(RepoError): """Raised when there are no repositories configured.""" +class InvalidNamespaceError(RepoError): + """Raised when an invalid namespace is encountered.""" + + class BadRepoError(RepoError): """Raised when repo layout is invalid.""" diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 2bf92947fd..ddfaaf4cb0 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 83fb34bf76..2023ba6d96 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index eb1e4a3240..4163ab95dd 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://scalability-software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ class Tally(Plugin): self.successCount = 0 self.failCount = 0 self.errorCount = 0 - + @property def numberOfTestsRun(self): """Excludes skipped tests""" @@ -48,10 +48,10 @@ class Tally(Plugin): def addSuccess(self, test): self.successCount += 1 - + def addError(self, test, err): self.errorCount += 1 - + def addFailure(self, test, err): self.failCount += 1 diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index cd413dcfbc..ae8e5708be 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys): for key in keys: if key in dictionary: dictionary.pop(key) + + +def dump_environment(path): + """Dump the current environment out to a file.""" + with open(path, 'w') as env_file: + for key,val in sorted(os.environ.items()): + env_file.write("%s=%s\n" % (key, val)) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index e26daef296..73f4858b02 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -86,12 +86,12 @@ def _spider(args): if not "Content-type" in resp.headers: tty.debug("ignoring page " + url) - return pages + return pages, links if not resp.headers["Content-type"].startswith('text/html'): tty.debug("ignoring page " + url + " with content type " + resp.headers["Content-type"]) - return pages + return pages, links # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" @@ -173,7 +173,7 @@ def spider(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) return pages, links |