From c615d2be06eeaf10058ddece27f69d6e148f44af Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Apr 2018 23:23:16 -0700 Subject: init: move spack.repo global variable into its own module - spack.repository module is now spack.repo - `spack.repo` is now `spack.repo.path()` and loaded lazily - Added `spack.repo.get()` and `spack.repo.all_package_names()` as convenience functions to simplify the new lazy interface. - updated tests and code --- lib/spack/docs/repositories.rst | 2 +- lib/spack/spack/__init__.py | 21 - lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/cmd/blame.py | 1 + lib/spack/spack/cmd/bootstrap.py | 3 +- lib/spack/spack/cmd/buildcache.py | 4 +- lib/spack/spack/cmd/checksum.py | 3 +- lib/spack/spack/cmd/clean.py | 3 +- lib/spack/spack/cmd/create.py | 8 +- lib/spack/spack/cmd/dependencies.py | 7 +- lib/spack/spack/cmd/dependents.py | 8 +- lib/spack/spack/cmd/diy.py | 4 +- lib/spack/spack/cmd/edit.py | 8 +- lib/spack/spack/cmd/extensions.py | 4 +- lib/spack/spack/cmd/fetch.py | 3 +- lib/spack/spack/cmd/find.py | 6 +- lib/spack/spack/cmd/info.py | 8 +- lib/spack/spack/cmd/list.py | 10 +- lib/spack/spack/cmd/location.py | 5 +- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/module.py | 11 +- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 5 +- lib/spack/spack/cmd/providers.py | 6 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/setup.py | 10 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/uninstall.py | 4 +- lib/spack/spack/cmd/url.py | 6 +- lib/spack/spack/cmd/versions.py | 3 +- lib/spack/spack/concretize.py | 4 +- lib/spack/spack/database.py | 4 +- lib/spack/spack/main.py | 9 +- lib/spack/spack/package.py | 19 +- lib/spack/spack/package_prefs.py | 4 +- lib/spack/spack/relocate.py | 2 +- lib/spack/spack/repo.py | 1200 ++++++++++++++++++++++++ lib/spack/spack/repository.py | 1153 ----------------------- lib/spack/spack/spec.py | 11 +- lib/spack/spack/test/build_systems.py | 1 + lib/spack/spack/test/cmd/flake8.py | 2 +- lib/spack/spack/test/cmd/install.py | 4 +- lib/spack/spack/test/concretize.py | 16 +- lib/spack/spack/test/concretize_preferences.py | 3 +- lib/spack/spack/test/conftest.py | 19 +- lib/spack/spack/test/data/repos.yaml | 2 + lib/spack/spack/test/database.py | 9 +- lib/spack/spack/test/directory_layout.py | 36 +- lib/spack/spack/test/flag_handlers.py | 1 + lib/spack/spack/test/git_fetch.py | 2 +- lib/spack/spack/test/hg_fetch.py | 2 +- lib/spack/spack/test/install.py | 2 +- lib/spack/spack/test/mirror.py | 2 +- lib/spack/spack/test/multimethod.py | 3 +- lib/spack/spack/test/package_sanity.py | 11 +- lib/spack/spack/test/packages.py | 7 +- lib/spack/spack/test/packaging.py | 2 +- lib/spack/spack/test/provider_index.py | 2 +- lib/spack/spack/test/repo.py | 10 +- lib/spack/spack/test/spec_dag.py | 7 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/test/url_fetch.py | 2 +- lib/spack/spack/util/package_hash.py | 4 +- 64 files changed, 1374 insertions(+), 1350 deletions(-) create mode 100644 lib/spack/spack/repo.py delete mode 100644 lib/spack/spack/repository.py create mode 100644 lib/spack/spack/test/data/repos.yaml (limited to 'lib') diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 5e722e2139..5ec3304a35 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -445,7 +445,7 @@ Spack repo namespaces are actually Python namespaces tacked on under ``spack.pkg``. The search semantics of ``repos.yaml`` are actually implemented using Python's built-in `sys.path `_ search. The -:py:mod:`spack.repository` module implements a custom `Python importer +:py:mod:`spack.repo` module implements a custom `Python importer `_. .. warning:: diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 4dc07d43fe..0b5b2717b5 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,7 +23,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys #: major, minor, patch version for Spack, in a tuple spack_version_info = (0, 11, 2) @@ -32,29 +31,9 @@ spack_version_info = (0, 11, 2) spack_version = '.'.join(str(v) for v in spack_version_info) -# Set up the default packages database. -import spack.error -try: - import spack.repository - repo = spack.repository.RepoPath() - sys.meta_path.append(repo) -except spack.error.SpackError as e: - import llnl.util.tty as tty - tty.die('while initializing Spack RepoPath:', e.message) - - #----------------------------------------------------------------------------- # When packages call 'from spack import *', we import a set of things that # should be useful for builds. -# -# Spack internal code should call 'import spack' and accesses other -# variables (spack.repo, paths, etc.) directly. -# -# TODO: maybe this should be separated out to build_environment.py? -# TODO: it's not clear where all the stuff that needs to be included in -# packages should live. This file is overloaded for spack core vs. -# for packages. -# #----------------------------------------------------------------------------- __all__ = [] diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7a2e2e5094..cdf3e86577 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -587,8 +587,8 @@ def setup_package(pkg, dirty): for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): # If a user makes their own package repo, e.g. - # spack.repos.mystuff.libelf.Libelf, and they inherit from - # an existing class like spack.repos.original.libelf.Libelf, + # spack.pkg.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.pkg.original.libelf.Libelf, # then set the module variables for both classes so the # parent class can still use them if it gets called. spkg = dspec.package diff --git a/lib/spack/spack/cmd/blame.py b/lib/spack/spack/cmd/blame.py index 38b2a13879..54a1469cbc 100644 --- a/lib/spack/spack/cmd/blame.py +++ b/lib/spack/spack/cmd/blame.py @@ -31,6 +31,7 @@ from llnl.util.filesystem import working_dir from llnl.util.tty.colify import colify_table import spack.paths +import spack.repo from spack.util.executable import which from spack.cmd import spack_is_git_repo diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 41653ca1b6..80abbd8045 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -23,7 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import llnl.util.tty as tty -import spack + +import spack.repo import spack.cmd import spack.cmd.common.arguments as arguments diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 8161548283..a9f9b6d6b8 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -26,8 +26,10 @@ import argparse import llnl.util.tty as tty -import spack import spack.cmd +import spack.repo +import spack.store +import spack.spec import spack.binary_distribution as bindist description = "create, download and install binary packages" diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 1cb949efb8..7cf1aa42be 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -27,8 +27,9 @@ from __future__ import print_function import argparse import llnl.util.tty as tty -import spack + import spack.cmd +import spack.repo import spack.util.crypto import spack.util.web from spack.util.naming import valid_fully_qualified_module_name diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 83d5116d0e..df0f62e167 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -26,9 +26,10 @@ import argparse import llnl.util.tty as tty -import spack import spack.caches import spack.cmd +import spack.repo +import spack.stage description = "remove temporary build files and/or downloaded archives" section = "build" diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 6bd3a80925..6eab4e4834 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -32,7 +32,7 @@ import spack import spack.cmd import spack.util.web from llnl.util.filesystem import mkdirp -from spack.repository import Repo +import spack.repo from spack.spec import Spec from spack.util.executable import which, ProcessError from spack.util.naming import mod_to_class @@ -648,17 +648,17 @@ def get_repository(args, name): # Figure out where the new package should live repo_path = args.repo if repo_path is not None: - repo = Repo(repo_path) + repo = spack.repo.Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: tty.die("Can't create package with namespace {0} in repo with " "namespace {1}".format(spec.namespace, repo.namespace)) else: if spec.namespace: - repo = spack.repo.get_repo(spec.namespace, None) + repo = spack.repo.path().get_repo(spec.namespace, None) if not repo: tty.die("Unknown namespace: '{0}'".format(spec.namespace)) else: - repo = spack.repo.first_repo() + repo = spack.repo.path().first_repo() # Set the namespace on the spec if it's not there already if not spec.namespace: diff --git a/lib/spack/spack/cmd/dependencies.py b/lib/spack/spack/cmd/dependencies.py index 492264fe24..ad34fa16ce 100644 --- a/lib/spack/spack/cmd/dependencies.py +++ b/lib/spack/spack/cmd/dependencies.py @@ -27,8 +27,8 @@ import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -import spack import spack.store +import spack.repo import spack.cmd description = "show dependencies of a package" @@ -73,8 +73,9 @@ def dependencies(parser, args): if not spec.virtual: packages = [spec.package] else: - packages = [spack.repo.get(s.name) - for s in spack.repo.providers_for(spec)] + packages = [ + spack.repo.get(s.name) + for s in spack.repo.path().providers_for(spec)] dependencies = set() for pkg in packages: diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index f7157514c1..602d386c0a 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -27,7 +27,7 @@ import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -import spack +import spack.repo import spack.store import spack.cmd @@ -57,14 +57,14 @@ def inverted_dependencies(): actual dependents. """ dag = {} - for pkg in spack.repo.all_packages(): + for pkg in spack.repo.path().all_packages(): dag.setdefault(pkg.name, set()) for dep in pkg.dependencies: deps = [dep] # expand virtuals if necessary - if spack.repo.is_virtual(dep): - deps += [s.name for s in spack.repo.providers_for(dep)] + if spack.repo.path().is_virtual(dep): + deps += [s.name for s in spack.repo.path().providers_for(dep)] for d in deps: dag.setdefault(d, set()).add(pkg.name) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 1341715d78..eec6f3159c 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -28,9 +28,9 @@ import argparse import llnl.util.tty as tty -import spack import spack.config import spack.cmd +import spack.repo import spack.cmd.common.arguments as arguments from spack.stage import DIYStage @@ -78,7 +78,7 @@ def diy(self, args): tty.die("spack diy only takes one spec.") spec = specs[0] - if not spack.repo.exists(spec.name): + if not spack.repo.path().exists(spec.name): tty.die("No package for '{0}' was found.".format(spec.name), " Use `spack create` to create a new package") diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index fb85acaacc..cd9a5ccee7 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -29,8 +29,8 @@ import llnl.util.tty as tty import spack.cmd import spack.paths +import spack.repo from spack.spec import Spec -from spack.repository import Repo from spack.util.editor import editor description = "open package files in $EDITOR" @@ -48,11 +48,11 @@ def edit_package(name, repo_path, namespace): """ # Find the location of the package if repo_path: - repo = Repo(repo_path) + repo = spack.repo.Repo(repo_path) elif namespace: - repo = spack.repo.get_repo(namespace) + repo = spack.repo.path().get_repo(namespace) else: - repo = spack.repo + repo = spack.repo.path() path = repo.filename_for_package_name(name) spec = Spec(name) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index ccd692147d..6ea115e42f 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -27,9 +27,9 @@ import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -import spack import spack.cmd import spack.cmd.find +import spack.repo import spack.store from spack.directory_layout import YamlViewExtensionsLayout @@ -105,7 +105,7 @@ def extensions(parser, args): if show_packages: # # List package names of extensions - extensions = spack.repo.extensions_for(spec) + extensions = spack.repo.path().extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) else: diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index ca3b49fa10..973b45fbbd 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -26,8 +26,9 @@ import argparse import llnl.util.tty as tty -import spack import spack.cmd +import spack.config +import spack.repo import spack.cmd.common.arguments as arguments description = "fetch archives for packages" diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index c75a5332f7..3b6de0ca84 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -26,8 +26,8 @@ import sys import llnl.util.tty as tty import llnl.util.lang -import spack -import spack.database + +import spack.repo import spack.cmd.common.arguments as arguments from spack.cmd import display_specs @@ -147,7 +147,7 @@ def find(parser, args): # If tags have been specified on the command line, filter by tags if args.tags: - packages_with_tags = spack.repo.packages_with_tags(*args.tags) + packages_with_tags = spack.repo.path().packages_with_tags(*args.tags) query_specs = [x for x in query_specs if x.name in packages_with_tags] # Display the result diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index f091d0d733..008863b0a2 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -25,15 +25,15 @@ from __future__ import print_function import textwrap - from six.moves import zip_longest +import llnl.util.tty.color as color from llnl.util.tty.colify import colify -import llnl.util.tty.color as color -import spack -import spack.fetch_strategy as fs +import spack.repo import spack.spec +import spack.fetch_strategy as fs + description = 'get detailed information on a particular package' section = 'basic' diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 01e5075de0..42370d1a10 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -37,7 +37,8 @@ from six import StringIO import llnl.util.tty as tty from llnl.util.tty.colify import colify -import spack +import spack.dependency +import spack.repo import spack.cmd.common.arguments as arguments description = "list and search available packages" @@ -184,7 +185,7 @@ def rst(pkg_names): reversed(sorted(pkg.versions)))) print() - for deptype in spack.all_deptypes: + for deptype in spack.dependency.all_deptypes: deps = pkg.dependencies_of_type(deptype) if deps: print('%s Dependencies' % deptype.capitalize()) @@ -272,7 +273,7 @@ def html(pkg_names): print(', '.join(str(v) for v in reversed(sorted(pkg.versions)))) print('') - for deptype in spack.all_deptypes: + for deptype in spack.dependency.all_deptypes: deps = pkg.dependencies_of_type(deptype) if deps: print('
%s Dependencies:
' % deptype.capitalize()) @@ -301,7 +302,8 @@ def list(parser, args): # Filter by tags if args.tags: - packages_with_tags = set(spack.repo.packages_with_tags(*args.tags)) + packages_with_tags = set( + spack.repo.path().packages_with_tags(*args.tags)) sorted_packages = set(sorted_packages) & packages_with_tags sorted_packages = sorted(sorted_packages) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index ab3faef7f5..847a56bd5e 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -29,6 +29,7 @@ import llnl.util.tty as tty import spack.paths import spack.cmd +import spack.repo description = "print out locations of various directories used by Spack" section = "environment" @@ -79,7 +80,7 @@ def location(parser, args): print(spack.paths.prefix) elif args.packages: - print(spack.repo.first_repo().root) + print(spack.repo.path().first_repo().root) elif args.stages: print(spack.paths.stage_path) @@ -101,7 +102,7 @@ def location(parser, args): if args.package_dir: # This one just needs the spec name. - print(spack.repo.dirname_for_package_name(spec.name)) + print(spack.repo.path().dirname_for_package_name(spec.name)) else: # These versions need concretized specs. diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 34f342fb9e..42703fd232 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -29,11 +29,11 @@ import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -import spack import spack.cmd import spack.concretize import spack.config import spack.mirror +import spack.repo import spack.cmd.common.arguments as arguments from spack.spec import Spec from spack.error import SpackError diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 908f0e568a..6ec069e50f 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -27,10 +27,12 @@ from __future__ import print_function import collections import os import shutil -import spack.modules -import spack.cmd from llnl.util import filesystem, tty + +import spack.cmd +import spack.modules +import spack.repo from spack.cmd.common import arguments description = "manipulate module files" @@ -278,9 +280,10 @@ def refresh(module_types, specs, args): cls = spack.modules.module_types[module_type] + # skip unknown packages. writers = [ - cls(spec) for spec in specs if spack.repo.exists(spec.name) - ] # skip unknown packages. + cls(spec) for spec in specs + if spack.repo.path().exists(spec.name)] # Filter blacklisted packages early writers = [x for x in writers if not x.conf.blacklisted] diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index efc55667c5..49edf6634a 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -26,7 +26,7 @@ import argparse import llnl.util.tty as tty -import spack +import spack.repo import spack.cmd import spack.cmd.common.arguments as arguments diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index cdb4e92682..2562a3a58a 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -25,13 +25,14 @@ from __future__ import print_function import os - import argparse + import llnl.util.tty as tty from llnl.util.tty.colify import colify from llnl.util.filesystem import working_dir import spack.paths +import spack.repo from spack.util.executable import which from spack.cmd import spack_is_git_repo @@ -90,7 +91,7 @@ def list_packages(rev): def pkg_add(args): for pkg_name in args.packages: - filename = spack.repo.filename_for_package_name(pkg_name) + filename = spack.repo.path().filename_for_package_name(pkg_name) if not os.path.isfile(filename): tty.die("No such package: %s. Path does not exist:" % pkg_name, filename) diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 3b07036d48..d9622ea0d3 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -27,8 +27,8 @@ import sys import llnl.util.tty.colify as colify -import spack import spack.cmd +import spack.repo description = "list packages that provide a particular virtual package" section = "basic" @@ -46,7 +46,7 @@ def setup_parser(subparser): def providers(parser, args): - valid_virtuals = sorted(spack.repo.provider_index.providers.keys()) + valid_virtuals = sorted(spack.repo.path().provider_index.providers.keys()) buffer = six.StringIO() isatty = sys.stdout.isatty() @@ -77,5 +77,5 @@ def providers(parser, args): for spec in specs: if sys.stdout.isatty(): print("{0}:".format(spec)) - spack.cmd.display_specs(sorted(spack.repo.providers_for(spec))) + spack.cmd.display_specs(sorted(spack.repo.path().providers_for(spec))) print('') diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index ca720d7308..802dbd3950 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -30,7 +30,7 @@ import llnl.util.tty as tty import spack.spec import spack.config -from spack.repository import Repo, create_repo, canonicalize_path, RepoError +from spack.repo import Repo, create_repo, canonicalize_path, RepoError description = "manage package source repositories" section = "config" diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index 2a9e2ca93a..9107fdd138 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -26,8 +26,8 @@ import argparse import llnl.util.tty as tty -import spack import spack.cmd +import spack.repo description = "revert checked out package source code" section = "build" diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index 65b71e0cb7..abf1e2957b 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -29,13 +29,15 @@ import string import sys import llnl.util.tty as tty -import spack +from llnl.util.filesystem import set_executable + +import spack.repo import spack.store import spack.cmd import spack.cmd.install as install import spack.cmd.common.arguments as arguments -from llnl.util.filesystem import set_executable -from spack import which +from spack.util.executable import which + from spack.stage import DIYStage description = "create a configuration script and module, but don't build" @@ -135,7 +137,7 @@ def setup(self, args): # Take a write lock before checking for existence. with spack.store.db.write_transaction(): spec = specs[0] - if not spack.repo.exists(spec.name): + if not spack.repo.path().exists(spec.name): tty.die("No package for '{0}' was found.".format(spec.name), " Use `spack create` to create a new package") if not spec.versions.concrete: diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index bceb9b8a6b..47e3eb303a 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -26,7 +26,7 @@ import argparse import llnl.util.tty as tty -import spack +import spack.repo import spack.cmd import spack.cmd.common.arguments as arguments diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 47eead78f9..b9f3833999 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -29,7 +29,7 @@ import argparse import spack import spack.cmd import spack.store -import spack.repository +import spack.repo from llnl.util import tty @@ -149,7 +149,7 @@ def do_uninstall(specs, force): try: # should work if package is known to spack packages.append(item.package) - except spack.repository.UnknownEntityError: + except spack.repo.UnknownEntityError: # The package.py file has gone away -- but still # want to uninstall. spack.Package.uninstall_by_spec(item, force=True) diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py index e31f98171b..cb2695833d 100644 --- a/lib/spack/spack/cmd/url.py +++ b/lib/spack/spack/cmd/url.py @@ -26,7 +26,7 @@ from __future__ import division, print_function from collections import defaultdict -import spack +import spack.repo from llnl.util import tty from spack.url import parse_version_offset, parse_name_offset @@ -144,7 +144,7 @@ def url_list(args): urls = set() # Gather set of URLs from all packages - for pkg in spack.repo.all_packages(): + for pkg in spack.repo.path().all_packages(): url = getattr(pkg.__class__, 'url', None) urls = url_list_parsing(args, urls, url, pkg) @@ -178,7 +178,7 @@ def url_summary(args): tty.msg('Generating a summary of URL parsing in Spack...') # Loop through all packages - for pkg in spack.repo.all_packages(): + for pkg in spack.repo.path().all_packages(): urls = set() url = getattr(pkg.__class__, 'url', None) diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index ea39c96ac5..8c55f83769 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -26,7 +26,8 @@ from __future__ import print_function from llnl.util.tty.colify import colify import llnl.util.tty as tty -import spack + +import spack.repo description = "list available versions of a package" section = "packaging" diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c38dfd024a..d9880e3755 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -39,7 +39,7 @@ from functools_backport import reverse_order from contextlib import contextmanager from six import iteritems -import spack +import spack.repo import spack.abi import spack.spec import spack.compilers @@ -102,7 +102,7 @@ class Concretizer(object): pref_key = lambda spec: 0 # no-op pref key if spec.virtual: - candidates = spack.repo.providers_for(spec) + candidates = spack.repo.path().providers_for(spec) if not candidates: raise spack.spec.UnsatisfiableProviderSpecError( candidates[0], spec) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index f7503dce75..0e98cffc69 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -55,7 +55,7 @@ from llnl.util.filesystem import join_path, mkdirp from llnl.util.lock import Lock, WriteTransaction, ReadTransaction import spack.store -import spack.repository +import spack.repo import spack.spec import spack.util.spack_yaml as syaml import spack.util.spack_json as sjson @@ -903,7 +903,7 @@ class Database(object): if explicit is not any and rec.explicit != explicit: continue - if known is not any and spack.repo.exists( + if known is not any and spack.repo.path().exists( rec.spec.name) != known: continue diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 6a2a1e2470..68ea436be2 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -43,6 +43,8 @@ from llnl.util.tty.log import log_output import spack import spack.config import spack.paths +import spack.repo +import spack.util.debug from spack.error import SpackError @@ -347,13 +349,12 @@ def setup_main_options(args): tty.set_stacktrace(args.stacktrace) if args.debug: - import spack.util.debug as debug - debug.register_interrupt_handler() + spack.util.debug.register_interrupt_handler() spack.config.set('config:debug', True, scope='command_line') if args.mock: - from spack.repository import RepoPath - spack.repo.swap(RepoPath(spack.paths.mock_packages_path)) + rp = spack.repo.RepoPath(spack.paths.mock_packages_path) + spack.repo.set_path(rp) # If the user asked for it, don't check ssl certs. if args.insecure: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8df46c9963..b046a6c661 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -53,7 +53,6 @@ from six import with_metaclass import llnl.util.tty as tty -import spack import spack.config import spack.paths import spack.store @@ -64,7 +63,7 @@ import spack.fetch_strategy as fs import spack.hooks import spack.mirror import spack.mixins -import spack.repository +import spack.repo import spack.url import spack.util.web import spack.multimethod @@ -662,9 +661,9 @@ class PackageBase(with_metaclass(PackageMeta, object)): visited = set([self.name]) for i, name in enumerate(self.dependencies): - if spack.repo.is_virtual(name): + if spack.repo.path().is_virtual(name): if expand_virtuals: - providers = spack.repo.providers_for(name) + providers = spack.repo.path().providers_for(name) dep_names = [spec.name for spec in providers] else: visited.add(name) @@ -1934,7 +1933,7 @@ class PackageBase(with_metaclass(PackageMeta, object)): # Try to get the pcakage for the spec try: pkg = spec.package - except spack.repository.UnknownEntityError: + except spack.repo.UnknownEntityError: pkg = None # Pre-uninstall hook runs first. @@ -2313,25 +2312,25 @@ def dump_packages(spec, path): # Create a source repo and get the pkg directory out of it. try: - source_repo = spack.repository.Repo(source_repo_root) + source_repo = spack.repo.Repo(source_repo_root) source_pkg_dir = source_repo.dirname_for_package_name( node.name) - except spack.repository.RepoError: + except spack.repo.RepoError: tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) # Create a destination repository dest_repo_root = join_path(path, node.namespace) if not os.path.exists(dest_repo_root): - spack.repository.create_repo(dest_repo_root) - repo = spack.repository.Repo(dest_repo_root) + spack.repo.create_repo(dest_repo_root) + repo = spack.repo.Repo(dest_repo_root) # Get the location of the package in the dest repo. dest_pkg_dir = repo.dirname_for_package_name(node.name) if node is not spec: install_tree(source_pkg_dir, dest_pkg_dir) else: - spack.repo.dump_provenance(node, dest_pkg_dir) + spack.repo.path().dump_provenance(node, dest_pkg_dir) def print_pkg(message): diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py index ca27210678..ce84606b00 100644 --- a/lib/spack/spack/package_prefs.py +++ b/lib/spack/spack/package_prefs.py @@ -27,7 +27,7 @@ from six import iteritems from llnl.util.lang import classproperty -import spack +import spack.repo import spack.error from spack.util.path import canonicalize_path from spack.version import VersionList @@ -51,7 +51,7 @@ def get_packages_config(): # by sometihng, not just packages/names that don't exist. # So, this won't include, e.g., 'all'. virtuals = [(pkg_name, pkg_name._start_mark) for pkg_name in config - if spack.repo.is_virtual(pkg_name)] + if spack.repo.path().is_virtual(pkg_name)] # die if there are virtuals in `packages.py` if virtuals: diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 90b082371d..ffda18f350 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -26,7 +26,7 @@ import os import platform import re -import spack +import spack.repo import spack.cmd from spack.util.executable import Executable, ProcessError from llnl.util.filesystem import filter_file diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py new file mode 100644 index 0000000000..448d31ae1d --- /dev/null +++ b/lib/spack/spack/repo.py @@ -0,0 +1,1200 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import collections +import os +import stat +import shutil +import errno +import sys +import inspect +import imp +import re +import traceback +import json +from contextlib import contextmanager +from six import string_types + +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping + +from types import ModuleType + +import yaml + +import llnl.util.lang +import llnl.util.tty as tty +from llnl.util.filesystem import mkdirp, join_path, install + +import spack +import spack.config +import spack.caches +import spack.error +import spack.spec +from spack.provider_index import ProviderIndex +from spack.util.path import canonicalize_path +from spack.util.naming import NamespaceTrie, valid_module_name +from spack.util.naming import mod_to_class, possible_spack_module_names + +# +# Super-namespace for all packages. +# Package modules are imported as spack.pkg... +# +repo_namespace = 'spack.pkg' + +# +# These names describe how repos should be laid out in the filesystem. +# +repo_config_name = 'repo.yaml' # Top-level filename for repo config. +repo_index_name = 'index.yaml' # Top-level filename for repository index. +packages_dir_name = 'packages' # Top-level repo directory containing pkgs. +package_file_name = 'package.py' # Filename for packages in a repository. + +# Guaranteed unused default value for some functions. +NOT_PROVIDED = object() + + +def _autospec(function): + """Decorator that automatically converts the argument of a single-arg + function to a Spec.""" + + def converter(self, spec_like, *args, **kwargs): + if not isinstance(spec_like, spack.spec.Spec): + spec_like = spack.spec.Spec(spec_like) + return function(self, spec_like, *args, **kwargs) + return converter + + +class SpackNamespace(ModuleType): + """ Allow lazy loading of modules.""" + + def __init__(self, namespace): + super(SpackNamespace, self).__init__(namespace) + self.__file__ = "(spack namespace)" + self.__path__ = [] + self.__name__ = namespace + self.__package__ = namespace + self.__modules = {} + + def __getattr__(self, name): + """Getattr lazily loads modules if they're not already loaded.""" + submodule = self.__package__ + '.' + name + setattr(self, name, __import__(submodule)) + return getattr(self, name) + + +class FastPackageChecker(Mapping): + """Cache that maps package names to the stats obtained on the + 'package.py' files associated with them. + + For each repository a cache is maintained at class level, and shared among + all instances referring to it. Update of the global cache is done lazily + during instance initialization. + """ + #: Global cache, reused by every instance + _paths_cache = {} + + def __init__(self, packages_path): + + #: The path of the repository managed by this instance + self.packages_path = packages_path + + # If the cache we need is not there yet, then build it appropriately + if packages_path not in self._paths_cache: + self._paths_cache[packages_path] = self._create_new_cache() + + #: Reference to the appropriate entry in the global cache + self._packages_to_stats = self._paths_cache[packages_path] + + def _create_new_cache(self): + """Create a new cache for packages in a repo. + + The implementation here should try to minimize filesystem + calls. At the moment, it is O(number of packages) and makes + about one stat call per package. This is reasonably fast, and + avoids actually importing packages in Spack, which is slow. + """ + # Create a dictionary that will store the mapping between a + # package name and its stat info + cache = {} + for pkg_name in os.listdir(self.packages_path): + # Skip non-directories in the package root. + pkg_dir = join_path(self.packages_path, pkg_name) + + # Warn about invalid names that look like packages. + if not valid_module_name(pkg_name): + msg = 'Skipping package at {0}. ' + msg += '"{1}" is not a valid Spack module name.' + tty.warn(msg.format(pkg_dir, pkg_name)) + continue + + # Construct the file name from the directory + pkg_file = os.path.join( + self.packages_path, pkg_name, package_file_name + ) + + # Use stat here to avoid lots of calls to the filesystem. + try: + sinfo = os.stat(pkg_file) + except OSError as e: + if e.errno == errno.ENOENT: + # No package.py file here. + continue + elif e.errno == errno.EACCES: + tty.warn("Can't read package file %s." % pkg_file) + continue + raise e + + # If it's not a file, skip it. + if stat.S_ISDIR(sinfo.st_mode): + continue + + # If it is a file, then save the stats under the + # appropriate key + cache[pkg_name] = sinfo + + return cache + + def __getitem__(self, item): + return self._packages_to_stats[item] + + def __iter__(self): + return iter(self._packages_to_stats) + + def __len__(self): + return len(self._packages_to_stats) + + +class TagIndex(Mapping): + """Maps tags to list of packages.""" + + def __init__(self): + self._tag_dict = collections.defaultdict(list) + + def to_json(self, stream): + json.dump({'tags': self._tag_dict}, stream) + + @staticmethod + def from_json(stream): + d = json.load(stream) + + r = TagIndex() + + for tag, list in d['tags'].items(): + r[tag].extend(list) + + return r + + def __getitem__(self, item): + return self._tag_dict[item] + + def __iter__(self): + return iter(self._tag_dict) + + def __len__(self): + return len(self._tag_dict) + + def update_package(self, pkg_name): + """Updates a package in the tag index. + + Args: + pkg_name (str): name of the package to be removed from the index + + """ + package = path().get(pkg_name) + + # Remove the package from the list of packages, if present + for pkg_list in self._tag_dict.values(): + if pkg_name in pkg_list: + pkg_list.remove(pkg_name) + + # Add it again under the appropriate tags + for tag in getattr(package, 'tags', []): + self._tag_dict[tag].append(package.name) + + +@llnl.util.lang.memoized +def make_provider_index_cache(packages_path, namespace): + """Lazily updates the provider index cache associated with a repository, + if need be, then returns it. Caches results for later look-ups. + + Args: + packages_path: path of the repository + namespace: namespace of the repository + + Returns: + instance of ProviderIndex + """ + # Map that goes from package names to stat info + fast_package_checker = FastPackageChecker(packages_path) + + # Filename of the provider index cache + cache_filename = 'providers/{0}-index.yaml'.format(namespace) + + # Compute which packages needs to be updated in the cache + misc_cache = spack.caches.misc_cache() + index_mtime = misc_cache.mtime(cache_filename) + + needs_update = [ + x for x, sinfo in fast_package_checker.items() + if sinfo.st_mtime > index_mtime + ] + + # Read the old ProviderIndex, or make a new one. + index_existed = misc_cache.init_entry(cache_filename) + + if index_existed and not needs_update: + + # If the provider index exists and doesn't need an update + # just read from it + with misc_cache.read_transaction(cache_filename) as f: + index = ProviderIndex.from_yaml(f) + + else: + + # Otherwise we need a write transaction to update it + with misc_cache.write_transaction(cache_filename) as (old, new): + + index = ProviderIndex.from_yaml(old) if old else ProviderIndex() + + for pkg_name in needs_update: + namespaced_name = '{0}.{1}'.format(namespace, pkg_name) + index.remove_provider(namespaced_name) + index.update(namespaced_name) + + index.to_yaml(new) + + return index + + +@llnl.util.lang.memoized +def make_tag_index_cache(packages_path, namespace): + """Lazily updates the tag index cache associated with a repository, + if need be, then returns it. Caches results for later look-ups. + + Args: + packages_path: path of the repository + namespace: namespace of the repository + + Returns: + instance of TagIndex + """ + # Map that goes from package names to stat info + fast_package_checker = FastPackageChecker(packages_path) + + # Filename of the provider index cache + cache_filename = 'tags/{0}-index.json'.format(namespace) + + # Compute which packages needs to be updated in the cache + misc_cache = spack.caches.misc_cache() + index_mtime = misc_cache.mtime(cache_filename) + + needs_update = [ + x for x, sinfo in fast_package_checker.items() + if sinfo.st_mtime > index_mtime + ] + + # Read the old ProviderIndex, or make a new one. + index_existed = misc_cache.init_entry(cache_filename) + + if index_existed and not needs_update: + + # If the provider index exists and doesn't need an update + # just read from it + with misc_cache.read_transaction(cache_filename) as f: + index = TagIndex.from_json(f) + + else: + + # Otherwise we need a write transaction to update it + with misc_cache.write_transaction(cache_filename) as (old, new): + + index = TagIndex.from_json(old) if old else TagIndex() + + for pkg_name in needs_update: + namespaced_name = '{0}.{1}'.format(namespace, pkg_name) + index.update_package(namespaced_name) + + index.to_json(new) + + return index + + +class RepoPath(object): + """A RepoPath is a list of repos that function as one. + + It functions exactly like a Repo, but it operates on the combined + results of the Repos in its list instead of on a single package + repository. + + Args: + repos (list): list Repo objects or paths to put in this RepoPath + + Optional Args: + repo_namespace (str): super-namespace for all packages in this + RepoPath (used when importing repos as modules) + """ + + def __init__(self, *repos, **kwargs): + self.super_namespace = kwargs.get('namespace', repo_namespace) + + self.repos = [] + self.by_namespace = NamespaceTrie() + self.by_path = {} + + self._all_package_names = None + self._provider_index = None + + # Add each repo to this path. + for repo in repos: + try: + if isinstance(repo, string_types): + repo = Repo(repo, self.super_namespace) + self.put_last(repo) + except RepoError as e: + tty.warn("Failed to initialize repository: '%s'." % repo, + e.message, + "To remove the bad repository, run this command:", + " spack repo rm %s" % repo) + + def _add(self, repo): + """Add a repository to the namespace and path indexes. + + Checks for duplicates -- two repos can't have the same root + directory, and they provide have the same namespace. + + """ + if repo.root in self.by_path: + raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root) + + if repo.namespace in self.by_namespace: + raise DuplicateRepoError( + "Package repos '%s' and '%s' both provide namespace %s" + % (repo.root, self.by_namespace[repo.namespace].root, + repo.namespace)) + + # Add repo to the pkg indexes + self.by_namespace[repo.full_namespace] = repo + self.by_path[repo.root] = repo + + def put_first(self, repo): + """Add repo first in the search path.""" + self._add(repo) + self.repos.insert(0, repo) + + def put_last(self, repo): + """Add repo last in the search path.""" + self._add(repo) + self.repos.append(repo) + + def remove(self, repo): + """Remove a repo from the search path.""" + if repo in self.repos: + self.repos.remove(repo) + + def get_repo(self, namespace, default=NOT_PROVIDED): + """Get a repository by namespace. + + Arguments: + + namespace: + + Look up this namespace in the RepoPath, and return it if found. + + Optional Arguments: + + default: + + If default is provided, return it when the namespace + isn't found. If not, raise an UnknownNamespaceError. + """ + fullspace = '%s.%s' % (self.super_namespace, namespace) + if fullspace not in self.by_namespace: + if default == NOT_PROVIDED: + raise UnknownNamespaceError(namespace) + return default + return self.by_namespace[fullspace] + + def first_repo(self): + """Get the first repo in precedence order.""" + return self.repos[0] if self.repos else None + + def all_package_names(self): + """Return all unique package names in all repositories.""" + if self._all_package_names is None: + all_pkgs = set() + for repo in self.repos: + for name in repo.all_package_names(): + all_pkgs.add(name) + self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower()) + return self._all_package_names + + def packages_with_tags(self, *tags): + r = set() + for repo in self.repos: + r |= set(repo.packages_with_tags(*tags)) + return sorted(r) + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + @property + def provider_index(self): + """Merged ProviderIndex from all Repos in the RepoPath.""" + if self._provider_index is None: + self._provider_index = ProviderIndex() + for repo in reversed(self.repos): + self._provider_index.merge(repo.provider_index) + + return self._provider_index + + @_autospec + def providers_for(self, vpkg_spec): + providers = self.provider_index.providers_for(vpkg_spec) + if not providers: + raise UnknownPackageError(vpkg_spec.name) + return providers + + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def find_module(self, fullname, path=None): + """Implements precedence for overlaid namespaces. + + Loop checks each namespace in self.repos for packages, and + also handles loading empty containing namespaces. + + """ + # namespaces are added to repo, and package modules are leaves. + namespace, dot, module_name = fullname.rpartition('.') + + # If it's a module in some repo, or if it is the repo's + # namespace, let the repo handle it. + for repo in self.repos: + if namespace == repo.full_namespace: + if repo.real_name(module_name): + return repo + elif fullname == repo.full_namespace: + return repo + + # No repo provides the namespace, but it is a valid prefix of + # something in the RepoPath. + if self.by_namespace.is_prefix(fullname): + return self + + return None + + def load_module(self, fullname): + """Handles loading container namespaces when necessary. + + See ``Repo`` for how actual package modules are loaded. + """ + if fullname in sys.modules: + return sys.modules[fullname] + + if not self.by_namespace.is_prefix(fullname): + raise ImportError("No such Spack repo: %s" % fullname) + + module = SpackNamespace(fullname) + module.__loader__ = self + sys.modules[fullname] = module + return module + + def repo_for_pkg(self, spec): + """Given a spec, get the repository for its package.""" + # We don't @_autospec this function b/c it's called very frequently + # and we want to avoid parsing str's into Specs unnecessarily. + namespace = None + if isinstance(spec, spack.spec.Spec): + namespace = spec.namespace + name = spec.name + else: + # handle strings directly for speed instead of @_autospec'ing + namespace, _, name = spec.rpartition('.') + + # If the spec already has a namespace, then return the + # corresponding repo if we know about it. + if namespace: + fullspace = '%s.%s' % (self.super_namespace, namespace) + if fullspace not in self.by_namespace: + raise UnknownNamespaceError(spec.namespace) + return self.by_namespace[fullspace] + + # If there's no namespace, search in the RepoPath. + for repo in self.repos: + if name in repo: + return repo + + # If the package isn't in any repo, return the one with + # highest precedence. This is for commands like `spack edit` + # that can operate on packages that don't exist yet. + return self.first_repo() + + @_autospec + def get(self, spec, new=False): + """Find a repo that contains the supplied spec's package. + + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).get(spec) + + def get_pkg_class(self, pkg_name): + """Find a class for the spec's package and return the class object.""" + return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) + + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).dump_provenance(spec, path) + + def dirname_for_package_name(self, pkg_name): + return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) + + def filename_for_package_name(self, pkg_name): + return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name) + + def exists(self, pkg_name): + """Whether package with the give name exists in the path's repos. + + Note that virtual packages do not "exist". + """ + return any(repo.exists(pkg_name) for repo in self.repos) + + def is_virtual(self, pkg_name): + """True if the package with this name is virtual, False otherwise.""" + return pkg_name in self.provider_index + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + +class Repo(object): + """Class representing a package repository in the filesystem. + + Each package repository must have a top-level configuration file + called `repo.yaml`. + + Currently, `repo.yaml` this must define: + + `namespace`: + A Python namespace where the repository's packages should live. + + """ + + def __init__(self, root, namespace=repo_namespace): + """Instantiate a package repository from a filesystem path. + + Arguments: + root The root directory of the repository. + + namespace A super-namespace that will contain the repo-defined + namespace (this is generally jsut `spack.pkg`). The + super-namespace is Spack's way of separating repositories + from other python namespaces. + + """ + # Root directory, containing _repo.yaml and package dirs + # Allow roots to by spack-relative by starting with '$spack' + self.root = canonicalize_path(root) + + # super-namespace for all packages in the Repo + self.super_namespace = namespace + + # check and raise BadRepoError on fail. + def check(condition, msg): + if not condition: + raise BadRepoError(msg) + + # Validate repository layout. + self.config_file = join_path(self.root, repo_config_name) + check(os.path.isfile(self.config_file), + "No %s found in '%s'" % (repo_config_name, root)) + + self.packages_path = join_path(self.root, packages_dir_name) + check(os.path.isdir(self.packages_path), + "No directory '%s' found in '%s'" % (repo_config_name, root)) + + # Read configuration and validate namespace + config = self._read_config() + check('namespace' in config, '%s must define a namespace.' + % join_path(root, repo_config_name)) + + self.namespace = config['namespace'] + check(re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace), + ("Invalid namespace '%s' in repo '%s'. " + % (self.namespace, self.root)) + + "Namespaces must be valid python identifiers separated by '.'") + + # Set up 'full_namespace' to include the super-namespace + if self.super_namespace: + self.full_namespace = "%s.%s" % ( + self.super_namespace, self.namespace) + else: + self.full_namespace = self.namespace + + # Keep name components around for checking prefixes. + self._names = self.full_namespace.split('.') + + # These are internal cache variables. + self._modules = {} + self._classes = {} + self._instances = {} + + # Maps that goes from package name to corresponding file stat + self._fast_package_checker = None + + # Index of virtual dependencies, computed lazily + self._provider_index = None + + # Index of tags, computed lazily + self._tag_index = None + + # make sure the namespace for packages in this repo exists. + self._create_namespace() + + def _create_namespace(self): + """Create this repo's namespace module and insert it into sys.modules. + + Ensures that modules loaded via the repo have a home, and that + we don't get runtime warnings from Python's module system. + + """ + parent = None + for l in range(1, len(self._names) + 1): + ns = '.'.join(self._names[:l]) + + if ns not in sys.modules: + module = SpackNamespace(ns) + module.__loader__ = self + sys.modules[ns] = module + + # Ensure the namespace is an atrribute of its parent, + # if it has not been set by something else already. + # + # This ensures that we can do things like: + # import spack.pkg.builtin.mpich as mpich + if parent: + modname = self._names[l - 1] + setattr(parent, modname, module) + else: + # no need to set up a module + module = sys.modules[ns] + + # but keep track of the parent in this loop + parent = module + + def real_name(self, import_name): + """Allow users to import Spack packages using Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz + """ + if import_name in self: + return import_name + + options = possible_spack_module_names(import_name) + options.remove(import_name) + for name in options: + if name in self: + return name + return None + + def is_prefix(self, fullname): + """True if fullname is a prefix of this Repo's namespace.""" + parts = fullname.split('.') + return self._names[:len(parts)] == parts + + def find_module(self, fullname, path=None): + """Python find_module import hook. + + Returns this Repo if it can load the module; None if not. + """ + if self.is_prefix(fullname): + return self + + namespace, dot, module_name = fullname.rpartition('.') + if namespace == self.full_namespace: + if self.real_name(module_name): + return self + + return None + + def load_module(self, fullname): + """Python importer load hook. + + Tries to load the module; raises an ImportError if it can't. + """ + if fullname in sys.modules: + return sys.modules[fullname] + + namespace, dot, module_name = fullname.rpartition('.') + + if self.is_prefix(fullname): + module = SpackNamespace(fullname) + + elif namespace == self.full_namespace: + real_name = self.real_name(module_name) + if not real_name: + raise ImportError("No module %s in %s" % (module_name, self)) + module = self._get_pkg_module(real_name) + + else: + raise ImportError("No module %s in %s" % (fullname, self)) + + module.__loader__ = self + sys.modules[fullname] = module + if namespace != fullname: + parent = sys.modules[namespace] + if not hasattr(parent, module_name): + setattr(parent, module_name, module) + + return module + + def _read_config(self): + """Check for a YAML config file in this db's root directory.""" + try: + with open(self.config_file) as reponame_file: + yaml_data = yaml.load(reponame_file) + + if (not yaml_data or 'repo' not in yaml_data or + not isinstance(yaml_data['repo'], dict)): + tty.die("Invalid %s in repository %s" % ( + repo_config_name, self.root)) + + return yaml_data['repo'] + + except IOError: + tty.die("Error reading %s when opening %s" + % (self.config_file, self.root)) + + @_autospec + def get(self, spec): + if not self.exists(spec.name): + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError( + "Repository %s does not contain package %s" + % (self.namespace, spec.fullname)) + + package_class = self.get_pkg_class(spec.name) + try: + return package_class(spec) + except Exception: + if spack.config.get('config:debug'): + sys.excepthook(*sys.exc_info()) + raise FailedConstructorError(spec.fullname, *sys.exc_info()) + + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + # Some preliminary checks. + if spec.virtual: + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError( + "Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) + + # Install any patch files needed by packages. + mkdirp(path) + for spec, patches in spec.package.patches.items(): + for patch in patches: + if patch.path: + if os.path.exists(patch.path): + install(patch.path, path) + else: + tty.warn("Patch file did not exist: %s" % patch.path) + + # Install the package.py file itself. + install(self.filename_for_package_name(spec), path) + + def purge(self): + """Clear entire package instance cache.""" + self._instances.clear() + + @property + def provider_index(self): + """A provider index with names *specific* to this repo.""" + + if self._provider_index is None: + self._provider_index = make_provider_index_cache( + self.packages_path, self.namespace + ) + + return self._provider_index + + @property + def tag_index(self): + """A provider index with names *specific* to this repo.""" + + if self._tag_index is None: + self._tag_index = make_tag_index_cache( + self.packages_path, self.namespace + ) + + return self._tag_index + + @_autospec + def providers_for(self, vpkg_spec): + providers = self.provider_index.providers_for(vpkg_spec) + if not providers: + raise UnknownPackageError(vpkg_spec.name) + return providers + + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def _check_namespace(self, spec): + """Check that the spec's namespace is the same as this repository's.""" + if spec.namespace and spec.namespace != self.namespace: + raise UnknownNamespaceError(spec.namespace) + + @_autospec + def dirname_for_package_name(self, spec): + """Get the directory name for a particular package. This is the + directory that contains its package.py file.""" + self._check_namespace(spec) + return join_path(self.packages_path, spec.name) + + @_autospec + def filename_for_package_name(self, spec): + """Get the filename for the module we should load for a particular + package. Packages for a Repo live in + ``$root//package.py`` + + This will return a proper package.py path even if the + package doesn't exist yet, so callers will need to ensure + the package exists before importing. + """ + self._check_namespace(spec) + pkg_dir = self.dirname_for_package_name(spec.name) + return join_path(pkg_dir, package_file_name) + + @property + def _pkg_checker(self): + if self._fast_package_checker is None: + self._fast_package_checker = FastPackageChecker(self.packages_path) + return self._fast_package_checker + + def all_package_names(self): + """Returns a sorted list of all package names in the Repo.""" + return sorted(self._pkg_checker.keys()) + + def packages_with_tags(self, *tags): + v = set(self.all_package_names()) + index = self.tag_index + + for t in tags: + v &= set(index[t]) + + return sorted(v) + + def all_packages(self): + """Iterator over all packages in the repository. + + Use this with care, because loading packages is slow. + + """ + for name in self.all_package_names(): + yield self.get(name) + + def exists(self, pkg_name): + """Whether a package with the supplied name exists.""" + return pkg_name in self._pkg_checker + + def is_virtual(self, pkg_name): + """True if the package with this name is virtual, False otherwise.""" + return self.provider_index.contains(pkg_name) + + def _get_pkg_module(self, pkg_name): + """Create a module for a particular package. + + This caches the module within this Repo *instance*. It does + *not* add it to ``sys.modules``. So, you can construct + multiple Repos for testing and ensure that the module will be + loaded once per repo. + + """ + if pkg_name not in self._modules: + file_path = self.filename_for_package_name(pkg_name) + + if not os.path.exists(file_path): + raise UnknownPackageError(pkg_name, self) + + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + + # e.g., spack.pkg.builtin.mpich + fullname = "%s.%s" % (self.full_namespace, pkg_name) + + try: + module = imp.load_source(fullname, file_path) + except SyntaxError as e: + # SyntaxError strips the path from the filename so we need to + # manually construct the error message in order to give the + # user the correct package.py where the syntax error is located + raise SyntaxError('invalid syntax in {0:}, line {1:}' + ''.format(file_path, e.lineno)) + module.__package__ = self.full_namespace + module.__loader__ = self + self._modules[pkg_name] = module + + return self._modules[pkg_name] + + def get_pkg_class(self, pkg_name): + """Get the class for the package out of its module. + + First loads (or fetches from cache) a module for the + package. Then extracts the package class from the module + according to Spack's naming convention. + """ + namespace, _, pkg_name = pkg_name.rpartition('.') + if namespace and (namespace != self.namespace): + raise InvalidNamespaceError('Invalid namespace for %s repo: %s' + % (self.namespace, namespace)) + + class_name = mod_to_class(pkg_name) + module = self._get_pkg_module(pkg_name) + + cls = getattr(module, class_name) + if not inspect.isclass(cls): + tty.die("%s.%s is not a class" % (pkg_name, class_name)) + + return cls + + def __str__(self): + return "[Repo '%s' at '%s']" % (self.namespace, self.root) + + def __repr__(self): + return self.__str__() + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + +def create_repo(root, namespace=None): + """Create a new repository in root with the specified namespace. + + If the namespace is not provided, use basename of root. + Return the canonicalized path and namespace of the created repository. + """ + root = canonicalize_path(root) + if not namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + raise InvalidNamespaceError( + "'%s' is not a valid namespace." % namespace) + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + raise BadRepoError('File %s already exists and is not a directory' + % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + raise BadRepoError( + 'Cannot create new repo in %s: cannot access directory.' + % root) + if os.listdir(root): + raise BadRepoError( + 'Cannot create new repo in %s: directory is not empty.' + % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + raise BadRepoError( + "Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + raise BadRepoError('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + return full_path, namespace + + +#: Singleton repo path instance +_path = None + + +def set_path(repo): + """Set the path() singleton to a specific value. + + Overwrite _path and register it as an importer in sys.meta_path if + it is a ``Repo`` or ``RepoPath``. + """ + global _path + _path = repo + + # make the new repo_path an importer if needed + append = isinstance(repo, (Repo, RepoPath)) + if append: + sys.meta_path.append(_path) + return append + + +def path(): + """Get the singleton RepoPath instance for Spack. + + Create a RepoPath, add it to sys.meta_path, and return it. + + TODO: consider not making this a singleton. + """ + if _path is None: + repo_dirs = spack.config.get('repos') + if not repo_dirs: + raise NoRepoConfiguredError( + "Spack configuration contains no package repositories.") + set_path(RepoPath(*repo_dirs)) + + return _path + + +def get(spec): + """Convenience wrapper around ``spack.repo.get()``.""" + return path().get(spec) + + +def all_package_names(): + """Convenience wrapper around ``spack.repo.all_package_names()``.""" + return path().all_package_names() + + +@contextmanager +def swap(repo_path): + """Temporarily use another RepoPath.""" + global _path + + # swap out _path for repo_path + saved = _path + remove_from_meta = set_path(repo_path) + + yield + + # restore _path and sys.meta_path + if remove_from_meta: + sys.meta_path.remove(repo_path) + _path = saved + + +class RepoError(spack.error.SpackError): + """Superclass for repository-related errors.""" + + +class NoRepoConfiguredError(RepoError): + """Raised when there are no repositories configured.""" + + +class InvalidNamespaceError(RepoError): + """Raised when an invalid namespace is encountered.""" + + +class BadRepoError(RepoError): + """Raised when repo layout is invalid.""" + + +class DuplicateRepoError(RepoError): + """Raised when duplicate repos are added to a RepoPath.""" + + +class UnknownEntityError(RepoError): + """Raised when we encounter a package spack doesn't have.""" + + +class UnknownPackageError(UnknownEntityError): + """Raised when we encounter a package spack doesn't have.""" + + def __init__(self, name, repo=None): + msg = None + if repo: + msg = "Package %s not found in repository %s" % (name, repo) + else: + msg = "Package %s not found." % name + super(UnknownPackageError, self).__init__(msg) + self.name = name + + +class UnknownNamespaceError(UnknownEntityError): + """Raised when we encounter an unknown namespace""" + + def __init__(self, namespace): + super(UnknownNamespaceError, self).__init__( + "Unknown namespace: %s" % namespace) + + +class FailedConstructorError(RepoError): + """Raised when a package's class constructor fails.""" + + def __init__(self, name, exc_type, exc_obj, exc_tb): + super(FailedConstructorError, self).__init__( + "Class constructor failed for package '%s'." % name, + '\nCaused by:\n' + + ('%s: %s\n' % (exc_type.__name__, exc_obj)) + + ''.join(traceback.format_tb(exc_tb))) + self.name = name diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py deleted file mode 100644 index b1a6c78249..0000000000 --- a/lib/spack/spack/repository.py +++ /dev/null @@ -1,1153 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/spack/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import collections -import os -import stat -import shutil -import errno -import sys -import inspect -import imp -import re -import traceback -import json - -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping - -from types import ModuleType - -import yaml - -import llnl.util.lang -import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp, join_path, install - -import spack -import spack.config -import spack.caches -import spack.error -import spack.spec -from spack.provider_index import ProviderIndex -from spack.util.path import canonicalize_path -from spack.util.naming import NamespaceTrie, valid_module_name -from spack.util.naming import mod_to_class, possible_spack_module_names - -# -# Super-namespace for all packages. -# Package modules are imported as spack.pkg... -# -repo_namespace = 'spack.pkg' - -# -# These names describe how repos should be laid out in the filesystem. -# -repo_config_name = 'repo.yaml' # Top-level filename for repo config. -repo_index_name = 'index.yaml' # Top-level filename for repository index. -packages_dir_name = 'packages' # Top-level repo directory containing pkgs. -package_file_name = 'package.py' # Filename for packages in a repository. - -# Guaranteed unused default value for some functions. -NOT_PROVIDED = object() - - -def _autospec(function): - """Decorator that automatically converts the argument of a single-arg - function to a Spec.""" - - def converter(self, spec_like, *args, **kwargs): - if not isinstance(spec_like, spack.spec.Spec): - spec_like = spack.spec.Spec(spec_like) - return function(self, spec_like, *args, **kwargs) - return converter - - -class SpackNamespace(ModuleType): - """ Allow lazy loading of modules.""" - - def __init__(self, namespace): - super(SpackNamespace, self).__init__(namespace) - self.__file__ = "(spack namespace)" - self.__path__ = [] - self.__name__ = namespace - self.__package__ = namespace - self.__modules = {} - - def __getattr__(self, name): - """Getattr lazily loads modules if they're not already loaded.""" - submodule = self.__package__ + '.' + name - setattr(self, name, __import__(submodule)) - return getattr(self, name) - - -class FastPackageChecker(Mapping): - """Cache that maps package names to the stats obtained on the - 'package.py' files associated with them. - - For each repository a cache is maintained at class level, and shared among - all instances referring to it. Update of the global cache is done lazily - during instance initialization. - """ - #: Global cache, reused by every instance - _paths_cache = {} - - def __init__(self, packages_path): - - #: The path of the repository managed by this instance - self.packages_path = packages_path - - # If the cache we need is not there yet, then build it appropriately - if packages_path not in self._paths_cache: - self._paths_cache[packages_path] = self._create_new_cache() - - #: Reference to the appropriate entry in the global cache - self._packages_to_stats = self._paths_cache[packages_path] - - def _create_new_cache(self): - """Create a new cache for packages in a repo. - - The implementation here should try to minimize filesystem - calls. At the moment, it is O(number of packages) and makes - about one stat call per package. This is reasonably fast, and - avoids actually importing packages in Spack, which is slow. - """ - # Create a dictionary that will store the mapping between a - # package name and its stat info - cache = {} - for pkg_name in os.listdir(self.packages_path): - # Skip non-directories in the package root. - pkg_dir = join_path(self.packages_path, pkg_name) - - # Warn about invalid names that look like packages. - if not valid_module_name(pkg_name): - msg = 'Skipping package at {0}. ' - msg += '"{1}" is not a valid Spack module name.' - tty.warn(msg.format(pkg_dir, pkg_name)) - continue - - # Construct the file name from the directory - pkg_file = os.path.join( - self.packages_path, pkg_name, package_file_name - ) - - # Use stat here to avoid lots of calls to the filesystem. - try: - sinfo = os.stat(pkg_file) - except OSError as e: - if e.errno == errno.ENOENT: - # No package.py file here. - continue - elif e.errno == errno.EACCES: - tty.warn("Can't read package file %s." % pkg_file) - continue - raise e - - # If it's not a file, skip it. - if stat.S_ISDIR(sinfo.st_mode): - continue - - # If it is a file, then save the stats under the - # appropriate key - cache[pkg_name] = sinfo - - return cache - - def __getitem__(self, item): - return self._packages_to_stats[item] - - def __iter__(self): - return iter(self._packages_to_stats) - - def __len__(self): - return len(self._packages_to_stats) - - -class TagIndex(Mapping): - """Maps tags to list of packages.""" - - def __init__(self): - self._tag_dict = collections.defaultdict(list) - - def to_json(self, stream): - json.dump({'tags': self._tag_dict}, stream) - - @staticmethod - def from_json(stream): - d = json.load(stream) - - r = TagIndex() - - for tag, list in d['tags'].items(): - r[tag].extend(list) - - return r - - def __getitem__(self, item): - return self._tag_dict[item] - - def __iter__(self): - return iter(self._tag_dict) - - def __len__(self): - return len(self._tag_dict) - - def update_package(self, pkg_name): - """Updates a package in the tag index. - - Args: - pkg_name (str): name of the package to be removed from the index - - """ - - package = spack.repo.get(pkg_name) - - # Remove the package from the list of packages, if present - for pkg_list in self._tag_dict.values(): - if pkg_name in pkg_list: - pkg_list.remove(pkg_name) - - # Add it again under the appropriate tags - for tag in getattr(package, 'tags', []): - self._tag_dict[tag].append(package.name) - - -@llnl.util.lang.memoized -def make_provider_index_cache(packages_path, namespace): - """Lazily updates the provider index cache associated with a repository, - if need be, then returns it. Caches results for later look-ups. - - Args: - packages_path: path of the repository - namespace: namespace of the repository - - Returns: - instance of ProviderIndex - """ - # Map that goes from package names to stat info - fast_package_checker = FastPackageChecker(packages_path) - - # Filename of the provider index cache - cache_filename = 'providers/{0}-index.yaml'.format(namespace) - - # Compute which packages needs to be updated in the cache - misc_cache = spack.caches.misc_cache() - index_mtime = misc_cache.mtime(cache_filename) - - needs_update = [ - x for x, sinfo in fast_package_checker.items() - if sinfo.st_mtime > index_mtime - ] - - # Read the old ProviderIndex, or make a new one. - index_existed = misc_cache.init_entry(cache_filename) - - if index_existed and not needs_update: - - # If the provider index exists and doesn't need an update - # just read from it - with misc_cache.read_transaction(cache_filename) as f: - index = ProviderIndex.from_yaml(f) - - else: - - # Otherwise we need a write transaction to update it - with misc_cache.write_transaction(cache_filename) as (old, new): - - index = ProviderIndex.from_yaml(old) if old else ProviderIndex() - - for pkg_name in needs_update: - namespaced_name = '{0}.{1}'.format(namespace, pkg_name) - index.remove_provider(namespaced_name) - index.update(namespaced_name) - - index.to_yaml(new) - - return index - - -@llnl.util.lang.memoized -def make_tag_index_cache(packages_path, namespace): - """Lazily updates the tag index cache associated with a repository, - if need be, then returns it. Caches results for later look-ups. - - Args: - packages_path: path of the repository - namespace: namespace of the repository - - Returns: - instance of TagIndex - """ - # Map that goes from package names to stat info - fast_package_checker = FastPackageChecker(packages_path) - - # Filename of the provider index cache - cache_filename = 'tags/{0}-index.json'.format(namespace) - - # Compute which packages needs to be updated in the cache - misc_cache = spack.caches.misc_cache() - index_mtime = misc_cache.mtime(cache_filename) - - needs_update = [ - x for x, sinfo in fast_package_checker.items() - if sinfo.st_mtime > index_mtime - ] - - # Read the old ProviderIndex, or make a new one. - index_existed = misc_cache.init_entry(cache_filename) - - if index_existed and not needs_update: - - # If the provider index exists and doesn't need an update - # just read from it - with misc_cache.read_transaction(cache_filename) as f: - index = TagIndex.from_json(f) - - else: - - # Otherwise we need a write transaction to update it - with misc_cache.write_transaction(cache_filename) as (old, new): - - index = TagIndex.from_json(old) if old else TagIndex() - - for pkg_name in needs_update: - namespaced_name = '{0}.{1}'.format(namespace, pkg_name) - index.update_package(namespaced_name) - - index.to_json(new) - - return index - - -class RepoPath(object): - """A RepoPath is a list of repos that function as one. - - It functions exactly like a Repo, but it operates on the - combined results of the Repos in its list instead of on a - single package repository. - """ - - def __init__(self, *repo_dirs, **kwargs): - # super-namespace for all packages in the RepoPath - self.super_namespace = kwargs.get('namespace', repo_namespace) - - self.repos = [] - self.by_namespace = NamespaceTrie() - self.by_path = {} - - self._all_package_names = None - self._provider_index = None - - # If repo_dirs is empty, just use the configuration - if not repo_dirs: - import spack.config - repo_dirs = spack.config.get('repos') - if not repo_dirs: - raise NoRepoConfiguredError( - "Spack configuration contains no package repositories.") - - # Add each repo to this path. - for root in repo_dirs: - try: - repo = Repo(root, self.super_namespace) - self.put_last(repo) - except RepoError as e: - tty.warn("Failed to initialize repository at '%s'." % root, - e.message, - "To remove the bad repository, run this command:", - " spack repo rm %s" % root) - - def swap(self, other): - """Convenience function to make swapping repositories easier. - - This is currently used by mock tests. - TODO: Maybe there is a cleaner way. - - """ - attrs = ['repos', - 'by_namespace', - 'by_path', - '_all_package_names', - '_provider_index'] - for attr in attrs: - tmp = getattr(self, attr) - setattr(self, attr, getattr(other, attr)) - setattr(other, attr, tmp) - - def _add(self, repo): - """Add a repository to the namespace and path indexes. - - Checks for duplicates -- two repos can't have the same root - directory, and they provide have the same namespace. - - """ - if repo.root in self.by_path: - raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root) - - if repo.namespace in self.by_namespace: - raise DuplicateRepoError( - "Package repos '%s' and '%s' both provide namespace %s" - % (repo.root, self.by_namespace[repo.namespace].root, - repo.namespace)) - - # Add repo to the pkg indexes - self.by_namespace[repo.full_namespace] = repo - self.by_path[repo.root] = repo - - def put_first(self, repo): - """Add repo first in the search path.""" - self._add(repo) - self.repos.insert(0, repo) - - def put_last(self, repo): - """Add repo last in the search path.""" - self._add(repo) - self.repos.append(repo) - - def remove(self, repo): - """Remove a repo from the search path.""" - if repo in self.repos: - self.repos.remove(repo) - - def get_repo(self, namespace, default=NOT_PROVIDED): - """Get a repository by namespace. - - Arguments: - - namespace: - - Look up this namespace in the RepoPath, and return it if found. - - Optional Arguments: - - default: - - If default is provided, return it when the namespace - isn't found. If not, raise an UnknownNamespaceError. - """ - fullspace = '%s.%s' % (self.super_namespace, namespace) - if fullspace not in self.by_namespace: - if default == NOT_PROVIDED: - raise UnknownNamespaceError(namespace) - return default - return self.by_namespace[fullspace] - - def first_repo(self): - """Get the first repo in precedence order.""" - return self.repos[0] if self.repos else None - - def all_package_names(self): - """Return all unique package names in all repositories.""" - if self._all_package_names is None: - all_pkgs = set() - for repo in self.repos: - for name in repo.all_package_names(): - all_pkgs.add(name) - self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower()) - return self._all_package_names - - def packages_with_tags(self, *tags): - r = set() - for repo in self.repos: - r |= set(repo.packages_with_tags(*tags)) - return sorted(r) - - def all_packages(self): - for name in self.all_package_names(): - yield self.get(name) - - @property - def provider_index(self): - """Merged ProviderIndex from all Repos in the RepoPath.""" - if self._provider_index is None: - self._provider_index = ProviderIndex() - for repo in reversed(self.repos): - self._provider_index.merge(repo.provider_index) - - return self._provider_index - - @_autospec - def providers_for(self, vpkg_spec): - providers = self.provider_index.providers_for(vpkg_spec) - if not providers: - raise UnknownPackageError(vpkg_spec.name) - return providers - - @_autospec - def extensions_for(self, extendee_spec): - return [p for p in self.all_packages() if p.extends(extendee_spec)] - - def find_module(self, fullname, path=None): - """Implements precedence for overlaid namespaces. - - Loop checks each namespace in self.repos for packages, and - also handles loading empty containing namespaces. - - """ - # namespaces are added to repo, and package modules are leaves. - namespace, dot, module_name = fullname.rpartition('.') - - # If it's a module in some repo, or if it is the repo's - # namespace, let the repo handle it. - for repo in self.repos: - if namespace == repo.full_namespace: - if repo.real_name(module_name): - return repo - elif fullname == repo.full_namespace: - return repo - - # No repo provides the namespace, but it is a valid prefix of - # something in the RepoPath. - if self.by_namespace.is_prefix(fullname): - return self - - return None - - def load_module(self, fullname): - """Handles loading container namespaces when necessary. - - See ``Repo`` for how actual package modules are loaded. - """ - if fullname in sys.modules: - return sys.modules[fullname] - - if not self.by_namespace.is_prefix(fullname): - raise ImportError("No such Spack repo: %s" % fullname) - - module = SpackNamespace(fullname) - module.__loader__ = self - sys.modules[fullname] = module - return module - - def repo_for_pkg(self, spec): - """Given a spec, get the repository for its package.""" - # We don't @_autospec this function b/c it's called very frequently - # and we want to avoid parsing str's into Specs unnecessarily. - namespace = None - if isinstance(spec, spack.spec.Spec): - namespace = spec.namespace - name = spec.name - else: - # handle strings directly for speed instead of @_autospec'ing - namespace, _, name = spec.rpartition('.') - - # If the spec already has a namespace, then return the - # corresponding repo if we know about it. - if namespace: - fullspace = '%s.%s' % (self.super_namespace, namespace) - if fullspace not in self.by_namespace: - raise UnknownNamespaceError(spec.namespace) - return self.by_namespace[fullspace] - - # If there's no namespace, search in the RepoPath. - for repo in self.repos: - if name in repo: - return repo - - # If the package isn't in any repo, return the one with - # highest precedence. This is for commands like `spack edit` - # that can operate on packages that don't exist yet. - return self.first_repo() - - @_autospec - def get(self, spec, new=False): - """Find a repo that contains the supplied spec's package. - - Raises UnknownPackageError if not found. - """ - return self.repo_for_pkg(spec).get(spec) - - def get_pkg_class(self, pkg_name): - """Find a class for the spec's package and return the class object.""" - return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) - - @_autospec - def dump_provenance(self, spec, path): - """Dump provenance information for a spec to a particular path. - - This dumps the package file and any associated patch files. - Raises UnknownPackageError if not found. - """ - return self.repo_for_pkg(spec).dump_provenance(spec, path) - - def dirname_for_package_name(self, pkg_name): - return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) - - def filename_for_package_name(self, pkg_name): - return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name) - - def exists(self, pkg_name): - """Whether package with the give name exists in the path's repos. - - Note that virtual packages do not "exist". - """ - return any(repo.exists(pkg_name) for repo in self.repos) - - def is_virtual(self, pkg_name): - """True if the package with this name is virtual, False otherwise.""" - return pkg_name in self.provider_index - - def __contains__(self, pkg_name): - return self.exists(pkg_name) - - -class Repo(object): - """Class representing a package repository in the filesystem. - - Each package repository must have a top-level configuration file - called `repo.yaml`. - - Currently, `repo.yaml` this must define: - - `namespace`: - A Python namespace where the repository's packages should live. - - """ - - def __init__(self, root, namespace=repo_namespace): - """Instantiate a package repository from a filesystem path. - - Arguments: - root The root directory of the repository. - - namespace A super-namespace that will contain the repo-defined - namespace (this is generally jsut `spack.pkg`). The - super-namespace is Spack's way of separating repositories - from other python namespaces. - - """ - # Root directory, containing _repo.yaml and package dirs - # Allow roots to by spack-relative by starting with '$spack' - self.root = canonicalize_path(root) - - # super-namespace for all packages in the Repo - self.super_namespace = namespace - - # check and raise BadRepoError on fail. - def check(condition, msg): - if not condition: - raise BadRepoError(msg) - - # Validate repository layout. - self.config_file = join_path(self.root, repo_config_name) - check(os.path.isfile(self.config_file), - "No %s found in '%s'" % (repo_config_name, root)) - - self.packages_path = join_path(self.root, packages_dir_name) - check(os.path.isdir(self.packages_path), - "No directory '%s' found in '%s'" % (repo_config_name, root)) - - # Read configuration and validate namespace - config = self._read_config() - check('namespace' in config, '%s must define a namespace.' - % join_path(root, repo_config_name)) - - self.namespace = config['namespace'] - check(re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace), - ("Invalid namespace '%s' in repo '%s'. " - % (self.namespace, self.root)) + - "Namespaces must be valid python identifiers separated by '.'") - - # Set up 'full_namespace' to include the super-namespace - if self.super_namespace: - self.full_namespace = "%s.%s" % ( - self.super_namespace, self.namespace) - else: - self.full_namespace = self.namespace - - # Keep name components around for checking prefixes. - self._names = self.full_namespace.split('.') - - # These are internal cache variables. - self._modules = {} - self._classes = {} - self._instances = {} - - # Maps that goes from package name to corresponding file stat - self._fast_package_checker = None - - # Index of virtual dependencies, computed lazily - self._provider_index = None - - # Index of tags, computed lazily - self._tag_index = None - - # make sure the namespace for packages in this repo exists. - self._create_namespace() - - def _create_namespace(self): - """Create this repo's namespace module and insert it into sys.modules. - - Ensures that modules loaded via the repo have a home, and that - we don't get runtime warnings from Python's module system. - - """ - parent = None - for l in range(1, len(self._names) + 1): - ns = '.'.join(self._names[:l]) - - if ns not in sys.modules: - module = SpackNamespace(ns) - module.__loader__ = self - sys.modules[ns] = module - - # Ensure the namespace is an atrribute of its parent, - # if it has not been set by something else already. - # - # This ensures that we can do things like: - # import spack.pkg.builtin.mpich as mpich - if parent: - modname = self._names[l - 1] - setattr(parent, modname, module) - else: - # no need to set up a module - module = sys.modules[ns] - - # but keep track of the parent in this loop - parent = module - - def real_name(self, import_name): - """Allow users to import Spack packages using Python identifiers. - - A python identifier might map to many different Spack package - names due to hyphen/underscore ambiguity. - - Easy example: - num3proxy -> 3proxy - - Ambiguous: - foo_bar -> foo_bar, foo-bar - - More ambiguous: - foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz - """ - if import_name in self: - return import_name - - options = possible_spack_module_names(import_name) - options.remove(import_name) - for name in options: - if name in self: - return name - return None - - def is_prefix(self, fullname): - """True if fullname is a prefix of this Repo's namespace.""" - parts = fullname.split('.') - return self._names[:len(parts)] == parts - - def find_module(self, fullname, path=None): - """Python find_module import hook. - - Returns this Repo if it can load the module; None if not. - """ - if self.is_prefix(fullname): - return self - - namespace, dot, module_name = fullname.rpartition('.') - if namespace == self.full_namespace: - if self.real_name(module_name): - return self - - return None - - def load_module(self, fullname): - """Python importer load hook. - - Tries to load the module; raises an ImportError if it can't. - """ - if fullname in sys.modules: - return sys.modules[fullname] - - namespace, dot, module_name = fullname.rpartition('.') - - if self.is_prefix(fullname): - module = SpackNamespace(fullname) - - elif namespace == self.full_namespace: - real_name = self.real_name(module_name) - if not real_name: - raise ImportError("No module %s in %s" % (module_name, self)) - module = self._get_pkg_module(real_name) - - else: - raise ImportError("No module %s in %s" % (fullname, self)) - - module.__loader__ = self - sys.modules[fullname] = module - if namespace != fullname: - parent = sys.modules[namespace] - if not hasattr(parent, module_name): - setattr(parent, module_name, module) - - return module - - def _read_config(self): - """Check for a YAML config file in this db's root directory.""" - try: - with open(self.config_file) as reponame_file: - yaml_data = yaml.load(reponame_file) - - if (not yaml_data or 'repo' not in yaml_data or - not isinstance(yaml_data['repo'], dict)): - tty.die("Invalid %s in repository %s" % ( - repo_config_name, self.root)) - - return yaml_data['repo'] - - except IOError: - tty.die("Error reading %s when opening %s" - % (self.config_file, self.root)) - - @_autospec - def get(self, spec): - if not self.exists(spec.name): - raise UnknownPackageError(spec.name) - - if spec.namespace and spec.namespace != self.namespace: - raise UnknownPackageError( - "Repository %s does not contain package %s" - % (self.namespace, spec.fullname)) - - package_class = self.get_pkg_class(spec.name) - try: - return package_class(spec) - except Exception: - if spack.config.get('config:debug'): - sys.excepthook(*sys.exc_info()) - raise FailedConstructorError(spec.fullname, *sys.exc_info()) - - @_autospec - def dump_provenance(self, spec, path): - """Dump provenance information for a spec to a particular path. - - This dumps the package file and any associated patch files. - Raises UnknownPackageError if not found. - """ - # Some preliminary checks. - if spec.virtual: - raise UnknownPackageError(spec.name) - - if spec.namespace and spec.namespace != self.namespace: - raise UnknownPackageError( - "Repository %s does not contain package %s." - % (self.namespace, spec.fullname)) - - # Install any patch files needed by packages. - mkdirp(path) - for spec, patches in spec.package.patches.items(): - for patch in patches: - if patch.path: - if os.path.exists(patch.path): - install(patch.path, path) - else: - tty.warn("Patch file did not exist: %s" % patch.path) - - # Install the package.py file itself. - install(self.filename_for_package_name(spec), path) - - def purge(self): - """Clear entire package instance cache.""" - self._instances.clear() - - @property - def provider_index(self): - """A provider index with names *specific* to this repo.""" - - if self._provider_index is None: - self._provider_index = make_provider_index_cache( - self.packages_path, self.namespace - ) - - return self._provider_index - - @property - def tag_index(self): - """A provider index with names *specific* to this repo.""" - - if self._tag_index is None: - self._tag_index = make_tag_index_cache( - self.packages_path, self.namespace - ) - - return self._tag_index - - @_autospec - def providers_for(self, vpkg_spec): - providers = self.provider_index.providers_for(vpkg_spec) - if not providers: - raise UnknownPackageError(vpkg_spec.name) - return providers - - @_autospec - def extensions_for(self, extendee_spec): - return [p for p in self.all_packages() if p.extends(extendee_spec)] - - def _check_namespace(self, spec): - """Check that the spec's namespace is the same as this repository's.""" - if spec.namespace and spec.namespace != self.namespace: - raise UnknownNamespaceError(spec.namespace) - - @_autospec - def dirname_for_package_name(self, spec): - """Get the directory name for a particular package. This is the - directory that contains its package.py file.""" - self._check_namespace(spec) - return join_path(self.packages_path, spec.name) - - @_autospec - def filename_for_package_name(self, spec): - """Get the filename for the module we should load for a particular - package. Packages for a Repo live in - ``$root//package.py`` - - This will return a proper package.py path even if the - package doesn't exist yet, so callers will need to ensure - the package exists before importing. - """ - self._check_namespace(spec) - pkg_dir = self.dirname_for_package_name(spec.name) - return join_path(pkg_dir, package_file_name) - - @property - def _pkg_checker(self): - if self._fast_package_checker is None: - self._fast_package_checker = FastPackageChecker(self.packages_path) - return self._fast_package_checker - - def all_package_names(self): - """Returns a sorted list of all package names in the Repo.""" - return sorted(self._pkg_checker.keys()) - - def packages_with_tags(self, *tags): - v = set(self.all_package_names()) - index = self.tag_index - - for t in tags: - v &= set(index[t]) - - return sorted(v) - - def all_packages(self): - """Iterator over all packages in the repository. - - Use this with care, because loading packages is slow. - - """ - for name in self.all_package_names(): - yield self.get(name) - - def exists(self, pkg_name): - """Whether a package with the supplied name exists.""" - return pkg_name in self._pkg_checker - - def is_virtual(self, pkg_name): - """True if the package with this name is virtual, False otherwise.""" - return self.provider_index.contains(pkg_name) - - def _get_pkg_module(self, pkg_name): - """Create a module for a particular package. - - This caches the module within this Repo *instance*. It does - *not* add it to ``sys.modules``. So, you can construct - multiple Repos for testing and ensure that the module will be - loaded once per repo. - - """ - if pkg_name not in self._modules: - file_path = self.filename_for_package_name(pkg_name) - - if not os.path.exists(file_path): - raise UnknownPackageError(pkg_name, self) - - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - - # e.g., spack.pkg.builtin.mpich - fullname = "%s.%s" % (self.full_namespace, pkg_name) - - try: - module = imp.load_source(fullname, file_path) - except SyntaxError as e: - # SyntaxError strips the path from the filename so we need to - # manually construct the error message in order to give the - # user the correct package.py where the syntax error is located - raise SyntaxError('invalid syntax in {0:}, line {1:}' - ''.format(file_path, e.lineno)) - module.__package__ = self.full_namespace - module.__loader__ = self - self._modules[pkg_name] = module - - return self._modules[pkg_name] - - def get_pkg_class(self, pkg_name): - """Get the class for the package out of its module. - - First loads (or fetches from cache) a module for the - package. Then extracts the package class from the module - according to Spack's naming convention. - """ - namespace, _, pkg_name = pkg_name.rpartition('.') - if namespace and (namespace != self.namespace): - raise InvalidNamespaceError('Invalid namespace for %s repo: %s' - % (self.namespace, namespace)) - - class_name = mod_to_class(pkg_name) - module = self._get_pkg_module(pkg_name) - - cls = getattr(module, class_name) - if not inspect.isclass(cls): - tty.die("%s.%s is not a class" % (pkg_name, class_name)) - - return cls - - def __str__(self): - return "[Repo '%s' at '%s']" % (self.namespace, self.root) - - def __repr__(self): - return self.__str__() - - def __contains__(self, pkg_name): - return self.exists(pkg_name) - - -def create_repo(root, namespace=None): - """Create a new repository in root with the specified namespace. - - If the namespace is not provided, use basename of root. - Return the canonicalized path and namespace of the created repository. - """ - root = canonicalize_path(root) - if not namespace: - namespace = os.path.basename(root) - - if not re.match(r'\w[\.\w-]*', namespace): - raise InvalidNamespaceError( - "'%s' is not a valid namespace." % namespace) - - existed = False - if os.path.exists(root): - if os.path.isfile(root): - raise BadRepoError('File %s already exists and is not a directory' - % root) - elif os.path.isdir(root): - if not os.access(root, os.R_OK | os.W_OK): - raise BadRepoError( - 'Cannot create new repo in %s: cannot access directory.' - % root) - if os.listdir(root): - raise BadRepoError( - 'Cannot create new repo in %s: directory is not empty.' - % root) - existed = True - - full_path = os.path.realpath(root) - parent = os.path.dirname(full_path) - if not os.access(parent, os.R_OK | os.W_OK): - raise BadRepoError( - "Cannot create repository in %s: can't access parent!" % root) - - try: - config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) - - mkdirp(packages_path) - with open(config_path, 'w') as config: - config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) - - except (IOError, OSError) as e: - raise BadRepoError('Failed to create new repository in %s.' % root, - "Caused by %s: %s" % (type(e), e)) - - # try to clean up. - if existed: - shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(packages_path, ignore_errors=True) - else: - shutil.rmtree(root, ignore_errors=True) - - return full_path, namespace - - -class RepoError(spack.error.SpackError): - """Superclass for repository-related errors.""" - - -class NoRepoConfiguredError(RepoError): - """Raised when there are no repositories configured.""" - - -class InvalidNamespaceError(RepoError): - """Raised when an invalid namespace is encountered.""" - - -class BadRepoError(RepoError): - """Raised when repo layout is invalid.""" - - -class DuplicateRepoError(RepoError): - """Raised when duplicate repos are added to a RepoPath.""" - - -class UnknownEntityError(RepoError): - """Raised when we encounter a package spack doesn't have.""" - - -class UnknownPackageError(UnknownEntityError): - """Raised when we encounter a package spack doesn't have.""" - - def __init__(self, name, repo=None): - msg = None - if repo: - msg = "Package %s not found in repository %s" % (name, repo) - else: - msg = "Package %s not found." % name - super(UnknownPackageError, self).__init__(msg) - self.name = name - - -class UnknownNamespaceError(UnknownEntityError): - """Raised when we encounter an unknown namespace""" - - def __init__(self, namespace): - super(UnknownNamespaceError, self).__init__( - "Unknown namespace: %s" % namespace) - - -class FailedConstructorError(RepoError): - """Raised when a package's class constructor fails.""" - - def __init__(self, name, exc_type, exc_obj, exc_tb): - super(FailedConstructorError, self).__init__( - "Class constructor failed for package '%s'." % name, - '\nCaused by:\n' + - ('%s: %s\n' % (exc_type.__name__, exc_obj)) + - ''.join(traceback.format_tb(exc_tb))) - self.name = name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 87800504f7..896927e3fd 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -114,12 +114,11 @@ from llnl.util.lang import key_ordering, HashableMap, ObjectWrapper, dedupe from llnl.util.lang import check_kwargs from llnl.util.tty.color import cwrite, colorize, cescape, get_color_when -import spack - import spack.architecture import spack.compilers as compilers import spack.error import spack.parse +import spack.repo import spack.store import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml @@ -1229,7 +1228,7 @@ class Spec(object): """Internal package call gets only the class object for a package. Use this to just get package metadata. """ - return spack.repo.get_pkg_class(self.fullname) + return spack.repo.path().get_pkg_class(self.fullname) @property def virtual(self): @@ -1245,7 +1244,7 @@ class Spec(object): @staticmethod def is_virtual(name): """Test if a name is virtual without requiring a Spec.""" - return (name is not None) and (not spack.repo.exists(name)) + return (name is not None) and (not spack.repo.path().exists(name)) @property def concrete(self): @@ -1850,7 +1849,7 @@ class Spec(object): # we can do it as late as possible to allow as much # compatibility across repositories as possible. if s.namespace is None: - s.namespace = spack.repo.repo_for_pkg(s.name).namespace + s.namespace = spack.repo.path().repo_for_pkg(s.name).namespace if s.concrete: continue @@ -2456,7 +2455,7 @@ class Spec(object): if not self.virtual and other.virtual: try: pkg = spack.repo.get(self.fullname) - except spack.repository.UnknownEntityError: + except spack.repo.UnknownEntityError: # If we can't get package info on this spec, don't treat # it as a provider of this vdep. return False diff --git a/lib/spack/spack/test/build_systems.py b/lib/spack/spack/test/build_systems.py index b90a084d09..4b0ca19aa2 100644 --- a/lib/spack/spack/test/build_systems.py +++ b/lib/spack/spack/test/build_systems.py @@ -26,6 +26,7 @@ import spack import pytest +import spack.repo from spack.build_environment import get_std_cmake_args from spack.spec import Spec diff --git a/lib/spack/spack/test/cmd/flake8.py b/lib/spack/spack/test/cmd/flake8.py index 646568117b..c6e19f4a3f 100644 --- a/lib/spack/spack/test/cmd/flake8.py +++ b/lib/spack/spack/test/cmd/flake8.py @@ -31,7 +31,7 @@ from llnl.util.filesystem import FileFilter import spack.paths from spack.cmd.flake8 import flake8, setup_parser, changed_files -from spack.repository import Repo +from spack.repo import Repo from spack.util.executable import which diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 185a08f4f6..e7b0fb9693 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -165,8 +165,8 @@ def test_install_output_on_build_error(mock_packages, mock_archive, mock_fetch, @pytest.mark.disable_clean_stage_check -def test_install_output_on_python_error(mock_packages, mock_archive, mock_fetch, - config, install_mockery): +def test_install_output_on_python_error( + mock_packages, mock_archive, mock_fetch, config, install_mockery): out = install('failing-build', fail_on_error=False) assert isinstance(install.error, spack.build_environment.ChildError) assert install.error.name == 'InstallError' diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 88ec9133c5..2a7943c894 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -24,8 +24,9 @@ ############################################################################## import pytest import llnl.util.lang -import spack + import spack.architecture +import spack.repo from spack.concretize import find_spec from spack.spec import Spec, CompilerSpec @@ -161,7 +162,7 @@ class TestConcretize(object): """Make sure insufficient versions of MPI are not in providers list when we ask for some advanced version. """ - repo = spack.repo + repo = spack.repo.path() assert not any( s.satisfies('mpich2@:1.0') for s in repo.providers_for('mpi@2.1') ) @@ -181,7 +182,7 @@ class TestConcretize(object): def test_provides_handles_multiple_providers_of_same_vesrion(self): """ """ - providers = spack.repo.providers_for('mpi@3.0') + providers = spack.repo.path().providers_for('mpi@3.0') # Note that providers are repo-specific, so we don't misinterpret # providers, but vdeps are not namespace-specific, so we can @@ -216,8 +217,6 @@ class TestConcretize(object): information from the root even when partial architecture information is provided by an intermediate dependency. """ - saved_repo = spack.repo - default_dep = ('link', 'build') bazpkg = MockPackage('bazpkg', [], []) @@ -225,9 +224,7 @@ class TestConcretize(object): foopkg = MockPackage('foopkg', [barpkg], [default_dep]) mock_repo = MockPackageMultiRepo([foopkg, barpkg, bazpkg]) - spack.repo = mock_repo - - try: + with spack.repo.swap(mock_repo): spec = Spec('foopkg %clang@3.3 os=CNL target=footar' + ' ^barpkg os=SuSE11 ^bazpkg os=be') spec.concretize() @@ -235,9 +232,6 @@ class TestConcretize(object): for s in spec.traverse(root=False): assert s.architecture.target == spec.architecture.target - finally: - spack.repo = saved_repo - def test_compiler_flags_from_user_are_grouped(self): spec = Spec('a%gcc cflags="-O -foo-flag foo-val" platform=test') spec.concretize() diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py index 2c0b36a6a8..a7ac13a64a 100644 --- a/lib/spack/spack/test/concretize_preferences.py +++ b/lib/spack/spack/test/concretize_preferences.py @@ -25,6 +25,7 @@ import pytest import spack.package_prefs +import spack.repo import spack.util.spack_yaml as syaml from spack.config import ConfigScope from spack.spec import Spec @@ -41,7 +42,7 @@ def concretize_scope(config, tmpdir): config.pop_scope() spack.package_prefs.PackagePrefs.clear_caches() - spack.repo._provider_index = None + spack.repo.path()._provider_index = None def concretize(abstract_spec): diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index df88a1efaf..46767133b2 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -33,7 +33,6 @@ import pytest from llnl.util.filesystem import remove_linked_tree -import spack import spack.architecture import spack.config import spack.caches @@ -41,7 +40,7 @@ import spack.database import spack.directory_layout import spack.paths import spack.platforms.test -import spack.repository +import spack.repo import spack.stage import spack.util.ordereddict import spack.util.executable @@ -173,25 +172,23 @@ spack.architecture.platform = lambda: spack.platforms.test.Test() @pytest.fixture(scope='session') def repo_path(): """Session scoped RepoPath object pointing to the mock repository""" - return spack.repository.RepoPath(spack.paths.mock_packages_path) + return spack.repo.RepoPath(spack.paths.mock_packages_path) @pytest.fixture(scope='module') def mock_packages(repo_path): """Use the 'builtin.mock' repository instead of 'builtin'""" mock_repo = copy.deepcopy(repo_path) - spack.repo.swap(mock_repo) - yield - spack.repo.swap(mock_repo) + with spack.repo.swap(mock_repo): + yield @pytest.fixture(scope='function') def mutable_mock_packages(mock_packages, repo_path): """Function-scoped mock packages, for tests that need to modify them.""" mock_repo = copy.deepcopy(repo_path) - spack.repo.swap(mock_repo) - yield - spack.repo.swap(mock_repo) + with spack.repo.swap(mock_repo): + yield @pytest.fixture(scope='session') @@ -221,6 +218,7 @@ def configuration_dir(tmpdir_factory, linux_os): compilers_yaml = test_path.join('data', 'compilers.yaml') packages_yaml = test_path.join('data', 'packages.yaml') config_yaml = test_path.join('data', 'config.yaml') + repos_yaml = test_path.join('data', 'repos.yaml') # Create temporary 'site' and 'user' folders tmpdir.ensure('site', dir=True) @@ -229,6 +227,7 @@ def configuration_dir(tmpdir_factory, linux_os): # Copy the configurations that don't need further work packages_yaml.copy(tmpdir.join('site', 'packages.yaml')) config_yaml.copy(tmpdir.join('site', 'config.yaml')) + repos_yaml.copy(tmpdir.join('site', 'repos.yaml')) # Write the one that needs modifications content = ''.join(compilers_yaml.read()).format(linux_os) @@ -245,8 +244,6 @@ def config(configuration_dir): real_configuration = spack.config._configuration - print real_configuration - scopes = [ spack.config.ConfigScope(name, str(configuration_dir.join(name))) for name in ['site', 'system', 'user']] diff --git a/lib/spack/spack/test/data/repos.yaml b/lib/spack/spack/test/data/repos.yaml new file mode 100644 index 0000000000..4fbbfe9d62 --- /dev/null +++ b/lib/spack/spack/test/data/repos.yaml @@ -0,0 +1,2 @@ +repos: + - $spack/var/spack/repos/builtin diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 186d96eb3a..147a61ad81 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -33,7 +33,7 @@ import pytest from llnl.util.tty.colify import colify -import spack +import spack.repo import spack.store from spack.test.conftest import MockPackageMultiRepo from spack.util.executable import Executable @@ -432,17 +432,12 @@ def test_110_no_write_with_exception_on_install(database): def test_115_reindex_with_packages_not_in_repo(database, refresh_db_on_exit): install_db = database.mock.db - saved_repo = spack.repo # Dont add any package definitions to this repository, the idea is that # packages should not have to be defined in the repository once they # are installed - mock_repo = MockPackageMultiRepo([]) - try: - spack.repo = mock_repo + with spack.repo.swap(MockPackageMultiRepo([])): spack.store.db.reindex(spack.store.layout) _check_db_sanity(install_db) - finally: - spack.repo = saved_repo def test_external_entries_in_db(database): diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index e1dd72b61d..de4000b651 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -30,11 +30,10 @@ import pytest from llnl.util.filesystem import join_path -import spack import spack.paths +import spack.repo from spack.directory_layout import YamlDirectoryLayout from spack.directory_layout import InvalidDirectoryLayoutParametersError -from spack.repository import RepoPath from spack.spec import Spec # number of packages to test (to reduce test time) @@ -105,7 +104,7 @@ def test_read_and_write_spec( layout. """ layout, tmpdir = layout_and_dir - packages = list(spack.repo.all_packages())[:max_packages] + packages = list(spack.repo.path().all_packages())[:max_packages] for pkg in packages: if pkg.name.startswith('external'): @@ -187,7 +186,7 @@ def test_handle_unknown_package( or query them again if the package goes away. """ layout, _ = layout_and_dir - mock_db = RepoPath(spack.paths.mock_packages_path) + mock_db = spack.repo.RepoPath(spack.paths.mock_packages_path) not_in_mock = set.difference( set(spack.repo.all_package_names()), @@ -209,28 +208,25 @@ def test_handle_unknown_package( layout.create_install_directory(spec) installed_specs[spec] = layout.path_for_spec(spec) - spack.repo.swap(mock_db) + with spack.repo.swap(mock_db): + # Now check that even without the package files, we know + # enough to read a spec from the spec file. + for spec, path in installed_specs.items(): + spec_from_file = layout.read_spec( + join_path(path, '.spack', 'spec.yaml')) - # Now check that even without the package files, we know - # enough to read a spec from the spec file. - for spec, path in installed_specs.items(): - spec_from_file = layout.read_spec( - join_path(path, '.spack', 'spec.yaml') - ) - # To satisfy these conditions, directory layouts need to - # read in concrete specs from their install dirs somehow. - assert path == layout.path_for_spec(spec_from_file) - assert spec == spec_from_file - assert spec.eq_dag(spec_from_file) - assert spec.dag_hash() == spec_from_file.dag_hash() - - spack.repo.swap(mock_db) + # To satisfy these conditions, directory layouts need to + # read in concrete specs from their install dirs somehow. + assert path == layout.path_for_spec(spec_from_file) + assert spec == spec_from_file + assert spec.eq_dag(spec_from_file) + assert spec.dag_hash() == spec_from_file.dag_hash() def test_find(layout_and_dir, config, mock_packages): """Test that finding specs within an install layout works.""" layout, _ = layout_and_dir - packages = list(spack.repo.all_packages())[:max_packages] + packages = list(spack.repo.path().all_packages())[:max_packages] # Create install prefixes for all packages in the list installed_specs = {} diff --git a/lib/spack/spack/test/flag_handlers.py b/lib/spack/spack/test/flag_handlers.py index d9433a2c3c..cd6955646d 100644 --- a/lib/spack/spack/test/flag_handlers.py +++ b/lib/spack/spack/test/flag_handlers.py @@ -26,6 +26,7 @@ import pytest import os import spack.spec +import spack.repo import spack.build_environment diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 603f54d975..127096b52b 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -28,7 +28,7 @@ import pytest from llnl.util.filesystem import working_dir, join_path, touch -import spack +import spack.repo import spack.config from spack.spec import Spec from spack.version import ver diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 787ca00280..7c3878e9f9 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -28,7 +28,7 @@ import pytest from llnl.util.filesystem import working_dir, join_path, touch -import spack +import spack.repo import spack.config from spack.spec import Spec from spack.version import ver diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 5f1b59be2b..739f9cbc75 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -25,7 +25,7 @@ import os import pytest -import spack +import spack.repo import spack.store from spack.spec import Spec diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 616d13c151..5b397cc129 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -28,7 +28,7 @@ import pytest from llnl.util.filesystem import join_path -import spack +import spack.repo import spack.mirror import spack.util.executable from spack.spec import Spec diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7d90c12c85..e9262f1c0a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -23,8 +23,9 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """Test for multi_method dispatch.""" -import spack import pytest + +import spack.repo from spack.multimethod import NoSuchMethodError diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 747552df97..849ebc9053 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -27,9 +27,9 @@ import re import pytest -import spack +import spack.repo from spack.paths import mock_packages_path -from spack.repository import RepoPath +from spack.repo import RepoPath def check_db(): @@ -47,9 +47,8 @@ def test_get_all_packages(): def test_get_all_mock_packages(): """Get the mock packages once each too.""" db = RepoPath(mock_packages_path) - spack.repo.swap(db) - check_db() - spack.repo.swap(db) + with spack.repo.swap(db): + check_db() def test_all_versions_are_lowercase(): @@ -66,7 +65,7 @@ def test_all_virtual_packages_have_default_providers(): """All virtual packages must have a default provider explicitly set.""" defaults = spack.config.get('packages', scope='defaults') default_providers = defaults['all']['providers'] - providers = spack.repo.provider_index.providers + providers = spack.repo.path().provider_index.providers for provider in providers: assert provider in default_providers diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index fde6e7d79d..5c71a81101 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -26,9 +26,8 @@ import pytest from llnl.util.filesystem import join_path -import spack +import spack.repo from spack.paths import mock_packages_path -from spack.repository import Repo from spack.util.naming import mod_to_class from spack.spec import Spec from spack.util.package_hash import package_content @@ -44,7 +43,7 @@ class TestPackage(object): assert pkg.name == 'mpich' def test_package_filename(self): - repo = Repo(mock_packages_path) + repo = spack.repo.Repo(mock_packages_path) filename = repo.filename_for_package_name('mpich') assert filename == join_path( mock_packages_path, @@ -54,7 +53,7 @@ class TestPackage(object): ) def test_nonexisting_package_filename(self): - repo = Repo(mock_packages_path) + repo = spack.repo.Repo(mock_packages_path) filename = repo.filename_for_package_name('some-nonexisting-package') assert filename == join_path( mock_packages_path, diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 30bff20213..2bc37cdd86 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -34,7 +34,7 @@ import argparse from llnl.util.filesystem import mkdirp -import spack +import spack.repo import spack.store import spack.binary_distribution as bindist import spack.cmd.buildcache as buildcache diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index a890f80f1e..3cf6d7b30d 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -39,7 +39,7 @@ Tests assume that mock packages provide this:: """ from six import StringIO -import spack +import spack.repo from spack.provider_index import ProviderIndex from spack.spec import Spec diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 27b5830d98..1bac0bb32a 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -24,7 +24,7 @@ ############################################################################## import pytest -import spack.repository +import spack.repo import spack.paths @@ -33,7 +33,7 @@ import spack.paths # given RepoPath @pytest.fixture() def repo_for_test(): - return spack.repository.RepoPath(spack.paths.mock_packages_path) + return spack.repo.RepoPath(spack.paths.mock_packages_path) @pytest.fixture() @@ -47,7 +47,7 @@ def extra_repo(tmpdir_factory): repo: namespace: extra_test_repo """) - return spack.repository.Repo(str(repo_dir)) + return spack.repo.Repo(str(repo_dir)) def test_repo_getpkg(repo_for_test): @@ -68,10 +68,10 @@ def test_repo_multi_getpkgclass(repo_for_test, extra_repo): def test_repo_pkg_with_unknown_namespace(repo_for_test): - with pytest.raises(spack.repository.UnknownNamespaceError): + with pytest.raises(spack.repo.UnknownNamespaceError): repo_for_test.get('unknown.a') def test_repo_unknown_pkg(repo_for_test): - with pytest.raises(spack.repository.UnknownPackageError): + with pytest.raises(spack.repo.UnknownPackageError): repo_for_test.get('builtin.mock.nonexistentpackage') diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 5550a0fb88..f75b7e3feb 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -86,8 +86,6 @@ packages in the following spec DAG:: w->y deptypes are (link, build), w->x and y->z deptypes are (test) """ - saved_repo = spack.repo - default = ('build', 'link') test_only = ('test',) @@ -97,15 +95,12 @@ w->y deptypes are (link, build), w->x and y->z deptypes are (test) w = MockPackage('w', [x, y], [test_only, default]) mock_repo = MockPackageMultiRepo([w, x, y, z]) - try: - spack.repo = mock_repo + with spack.repo.swap(mock_repo): spec = Spec('w') spec.concretize(tests=(w.name,)) assert ('x' in spec) assert ('z' not in spec) - finally: - spack.repo = saved_repo @pytest.mark.usefixtures('mutable_mock_packages') diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 1b3040489a..323fb7ddf8 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -28,7 +28,7 @@ import pytest from llnl.util.filesystem import join_path, touch, working_dir -import spack +import spack.repo import spack.config from spack.spec import Spec from spack.version import ver diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py index 5a9f996e3a..780b4bdf58 100644 --- a/lib/spack/spack/test/url_fetch.py +++ b/lib/spack/spack/test/url_fetch.py @@ -27,7 +27,7 @@ import pytest from llnl.util.filesystem import working_dir, is_exe -import spack +import spack.repo import spack.config from spack.fetch_strategy import from_list_url, URLFetchStrategy from spack.spec import Spec diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py index 6362d9d7bb..fe5316ca0a 100644 --- a/lib/spack/spack/util/package_hash.py +++ b/lib/spack/spack/util/package_hash.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import spack +import spack.repo from spack import directives from spack.error import SpackError from spack.spec import Spec @@ -131,7 +131,7 @@ def package_hash(spec, content=None): def package_ast(spec): spec = Spec(spec) - filename = spack.repo.filename_for_package_name(spec.name) + filename = spack.repo.path().filename_for_package_name(spec.name) with open(filename) as f: text = f.read() root = ast.parse(text) -- cgit v1.2.3-60-g2f50