diff options
| -rw-r--r-- | lib/spack/spack/__init__.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/build_environment.py | 25 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/create.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/packagerepo.py | 85 | ||||
| -rw-r--r-- | lib/spack/spack/packages.py | 147 | ||||
| -rw-r--r-- | lib/spack/spack/repo_loader.py | 115 | ||||
| -rw-r--r-- | var/spack/packages/reponame | 1 |
7 files changed, 343 insertions, 48 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..1d67b45341 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -63,6 +63,12 @@ mock_site_config = join_path(mock_config_path, "site_spackconfig") mock_user_config = join_path(mock_config_path, "user_spackconfig") # +# Setup the spack.repos namespace +# +from spack.repo_loader import RepoNamespace +repos = RepoNamespace() + +# # This controls how spack lays out install prefixes and # stage directories. # diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa629..03a4930259 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -157,7 +157,7 @@ def set_build_environment_variables(pkg): path_set("PKG_CONFIG_PATH", pkg_config_dirs) -def set_module_variables_for_package(pkg): +def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ @@ -228,11 +228,32 @@ def get_rpaths(pkg): return rpaths +def parent_class_modules(cls): + """Get list of super class modules that are all descend from spack.Package""" + if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): + return [] + result = [] + module = sys.modules.get(cls.__module__) + if module: + result = [ module ] + for c in cls.__bases__: + result.extend(parent_class_modules(c)) + return result + + def setup_package(pkg): """Execute all environment setup routines.""" set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) - set_module_variables_for_package(pkg) + + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + modules = parent_class_modules(pkg.__class__) + for mod in modules: + set_module_variables_for_package(pkg, mod) # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 46e6bcec14..1502942f2c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -94,6 +94,9 @@ def setup_parser(subparser): '-n', '--name', dest='alternate_name', default=None, help="Override the autodetected name for the created package.") subparser.add_argument( + '-p', '--package-repo', dest='package_repo', default=None, + help="Create the package in the specified packagerepo.") + subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="Overwrite any existing package file with the same name.") @@ -160,12 +163,21 @@ def create(parser, args): tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name <name> <url>") + package_repo = args.package_repo + if not valid_module_name(name): tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'") tty.msg("This looks like a URL for %s version %s." % (name, version)) tty.msg("Creating template for package %s" % name) + # Create a directory for the new package. + pkg_path = spack.db.filename_for_package_name(name, package_repo) + if os.path.exists(pkg_path) and not args.force: + tty.die("%s already exists." % pkg_path) + else: + mkdirp(os.path.dirname(pkg_path)) + versions = spack.package.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py new file mode 100644 index 0000000000..66bad0ecbf --- /dev/null +++ b/lib/spack/spack/cmd/packagerepo.py @@ -0,0 +1,85 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse + +import llnl.util.tty as tty +from llnl.util.tty.color import colorize +from llnl.util.tty.colify import colify +from llnl.util.lang import index_by + +import spack.spec +import spack.config +from spack.util.environment import get_path + +description = "Manage package sources" + +def setup_parser(subparser): + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='packagerepo_command') + + add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) + add_parser.add_argument('directory', help="Directory containing the packages.") + + remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) + remove_parser.add_argument('name') + + list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) + + +def packagerepo_add(args): + """Add package sources to the Spack configuration.""" + config = spack.config.get_config() + user_config = spack.config.get_config('user') + orig = None + if config.has_value('packagerepo', '', 'directories'): + orig = config.get_value('packagerepo', '', 'directories') + if orig and args.directory in orig.split(':'): + tty.die('Repo directory %s already exists in the repo list' % args.directory) + + newsetting = orig + ':' + args.directory if orig else args.directory + user_config.set_value('packagerepo', '', 'directories', newsetting) + user_config.write() + + +def packagerepo_remove(args): + """Remove a package source from the Spack configuration""" + pass + + +def packagerepo_list(args): + """List package sources and their mnemoics""" + root_names = spack.db.repos + max_len = max(len(s[0]) for s in root_names) + fmt = "%%-%ds%%s" % (max_len + 4) + for root in root_names: + print fmt % (root[0], root[1]) + + + +def packagerepo(parser, args): + action = { 'add' : packagerepo_add, + 'remove' : packagerepo_remove, + 'list' : packagerepo_list } + action[args.packagerepo_command](args) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index adfbc26c1d..79dbd60703 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -23,10 +23,14 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import exceptions import sys import inspect import glob import imp +import spack.config +import re +from contextlib import closing import llnl.util.tty as tty from llnl.util.filesystem import join_path @@ -36,13 +40,11 @@ import spack.error import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import mod_to_class, validate_module_name +from sets import Set +from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name -# Name of module under which packages are imported -_imported_packages_module = 'spack.packages' - -# Name of the package file inside a package directory -_package_file_name = 'package.py' - +# Filename for package repo names +_packagerepo_filename = 'reponame' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -55,13 +57,57 @@ def _autospec(function): class PackageDB(object): - def __init__(self, root): + def __init__(self, default_root): """Construct a new package database from a root directory.""" - self.root = root + + #Collect the repos from the config file and read their names from the file system + repo_dirs = self._repo_list_from_config() + repo_dirs.append(default_root) + self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs] + + # Check for duplicate repo names + s = set() + dups = set(r for r in self.repos if r[0] in s or s.add(r[0])) + if dups: + reponame = list(dups)[0][0] + dir1 = list(dups)[0][1] + dir2 = dict(s)[reponame] + tty.die("Package repo %s in directory %s has the same name as the " + "repo in directory %s" % + (reponame, dir1, dir2)) + + # For each repo, create a RepoLoader + self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) + self.instances = {} self.provider_index = None + def _read_reponame_from_directory(self, dir): + """For a packagerepo directory, read the repo name from the dir/reponame file""" + path = os.path.join(dir, 'reponame') + + try: + with closing(open(path, 'r')) as reponame_file: + name = reponame_file.read().lstrip().rstrip() + if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): + tty.die("Package repo name '%s', read from %s, is an invalid name. " + "Repo names must began with a letter and only contain letters " + "and numbers." % (name, path)) + return name + except exceptions.IOError, e: + tty.die("Could not read from package repo name file %s" % path) + + + + def _repo_list_from_config(self): + """Read through the spackconfig and return the list of packagerepo directories""" + config = spack.config.get_config() + if not config.has_option('packagerepo', 'directories'): return [] + dir_string = config.get('packagerepo', 'directories') + return dir_string.split(':') + + @_autospec def get(self, spec, **kwargs): if spec.virtual: @@ -130,13 +176,33 @@ class PackageDB(object): # catching exceptions. - def dirname_for_package_name(self, pkg_name): + def repo_for_package_name(self, pkg_name, packagerepo_name=None): + """Find the dirname for a package and the packagerepo it came from + if packagerepo_name is not None, then search for the package in the + specified packagerepo""" + #Look for an existing package under any matching packagerepos + roots = [pkgrepo for pkgrepo in self.repos + if not packagerepo_name or packagerepo_name == pkgrepo[0]] + + if not roots: + tty.die("Package repo %s does not exist" % packagerepo_name) + + for pkgrepo in roots: + path = join_path(pkgrepo[1], pkg_name) + if os.path.exists(path): + return (pkgrepo[0], path) + + repo_to_add_to = roots[-1] + return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) + + + def dirname_for_package_name(self, pkg_name, packagerepo_name=None): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.root, pkg_name) + return self.repo_for_package_name(pkg_name, packagerepo_name)[1] - def filename_for_package_name(self, pkg_name): + def filename_for_package_name(self, pkg_name, packagerepo_name=None): """Get the filename for the module we should load for a particular package. Packages for a pacakge DB live in ``$root/<package_name>/package.py`` @@ -144,10 +210,15 @@ class PackageDB(object): This will return a proper package.py path even if the package doesn't exist yet, so callers will need to ensure the package exists before importing. + + If a packagerepo is specified, then return existing + or new paths in the specified packagerepo directory. If no + package repo is supplied, return an existing path from any + package repo, and new paths in the default package repo. """ validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name) - return join_path(pkg_dir, _package_file_name) + pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name) + return join_path(pkg_dir, package_file_name) def installed_package_specs(self): @@ -176,14 +247,19 @@ class PackageDB(object): @memoized def all_package_names(self): """Generator function for all packages. This looks for - ``<pkg_name>/package.py`` files within the root direcotry""" - all_package_names = [] - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, _package_file_name) - if os.path.isfile(pkg_file): - all_package_names.append(pkg_name) - all_package_names.sort() + ``<pkg_name>/package.py`` files within the repo direcotories""" + all_packages = Set() + for repo in self.repos: + dir = repo[1] + if not os.path.isdir(dir): + continue + for pkg_name in os.listdir(dir): + pkg_dir = join_path(dir, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + all_packages.add(pkg_name) + all_package_names = list(all_packages) + all_package_names.sort() return all_package_names @@ -200,34 +276,13 @@ class PackageDB(object): @memoized def get_class_for_package_name(self, pkg_name): - """Get an instance of the class for a particular package. - - This method uses Python's ``imp`` package to load python - source from a Spack package's ``package.py`` file. A - normal python import would only load each package once, but - because we do this dynamically, the method needs to be - memoized to ensure there is only ONE package class - instance, per package, per database. - """ - file_path = self.filename_for_package_name(pkg_name) + """Get an instance of the class for a particular package.""" + repo = self.repo_for_package_name(pkg_name) + module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name - if os.path.exists(file_path): - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - else: - raise UnknownPackageError(pkg_name) + module = self.repo_loaders[repo[0]].get_module(pkg_name) class_name = mod_to_class(pkg_name) - try: - module_name = _imported_packages_module + '.' + pkg_name - module = imp.load_source(module_name, file_path) - - except ImportError, e: - tty.die("Error while importing %s from %s:\n%s" % ( - pkg_name, file_path, e.message)) - cls = getattr(module, class_name) if not inspect.isclass(cls): tty.die("%s.%s is not a class" % (pkg_name, class_name)) diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py new file mode 100644 index 0000000000..57c19a6c28 --- /dev/null +++ b/lib/spack/spack/repo_loader.py @@ -0,0 +1,115 @@ +import spack +import spack.repos +import re +import types +from llnl.util.lang import * + +# Name of module under which packages are imported +imported_packages_module = 'spack.repos' + +# Name of the package file inside a package directory +package_file_name = 'package.py' + +import sys +class LazyLoader: + """The LazyLoader handles cases when repo modules or classes + are imported. It watches for 'spack.repos.*' loads, then + redirects the load to the appropriate module.""" + def find_module(self, fullname, pathname): + if not fullname.startswith(imported_packages_module): + return None + partial_name = fullname[len(imported_packages_module)+1:] + repo = partial_name.split('.')[0] + module = partial_name.split('.')[1] + repo_loader = spack.db.repo_loaders.get(repo) + if repo_loader: + try: + self.mod = repo_loader.get_module(module) + return self + except (ImportError, spack.packages.UnknownPackageError): + return None + + def load_module(self, fullname): + return self.mod + +sys.meta_path.append(LazyLoader()) + +_reponames = {} +class RepoNamespace(types.ModuleType): + """The RepoNamespace holds the repository namespaces under + spack.repos. For example, when accessing spack.repos.original + this class will use __getattr__ to translate the 'original' + into one of spack's known repositories""" + def __init__(self): + import sys + sys.modules[imported_packages_module] = self + + def __getattr__(self, name): + if name in _reponames: + return _reponames[name] + raise AttributeError + + @property + def __file__(self): + return None + + @property + def __path__(self): + return [] + + +class RepoLoader(types.ModuleType): + """Each RepoLoader is associated with a repository, and the RepoLoader is + responsible for loading packages out of that repository. For example, + a RepoLoader may be responsible for spack.repos.original, and when someone + references spack.repos.original.libelf that RepoLoader will load the + libelf package.""" + def __init__(self, reponame, repopath): + self.path = repopath + self.reponame = reponame + self.module_name = imported_packages_module + '.' + reponame + if not reponame in _reponames: + _reponames[reponame] = self + spack.repos.add_repo(reponame, self) + + import sys + sys.modules[self.module_name] = self + + + @property + def __path__(self): + return [ self.path ] + + + def __getattr__(self, name): + if name[0] == '_': + raise AttributeError + return self.get_module(name) + + + @memoized + def get_module(self, pkg_name): + import os + import imp + import llnl.util.tty as tty + + file_path = os.path.join(self.path, pkg_name, package_file_name) + if os.path.exists(file_path): + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + else: + raise spack.packages.UnknownPackageError(pkg_name) + + try: + module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name + module = imp.load_source(module_name, file_path) + + except ImportError, e: + tty.die("Error while importing %s from %s:\n%s" % ( + pkg_name, file_path, e.message)) + + return module + + diff --git a/var/spack/packages/reponame b/var/spack/packages/reponame new file mode 100644 index 0000000000..4b48deed3a --- /dev/null +++ b/var/spack/packages/reponame @@ -0,0 +1 @@ +original |