summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/spack/spack/__init__.py6
-rw-r--r--lib/spack/spack/build_environment.py25
-rw-r--r--lib/spack/spack/cmd/create.py12
-rw-r--r--lib/spack/spack/cmd/packagerepo.py85
-rw-r--r--lib/spack/spack/packages.py147
-rw-r--r--lib/spack/spack/repo_loader.py115
-rw-r--r--var/spack/packages/reponame1
7 files changed, 343 insertions, 48 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index caa09eb6e0..1d67b45341 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -63,6 +63,12 @@ mock_site_config = join_path(mock_config_path, "site_spackconfig")
mock_user_config = join_path(mock_config_path, "user_spackconfig")
#
+# Setup the spack.repos namespace
+#
+from spack.repo_loader import RepoNamespace
+repos = RepoNamespace()
+
+#
# This controls how spack lays out install prefixes and
# stage directories.
#
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index a133faa629..03a4930259 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -157,7 +157,7 @@ def set_build_environment_variables(pkg):
path_set("PKG_CONFIG_PATH", pkg_config_dirs)
-def set_module_variables_for_package(pkg):
+def set_module_variables_for_package(pkg, m):
"""Populate the module scope of install() with some useful functions.
This makes things easier for package writers.
"""
@@ -228,11 +228,32 @@ def get_rpaths(pkg):
return rpaths
+def parent_class_modules(cls):
+ """Get list of super class modules that are all descend from spack.Package"""
+ if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls):
+ return []
+ result = []
+ module = sys.modules.get(cls.__module__)
+ if module:
+ result = [ module ]
+ for c in cls.__bases__:
+ result.extend(parent_class_modules(c))
+ return result
+
+
def setup_package(pkg):
"""Execute all environment setup routines."""
set_compiler_environment_variables(pkg)
set_build_environment_variables(pkg)
- set_module_variables_for_package(pkg)
+
+ # If a user makes their own package repo, e.g.
+ # spack.repos.mystuff.libelf.Libelf, and they inherit from
+ # an existing class like spack.repos.original.libelf.Libelf,
+ # then set the module variables for both classes so the
+ # parent class can still use them if it gets called.
+ modules = parent_class_modules(pkg.__class__)
+ for mod in modules:
+ set_module_variables_for_package(pkg, mod)
# Allow dependencies to set up environment as well.
for dep_spec in pkg.spec.traverse(root=False):
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index 46e6bcec14..1502942f2c 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -94,6 +94,9 @@ def setup_parser(subparser):
'-n', '--name', dest='alternate_name', default=None,
help="Override the autodetected name for the created package.")
subparser.add_argument(
+ '-p', '--package-repo', dest='package_repo', default=None,
+ help="Create the package in the specified packagerepo.")
+ subparser.add_argument(
'-f', '--force', action='store_true', dest='force',
help="Overwrite any existing package file with the same name.")
@@ -160,12 +163,21 @@ def create(parser, args):
tty.die("Couldn't guess a name for this package. Try running:", "",
"spack create --name <name> <url>")
+ package_repo = args.package_repo
+
if not valid_module_name(name):
tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
+ # Create a directory for the new package.
+ pkg_path = spack.db.filename_for_package_name(name, package_repo)
+ if os.path.exists(pkg_path) and not args.force:
+ tty.die("%s already exists." % pkg_path)
+ else:
+ mkdirp(os.path.dirname(pkg_path))
+
versions = spack.package.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py
new file mode 100644
index 0000000000..66bad0ecbf
--- /dev/null
+++ b/lib/spack/spack/cmd/packagerepo.py
@@ -0,0 +1,85 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from external import argparse
+
+import llnl.util.tty as tty
+from llnl.util.tty.color import colorize
+from llnl.util.tty.colify import colify
+from llnl.util.lang import index_by
+
+import spack.spec
+import spack.config
+from spack.util.environment import get_path
+
+description = "Manage package sources"
+
+def setup_parser(subparser):
+ sp = subparser.add_subparsers(
+ metavar='SUBCOMMAND', dest='packagerepo_command')
+
+ add_parser = sp.add_parser('add', help=packagerepo_add.__doc__)
+ add_parser.add_argument('directory', help="Directory containing the packages.")
+
+ remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__)
+ remove_parser.add_argument('name')
+
+ list_parser = sp.add_parser('list', help=packagerepo_list.__doc__)
+
+
+def packagerepo_add(args):
+ """Add package sources to the Spack configuration."""
+ config = spack.config.get_config()
+ user_config = spack.config.get_config('user')
+ orig = None
+ if config.has_value('packagerepo', '', 'directories'):
+ orig = config.get_value('packagerepo', '', 'directories')
+ if orig and args.directory in orig.split(':'):
+ tty.die('Repo directory %s already exists in the repo list' % args.directory)
+
+ newsetting = orig + ':' + args.directory if orig else args.directory
+ user_config.set_value('packagerepo', '', 'directories', newsetting)
+ user_config.write()
+
+
+def packagerepo_remove(args):
+ """Remove a package source from the Spack configuration"""
+ pass
+
+
+def packagerepo_list(args):
+ """List package sources and their mnemoics"""
+ root_names = spack.db.repos
+ max_len = max(len(s[0]) for s in root_names)
+ fmt = "%%-%ds%%s" % (max_len + 4)
+ for root in root_names:
+ print fmt % (root[0], root[1])
+
+
+
+def packagerepo(parser, args):
+ action = { 'add' : packagerepo_add,
+ 'remove' : packagerepo_remove,
+ 'list' : packagerepo_list }
+ action[args.packagerepo_command](args)
diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py
index adfbc26c1d..79dbd60703 100644
--- a/lib/spack/spack/packages.py
+++ b/lib/spack/spack/packages.py
@@ -23,10 +23,14 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
+import exceptions
import sys
import inspect
import glob
import imp
+import spack.config
+import re
+from contextlib import closing
import llnl.util.tty as tty
from llnl.util.filesystem import join_path
@@ -36,13 +40,11 @@ import spack.error
import spack.spec
from spack.virtual import ProviderIndex
from spack.util.naming import mod_to_class, validate_module_name
+from sets import Set
+from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name
-# Name of module under which packages are imported
-_imported_packages_module = 'spack.packages'
-
-# Name of the package file inside a package directory
-_package_file_name = 'package.py'
-
+# Filename for package repo names
+_packagerepo_filename = 'reponame'
def _autospec(function):
"""Decorator that automatically converts the argument of a single-arg
@@ -55,13 +57,57 @@ def _autospec(function):
class PackageDB(object):
- def __init__(self, root):
+ def __init__(self, default_root):
"""Construct a new package database from a root directory."""
- self.root = root
+
+ #Collect the repos from the config file and read their names from the file system
+ repo_dirs = self._repo_list_from_config()
+ repo_dirs.append(default_root)
+ self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs]
+
+ # Check for duplicate repo names
+ s = set()
+ dups = set(r for r in self.repos if r[0] in s or s.add(r[0]))
+ if dups:
+ reponame = list(dups)[0][0]
+ dir1 = list(dups)[0][1]
+ dir2 = dict(s)[reponame]
+ tty.die("Package repo %s in directory %s has the same name as the "
+ "repo in directory %s" %
+ (reponame, dir1, dir2))
+
+ # For each repo, create a RepoLoader
+ self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos])
+
self.instances = {}
self.provider_index = None
+ def _read_reponame_from_directory(self, dir):
+ """For a packagerepo directory, read the repo name from the dir/reponame file"""
+ path = os.path.join(dir, 'reponame')
+
+ try:
+ with closing(open(path, 'r')) as reponame_file:
+ name = reponame_file.read().lstrip().rstrip()
+ if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name):
+ tty.die("Package repo name '%s', read from %s, is an invalid name. "
+ "Repo names must began with a letter and only contain letters "
+ "and numbers." % (name, path))
+ return name
+ except exceptions.IOError, e:
+ tty.die("Could not read from package repo name file %s" % path)
+
+
+
+ def _repo_list_from_config(self):
+ """Read through the spackconfig and return the list of packagerepo directories"""
+ config = spack.config.get_config()
+ if not config.has_option('packagerepo', 'directories'): return []
+ dir_string = config.get('packagerepo', 'directories')
+ return dir_string.split(':')
+
+
@_autospec
def get(self, spec, **kwargs):
if spec.virtual:
@@ -130,13 +176,33 @@ class PackageDB(object):
# catching exceptions.
- def dirname_for_package_name(self, pkg_name):
+ def repo_for_package_name(self, pkg_name, packagerepo_name=None):
+ """Find the dirname for a package and the packagerepo it came from
+ if packagerepo_name is not None, then search for the package in the
+ specified packagerepo"""
+ #Look for an existing package under any matching packagerepos
+ roots = [pkgrepo for pkgrepo in self.repos
+ if not packagerepo_name or packagerepo_name == pkgrepo[0]]
+
+ if not roots:
+ tty.die("Package repo %s does not exist" % packagerepo_name)
+
+ for pkgrepo in roots:
+ path = join_path(pkgrepo[1], pkg_name)
+ if os.path.exists(path):
+ return (pkgrepo[0], path)
+
+ repo_to_add_to = roots[-1]
+ return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name))
+
+
+ def dirname_for_package_name(self, pkg_name, packagerepo_name=None):
"""Get the directory name for a particular package. This is the
directory that contains its package.py file."""
- return join_path(self.root, pkg_name)
+ return self.repo_for_package_name(pkg_name, packagerepo_name)[1]
- def filename_for_package_name(self, pkg_name):
+ def filename_for_package_name(self, pkg_name, packagerepo_name=None):
"""Get the filename for the module we should load for a particular
package. Packages for a pacakge DB live in
``$root/<package_name>/package.py``
@@ -144,10 +210,15 @@ class PackageDB(object):
This will return a proper package.py path even if the
package doesn't exist yet, so callers will need to ensure
the package exists before importing.
+
+ If a packagerepo is specified, then return existing
+ or new paths in the specified packagerepo directory. If no
+ package repo is supplied, return an existing path from any
+ package repo, and new paths in the default package repo.
"""
validate_module_name(pkg_name)
- pkg_dir = self.dirname_for_package_name(pkg_name)
- return join_path(pkg_dir, _package_file_name)
+ pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name)
+ return join_path(pkg_dir, package_file_name)
def installed_package_specs(self):
@@ -176,14 +247,19 @@ class PackageDB(object):
@memoized
def all_package_names(self):
"""Generator function for all packages. This looks for
- ``<pkg_name>/package.py`` files within the root direcotry"""
- all_package_names = []
- for pkg_name in os.listdir(self.root):
- pkg_dir = join_path(self.root, pkg_name)
- pkg_file = join_path(pkg_dir, _package_file_name)
- if os.path.isfile(pkg_file):
- all_package_names.append(pkg_name)
- all_package_names.sort()
+ ``<pkg_name>/package.py`` files within the repo direcotories"""
+ all_packages = Set()
+ for repo in self.repos:
+ dir = repo[1]
+ if not os.path.isdir(dir):
+ continue
+ for pkg_name in os.listdir(dir):
+ pkg_dir = join_path(dir, pkg_name)
+ pkg_file = join_path(pkg_dir, package_file_name)
+ if os.path.isfile(pkg_file):
+ all_packages.add(pkg_name)
+ all_package_names = list(all_packages)
+ all_package_names.sort()
return all_package_names
@@ -200,34 +276,13 @@ class PackageDB(object):
@memoized
def get_class_for_package_name(self, pkg_name):
- """Get an instance of the class for a particular package.
-
- This method uses Python's ``imp`` package to load python
- source from a Spack package's ``package.py`` file. A
- normal python import would only load each package once, but
- because we do this dynamically, the method needs to be
- memoized to ensure there is only ONE package class
- instance, per package, per database.
- """
- file_path = self.filename_for_package_name(pkg_name)
+ """Get an instance of the class for a particular package."""
+ repo = self.repo_for_package_name(pkg_name)
+ module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name
- if os.path.exists(file_path):
- if not os.path.isfile(file_path):
- tty.die("Something's wrong. '%s' is not a file!" % file_path)
- if not os.access(file_path, os.R_OK):
- tty.die("Cannot read '%s'!" % file_path)
- else:
- raise UnknownPackageError(pkg_name)
+ module = self.repo_loaders[repo[0]].get_module(pkg_name)
class_name = mod_to_class(pkg_name)
- try:
- module_name = _imported_packages_module + '.' + pkg_name
- module = imp.load_source(module_name, file_path)
-
- except ImportError, e:
- tty.die("Error while importing %s from %s:\n%s" % (
- pkg_name, file_path, e.message))
-
cls = getattr(module, class_name)
if not inspect.isclass(cls):
tty.die("%s.%s is not a class" % (pkg_name, class_name))
diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py
new file mode 100644
index 0000000000..57c19a6c28
--- /dev/null
+++ b/lib/spack/spack/repo_loader.py
@@ -0,0 +1,115 @@
+import spack
+import spack.repos
+import re
+import types
+from llnl.util.lang import *
+
+# Name of module under which packages are imported
+imported_packages_module = 'spack.repos'
+
+# Name of the package file inside a package directory
+package_file_name = 'package.py'
+
+import sys
+class LazyLoader:
+ """The LazyLoader handles cases when repo modules or classes
+ are imported. It watches for 'spack.repos.*' loads, then
+ redirects the load to the appropriate module."""
+ def find_module(self, fullname, pathname):
+ if not fullname.startswith(imported_packages_module):
+ return None
+ partial_name = fullname[len(imported_packages_module)+1:]
+ repo = partial_name.split('.')[0]
+ module = partial_name.split('.')[1]
+ repo_loader = spack.db.repo_loaders.get(repo)
+ if repo_loader:
+ try:
+ self.mod = repo_loader.get_module(module)
+ return self
+ except (ImportError, spack.packages.UnknownPackageError):
+ return None
+
+ def load_module(self, fullname):
+ return self.mod
+
+sys.meta_path.append(LazyLoader())
+
+_reponames = {}
+class RepoNamespace(types.ModuleType):
+ """The RepoNamespace holds the repository namespaces under
+ spack.repos. For example, when accessing spack.repos.original
+ this class will use __getattr__ to translate the 'original'
+ into one of spack's known repositories"""
+ def __init__(self):
+ import sys
+ sys.modules[imported_packages_module] = self
+
+ def __getattr__(self, name):
+ if name in _reponames:
+ return _reponames[name]
+ raise AttributeError
+
+ @property
+ def __file__(self):
+ return None
+
+ @property
+ def __path__(self):
+ return []
+
+
+class RepoLoader(types.ModuleType):
+ """Each RepoLoader is associated with a repository, and the RepoLoader is
+ responsible for loading packages out of that repository. For example,
+ a RepoLoader may be responsible for spack.repos.original, and when someone
+ references spack.repos.original.libelf that RepoLoader will load the
+ libelf package."""
+ def __init__(self, reponame, repopath):
+ self.path = repopath
+ self.reponame = reponame
+ self.module_name = imported_packages_module + '.' + reponame
+ if not reponame in _reponames:
+ _reponames[reponame] = self
+ spack.repos.add_repo(reponame, self)
+
+ import sys
+ sys.modules[self.module_name] = self
+
+
+ @property
+ def __path__(self):
+ return [ self.path ]
+
+
+ def __getattr__(self, name):
+ if name[0] == '_':
+ raise AttributeError
+ return self.get_module(name)
+
+
+ @memoized
+ def get_module(self, pkg_name):
+ import os
+ import imp
+ import llnl.util.tty as tty
+
+ file_path = os.path.join(self.path, pkg_name, package_file_name)
+ if os.path.exists(file_path):
+ if not os.path.isfile(file_path):
+ tty.die("Something's wrong. '%s' is not a file!" % file_path)
+ if not os.access(file_path, os.R_OK):
+ tty.die("Cannot read '%s'!" % file_path)
+ else:
+ raise spack.packages.UnknownPackageError(pkg_name)
+
+ try:
+ module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name
+ module = imp.load_source(module_name, file_path)
+
+ except ImportError, e:
+ tty.die("Error while importing %s from %s:\n%s" % (
+ pkg_name, file_path, e.message))
+
+ return module
+
+
diff --git a/var/spack/packages/reponame b/var/spack/packages/reponame
new file mode 100644
index 0000000000..4b48deed3a
--- /dev/null
+++ b/var/spack/packages/reponame
@@ -0,0 +1 @@
+original