From 269cf53a68400124ec31e677a0ac045293ef0439 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 22 Mar 2013 13:46:01 -0700 Subject: Documentation and small changes. --- lib/spack/spack/Package.py | 411 ----------------------- lib/spack/spack/__init__.py | 3 +- lib/spack/spack/cmd/fetch.py | 9 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/exception.py | 16 + lib/spack/spack/globals.py | 5 - lib/spack/spack/package.py | 612 ++++++++++++++++++++++++++++++++++ lib/spack/spack/packages/__init__.py | 2 +- lib/spack/spack/stage.py | 129 ++++--- lib/spack/spack/test/test_versions.py | 6 +- lib/spack/spack/version.py | 40 ++- 11 files changed, 758 insertions(+), 477 deletions(-) delete mode 100644 lib/spack/spack/Package.py create mode 100644 lib/spack/spack/package.py (limited to 'lib') diff --git a/lib/spack/spack/Package.py b/lib/spack/spack/Package.py deleted file mode 100644 index f8256b9add..0000000000 --- a/lib/spack/spack/Package.py +++ /dev/null @@ -1,411 +0,0 @@ -import sys -import inspect -import os -import re -import subprocess -import platform -import shutil - -from spack import * -import packages -import tty -import attr -import validate -import version -import arch -from stage import Stage - - -DEPENDS_ON = "depends_on" - -class Dependency(object): - """Represents a dependency from one package to another.""" - def __init__(self, name, **kwargs): - self.name = name - for key in kwargs: - setattr(self, key, kwargs[key]) - - @property - def package(self): - return packages.get(self.name) - - def __repr__(self): - return "" % self.name - - def __str__(self): - return self.__repr__() - - -def depends_on(*args, **kwargs): - """Adds a depends_on local variable in the locals of - the calling class, based on args. - """ - # This gets the calling frame so we can pop variables into it - locals = sys._getframe(1).f_locals - - # Put deps into the dependencies variable - dependencies = locals.setdefault("dependencies", []) - for name in args: - dependencies.append(Dependency(name)) - - -class MakeExecutable(Executable): - """Special Executable for make so the user can specify parallel or - not on a per-invocation basis. Using 'parallel' as a kwarg will - override whatever the package's global setting is, so you can - either default to true or false and override particular calls. - - Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides - everything. - """ - def __init__(self, name, parallel): - super(MakeExecutable, self).__init__(name) - self.parallel = parallel - - def __call__(self, *args, **kwargs): - parallel = kwargs.get('parallel', self.parallel) - disable_parallel = env_flag(SPACK_NO_PARALLEL_MAKE) - - if parallel and not disable_parallel: - jobs = "-j%d" % multiprocessing.cpu_count() - args = (jobs,) + args - - super(MakeExecutable, self).__call__(*args, **kwargs) - - -class Package(object): - def __init__(self, arch=arch.sys_type()): - attr.required(self, 'homepage') - attr.required(self, 'url') - attr.required(self, 'md5') - - attr.setdefault(self, 'dependencies', []) - attr.setdefault(self, 'parallel', True) - - # Architecture for this package. - self.arch = arch - - # Name of package is the name of its module (the file that contains it) - self.name = inspect.getmodulename(self.module.__file__) - - # Don't allow the default homepage. - if re.search(r'example.com', self.homepage): - tty.die("Bad homepage in %s: %s" % (self.name, self.homepage)) - - # Make sure URL is an allowed type - validate.url(self.url) - - # Set up version - attr.setdefault(self, 'version', version.parse_version(self.url)) - if not self.version: - tty.die("Couldn't extract version from %s. " + - "You must specify it explicitly for this URL." % self.url) - - # This adds a bunch of convenient commands to the package's module scope. - self.add_commands_to_module() - - # Controls whether install and uninstall check deps before acting. - self.ignore_dependencies = False - - # Empty at first; only compute dependents if necessary - self._dependents = None - - # Whether to remove intermediate build/install when things go wrong. - self.dirty = False - - # stage used to build this package. - self.stage = Stage(self.stage_name, self.url) - - - def make_make(self): - """Create a make command set up with the proper default arguments.""" - make = which('make', required=True) - return make - - - def add_commands_to_module(self): - """Populate the module scope of install() with some useful functions. - This makes things easier for package writers. - """ - self.module.make = MakeExecutable('make', self.parallel) - self.module.gmake = MakeExecutable('gmake', self.parallel) - - # number of jobs spack prefers to build with. - self.module.make_jobs = multiprocessing.cpu_count() - - # Find the configure script in the archive path - # Don't use which for this; we want to find it in the current dir. - self.module.configure = Executable('./configure') - self.module.cmake = which("cmake") - - # standard CMake arguments - self.module.std_cmake_args = [ - '-DCMAKE_INSTALL_PREFIX=%s' % self.prefix, - '-DCMAKE_BUILD_TYPE=None'] - if platform.mac_ver()[0]: - self.module.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') - - # Emulate some shell commands for convenience - self.module.cd = os.chdir - self.module.mkdir = os.mkdir - self.module.makedirs = os.makedirs - self.module.removedirs = os.removedirs - - self.module.mkdirp = mkdirp - self.module.install = install - self.module.rmtree = shutil.rmtree - self.module.move = shutil.move - self.module.remove = os.remove - - # Useful directories within the prefix - self.module.prefix = self.prefix - self.module.bin = new_path(self.prefix, 'bin') - self.module.sbin = new_path(self.prefix, 'sbin') - self.module.etc = new_path(self.prefix, 'etc') - self.module.include = new_path(self.prefix, 'include') - self.module.lib = new_path(self.prefix, 'lib') - self.module.lib64 = new_path(self.prefix, 'lib64') - self.module.libexec = new_path(self.prefix, 'libexec') - self.module.share = new_path(self.prefix, 'share') - self.module.doc = new_path(self.module.share, 'doc') - self.module.info = new_path(self.module.share, 'info') - self.module.man = new_path(self.module.share, 'man') - self.module.man1 = new_path(self.module.man, 'man1') - self.module.man2 = new_path(self.module.man, 'man2') - self.module.man3 = new_path(self.module.man, 'man3') - self.module.man4 = new_path(self.module.man, 'man4') - self.module.man5 = new_path(self.module.man, 'man5') - self.module.man6 = new_path(self.module.man, 'man6') - self.module.man7 = new_path(self.module.man, 'man7') - self.module.man8 = new_path(self.module.man, 'man8') - - - @property - def dependents(self): - """List of names of packages that depend on this one.""" - if self._dependents is None: - packages.compute_dependents() - return tuple(self._dependents) - - - @property - def installed(self): - return os.path.exists(self.prefix) - - - @property - def installed_dependents(self): - installed = [d for d in self.dependents if packages.get(d).installed] - all_deps = [] - for d in installed: - all_deps.append(d) - all_deps.extend(packages.get(d).installed_dependents) - return tuple(all_deps) - - - @property - def all_dependents(self): - all_deps = list(self.dependents) - for pkg in self.dependents: - all_deps.extend(packages.get(pkg).all_dependents) - return tuple(all_deps) - - - @property - def stage_name(self): - return "%s-%s" % (self.name, self.version) - - - @property - def platform_path(self): - """Directory for binaries for the current platform.""" - return new_path(install_path, self.arch) - - - @property - def package_path(self): - """Directory for different versions of this package. Lives just above prefix.""" - return new_path(self.platform_path, self.name) - - - @property - def installed_versions(self): - return [ver for ver in os.listdir(self.package_path) - if os.path.isdir(new_path(self.package_path, ver))] - - - @property - def prefix(self): - """Packages are installed in $spack_prefix/opt///""" - return new_path(self.package_path, self.version) - - - def remove_prefix(self): - """Removes the prefix for a package along with any empty parent directories.""" - if os.path.exists(self.prefix): - shutil.rmtree(self.prefix, True) - - for dir in (self.package_path, self.platform_path): - if not os.path.isdir(dir): - continue - if not os.listdir(dir): - os.rmdir(dir) - else: - break - - - def do_fetch(self): - """Creates a stage directory and downloads the taball for this package. - Working directory will be set to the stage directory. - """ - stage = self.stage - stage.setup() - stage.fetch() - - archive_md5 = md5(stage.archive_file) - if archive_md5 != self.md5: - tty.die("MD5 Checksum failed for %s. Expected %s but got %s." - % (self.name, self.md5, archive_md5)) - - - def do_stage(self): - """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" - self.do_fetch() - stage = self.stage - - archive_dir = stage.expanded_archive_path - if not archive_dir: - tty.msg("Staging archive: %s" % stage.archive_file) - stage.expand_archive() - else: - tty.msg("Already staged %s" % self.name) - stage.chdir_to_archive() - - - def do_install(self): - """This class should call this version of the install method. - Package implementations should override install(). - """ - if os.path.exists(self.prefix): - tty.msg("%s is already installed." % self.name) - tty.pkg(self.prefix) - return - - if not self.ignore_dependencies: - self.do_install_dependencies() - - self.do_stage() - self.setup_install_environment() - - tty.msg("Building %s." % self.name) - try: - self.install(self.prefix) - if not os.path.isdir(self.prefix): - tty.die("Install failed for %s. No install dir created." % self.name) - except subprocess.CalledProcessError, e: - if not self.dirty: - self.remove_prefix() - tty.die("Install failed for %s" % self.name, e.message) - except Exception, e: - if not self.dirty: - self.remove_prefix() - raise - - tty.msg("Successfully installed %s" % self.name) - tty.pkg(self.prefix) - - # Once the install is done, destroy the stage where we built it, - # unless the user wants it kept around. - if not self.dirty: - self.stage.destroy() - - - def setup_install_environment(self): - """This ensures a clean install environment when we build packages.""" - pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") - - # Add spack environment at front of path and pass the - # lib location along so the compiler script can find spack - os.environ[SPACK_LIB] = lib_path - - # Fix for case-insensitive file systems. Conflicting links are - # in directories called "case*" within the env directory. - env_paths = [env_path] - for file in os.listdir(env_path): - path = new_path(env_path, file) - if file.startswith("case") and os.path.isdir(path): - env_paths.append(path) - path_put_first("PATH", env_paths) - path_set(SPACK_ENV_PATH, env_paths) - - # Pass along prefixes of dependencies here - path_set(SPACK_DEPENDENCIES, - [dep.package.prefix for dep in self.dependencies]) - - # Install location - os.environ[SPACK_PREFIX] = self.prefix - - # Build root for logging. - os.environ[SPACK_BUILD_ROOT] = self.stage.expanded_archive_path - - - def do_install_dependencies(self): - # Pass along paths of dependencies here - for dep in self.dependencies: - dep.package.do_install() - - - @property - def module(self): - """Use this to add variables to the class's module's scope. - This lets us use custom syntax in the install method. - """ - return __import__(self.__class__.__module__, - fromlist=[self.__class__.__name__]) - - - def install(self, prefix): - """Package implementations override this with their own build configuration.""" - tty.die("Packages must provide an install method!") - - - def do_uninstall(self): - if not os.path.exists(self.prefix): - tty.die(self.name + " is not installed.") - - if not self.ignore_dependencies: - deps = self.installed_dependents - if deps: tty.die( - "Cannot uninstall %s. The following installed packages depend on it:" - % self.name, " ".join(deps)) - - self.remove_prefix() - tty.msg("Successfully uninstalled %s." % self.name) - - - def do_clean(self): - if self.stage.expanded_archive_path: - self.stage.chdir_to_archive() - self.clean() - - - def clean(self): - """By default just runs make clean. Override if this isn't good.""" - try: - make = MakeExecutable('make', self.parallel) - make('clean') - tty.msg("Successfully cleaned %s" % self.name) - except subprocess.CalledProcessError, e: - tty.warn("Warning: 'make clean' didn't work. Consider 'spack clean --work'.") - - - def do_clean_work(self): - """By default just blows away the stage directory and re-stages.""" - self.stage.restage() - - - def do_clean_dist(self): - """Removes the stage directory where this package was built.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - tty.msg("Successfully cleaned %s" % self.name) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 81dfd0c8eb..ef9e448413 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -1,6 +1,5 @@ - from globals import * from utils import * from exception import * -from Package import Package, depends_on +from package import Package, depends_on diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index c447435862..df5173fdaa 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -3,11 +3,10 @@ import spack.packages as packages description = "Fetch archives for packages" def setup_parser(subparser): - subparser.add_argument('name', help="name of package to fetch") - subparser.add_argument('-f', '--file', dest='file', default=None, - help="supply an archive file instead of fetching from the package's URL.") + subparser.add_argument('names', nargs='+', help="names of packages to fetch") def fetch(parser, args): - package = packages.get(args.name) - package.do_fetch() + for name in args.names: + package = packages.get(name) + package.do_fetch() diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 9494838832..766be9a3ea 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -4,7 +4,7 @@ import spack.packages as packages description = "Build and install packages" def setup_parser(subparser): - subparser.add_argument('names', nargs='+', help="name(s) of package(s) to install") + subparser.add_argument('names', nargs='+', help="names of packages to install") subparser.add_argument('-i', '--ignore-dependencies', action='store_true', dest='ignore_dependencies', help="Do not try to install dependencies of requested packages.") diff --git a/lib/spack/spack/exception.py b/lib/spack/spack/exception.py index 815cd9be25..32167cf36a 100644 --- a/lib/spack/spack/exception.py +++ b/lib/spack/spack/exception.py @@ -21,3 +21,19 @@ class CommandFailedException(SpackException): def __init__(self, command): super(CommandFailedException, self).__init__("Failed to execute command: " + command) self.command = command + + +class VersionParseException(SpackException): + def __init__(self, msg, spec): + super(VersionParseException, self).__init__(msg) + self.spec = spec + + +class UndetectableVersionException(VersionParseException): + def __init__(self, spec): + super(UndetectableVersionException, self).__init__("Couldn't detect version in: " + spec, spec) + + +class UndetectableNameException(VersionParseException): + def __init__(self, spec): + super(UndetectableNameException, self).__init__("Couldn't parse package name in: " + spec) diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index d7321417b2..ee67a461c3 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -1,11 +1,6 @@ import os -import re -import multiprocessing from version import Version - -import tty from utils import * -from spack.exception import * # This lives in $prefix/lib/spac/spack/__file__ prefix = ancestor(__file__, 4) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py new file mode 100644 index 0000000000..0885d7ba7b --- /dev/null +++ b/lib/spack/spack/package.py @@ -0,0 +1,612 @@ +""" +This is where most of the action happens in Spack. +See the Package docs for detailed instructions on how the class works +and on how to write your own packages. + +The spack package structure is based strongly on Homebrew +(http://wiki.github.com/mxcl/homebrew/), mainly because +Homebrew makes it very easy to create packages. For a complete +rundown on spack and how it differs from homebrew, look at the +README. +""" +import sys +import inspect +import os +import re +import subprocess +import platform +import shutil + +from spack import * +import packages +import tty +import attr +import validate +import version +import arch +from stage import Stage + + +class Package(object): + """This is the superclass for all spack packages. + + The Package class + ================== + Package is where the bulk of the work of installing packages is done. + + A package defines how to fetch, verfiy (via, e.g., md5), build, and + install a piece of software. A Package also defines what other + packages it depends on, so that dependencies can be installed along + with the package itself. Packages are written in pure python. + + Packages are all submodules of spack.packages. If spack is installed + in $prefix, all of its python files are in $prefix/lib/spack. Most + of them are in the spack module, so all the packages live in + $prefix/lib/spack/spack/packages. + + All you have to do to create a package is make a new subclass of Package + in this directory. Spack automatically scans the python files there + and figures out which one to import when you invoke it. + + An example package + ==================== + Let's look at the cmake package to start with. This package lives in + $prefix/lib/spack/spack/packages/cmake.py: + + from spack import * + class Cmake(object): + homepage = 'https://www.cmake.org' + url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz' + md5 = '097278785da7182ec0aea8769d06860c' + + def install(self, prefix): + configure('--prefix=%s' % prefix, + '--parallel=%s' % make_jobs) + make() + make('install') + + Naming conventions + --------------------- + There are two names you should care about: + + 1. The module name, 'cmake'. + - User will refers to this name, e.g. 'spack install cmake'. + - Corresponds to the name of the file, 'cmake.py', and it can + include _, -, and numbers (it can even start with a number). + + 2. The class name, "Cmake". This is formed by converting -'s or _'s + in the module name to camel case. If the name starts with a number, + we prefix the class name with 'Num_'. Examples: + + Module Name Class Name + foo_bar FooBar + docbook-xml DocbookXml + FooBar Foobar + 3proxy Num_3proxy + + The class name is what spack looks for when it loads a package module. + + Required Attributes + --------------------- + Aside from proper naming, here is the bare minimum set of things you + need when you make a package: + homepage informational URL, so that users know what they're + installing. + + url URL of the source archive that spack will fetch. + + md5 md5 hash of the source archive, so that we can + verify that it was downloaded securely and correctly. + + install() This function tells spack how to build and install the + software it downloaded. + + Creating Packages + =================== + As a package creator, you can probably ignore most of the preceding + information, because you can use the 'spack create' command to do it + all automatically. + + You as the package creator generally only have to worry about writing + your install function and specifying dependencies. + + spack create + ---------------- + Most software comes in nicely packaged tarballs, like this one: + http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + + Taking a page from homebrew, spack deduces pretty much everything it + needs to know from the URL above. If you simply type this: + + spack create http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + + Spack will download the tarball, generate an md5 hash, figure out the + version and the name of the package from the URL, and create a new + package file for you with all the names and attributes set correctly. + + Once this skeleton code is generated, spack pops up the new package in + your $EDITOR so that you can modify the parts that need changes. + + Dependencies + --------------- + If your package requires another in order to build, you can specify that + like this: + + class Stackwalker(Package): + ... + depends_on("libdwarf") + ... + + This tells spack that before it builds stackwalker, it needs to build + the libdwarf package as well. Note that this is the module name, not + the class name (The class name is really only used by spack to find + your package). + + Spack will download an install each dependency before it installs your + package. In addtion, it will add -L, -I, and rpath arguments to your + compiler and linker for each dependency. In most cases, this allows you + to avoid specifying any dependencies in your configure or cmake line; + you can just run configure or cmake without any additional arguments and + it will find the dependencies automatically. + + + The Install Function + ---------------------- + The install function is designed so that someone not too terribly familiar + with Python could write a package installer. For example, we put a number + of commands in install scope that you can use almost like shell commands. + These include make, configure, cmake, rm, rmtree, mkdir, mkdirp, and others. + + You can see above in the cmake script that these commands are used to run + configure and make almost like they're used on the command line. The + only difference is that they are python function calls and not shell + commands. + + It may be puzzling to you where the commands and functions in install live. + They are NOT instance variables on the class; this would require us to + type 'self.' all the time and it makes the install code unnecessarily long. + Rather, spack puts these commands and variables in *module* scope for your + Package subclass. Since each package has its own module, this doesn't + pollute other namespaces, and it allows you to more easily implement an + install function. + + For a full list of commands and variables available in module scope, see the + add_commands_to_module() function in this class. This is where most of + them are created and set on the module. + + + Parallel Builds + ------------------- + By default, Spack will run make in parallel when you run make() in your + install function. Spack figures out how many cores are available on + your system and runs make with -j. If you do not want this behavior, + you can explicitly mark a package not to use parallel make: + + class SomePackage(Package): + ... + parallel = False + ... + + This changes thd default behavior so that make is sequential. If you still + want to build some parts in parallel, you can do this in your install function: + + make(parallel=True) + + Likewise, if you do not supply parallel = True in your Package, you can keep + the default parallel behavior and run make like this when you want a + sequential build: + + make(parallel=False) + + Package Lifecycle + ================== + This section is really only for developers of new spack commands. + + A package's lifecycle over a run of Spack looks something like this: + + packge p = new Package() # Done for you by spack + + p.do_fetch() # called by spack commands in spack/cmd. + p.do_stage() # see spack.stage.Stage docs. + p.do_install() # calls package's install() function + p.do_uninstall() + + There are also some other commands that clean the build area: + p.do_clean() # runs make clean + p.do_clean_work() # removes the build directory and + # re-expands the archive. + p.do_clean_dist() # removes the stage directory entirely + + The convention used here is that a do_* function is intended to be called + internally by Spack commands (in spack.cmd). These aren't for package + writers to override, and doing so may break the functionality of the Package + class. + + Package creators override functions like install() (all of them do this), + clean() (some of them do this), and others to provide custom behavior. + """ + + def __init__(self, arch=arch.sys_type()): + attr.required(self, 'homepage') + attr.required(self, 'url') + attr.required(self, 'md5') + + attr.setdefault(self, 'dependencies', []) + attr.setdefault(self, 'parallel', True) + + # Architecture for this package. + self.arch = arch + + # Name of package is the name of its module (the file that contains it) + self.name = inspect.getmodulename(self.module.__file__) + + # Don't allow the default homepage. + if re.search(r'example.com', self.homepage): + tty.die("Bad homepage in %s: %s" % (self.name, self.homepage)) + + # Make sure URL is an allowed type + validate.url(self.url) + + # Set up version + attr.setdefault(self, 'version', version.parse_version(self.url)) + if not self.version: + tty.die("Couldn't extract version from %s. " + + "You must specify it explicitly for this URL." % self.url) + + # This adds a bunch of convenience commands to the package's module scope. + self.add_commands_to_module() + + # Controls whether install and uninstall check deps before acting. + self.ignore_dependencies = False + + # Empty at first; only compute dependents if necessary + self._dependents = None + + # Whether to remove intermediate build/install when things go wrong. + self.dirty = False + + # stage used to build this package. + Self.stage = Stage(self.stage_name, self.url) + + + def add_commands_to_module(self): + """Populate the module scope of install() with some useful functions. + This makes things easier for package writers. + """ + m = self.module + + m.make = MakeExecutable('make', self.parallel) + m.gmake = MakeExecutable('gmake', self.parallel) + + # number of jobs spack prefers to build with. + m.make_jobs = multiprocessing.cpu_count() + + # Find the configure script in the archive path + # Don't use which for this; we want to find it in the current dir. + m.configure = Executable('./configure') + m.cmake = which("cmake") + + # standard CMake arguments + m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % self.prefix, + '-DCMAKE_BUILD_TYPE=None'] + if platform.mac_ver()[0]: + m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') + + # Emulate some shell commands for convenience + m.cd = os.chdir + m.mkdir = os.mkdir + m.makedirs = os.makedirs + m.remove = os.remove + m.removedirs = os.removedirs + + m.mkdirp = mkdirp + m.install = install + m.rmtree = shutil.rmtree + m.move = shutil.move + + # Useful directories within the prefix + m.prefix = self.prefix + m.bin = new_path(self.prefix, 'bin') + m.sbin = new_path(self.prefix, 'sbin') + m.etc = new_path(self.prefix, 'etc') + m.include = new_path(self.prefix, 'include') + m.lib = new_path(self.prefix, 'lib') + m.lib64 = new_path(self.prefix, 'lib64') + m.libexec = new_path(self.prefix, 'libexec') + m.share = new_path(self.prefix, 'share') + m.doc = new_path(m.share, 'doc') + m.info = new_path(m.share, 'info') + m.man = new_path(m.share, 'man') + m.man1 = new_path(m.man, 'man1') + m.man2 = new_path(m.man, 'man2') + m.man3 = new_path(m.man, 'man3') + m.man4 = new_path(m.man, 'man4') + m.man5 = new_path(m.man, 'man5') + m.man6 = new_path(m.man, 'man6') + m.man7 = new_path(m.man, 'man7') + m.man8 = new_path(m.man, 'man8') + + @property + def dependents(self): + """List of names of packages that depend on this one.""" + if self._dependents is None: + packages.compute_dependents() + return tuple(self._dependents) + + + @property + def installed(self): + return os.path.exists(self.prefix) + + + @property + def installed_dependents(self): + installed = [d for d in self.dependents if packages.get(d).installed] + all_deps = [] + for d in installed: + all_deps.append(d) + all_deps.extend(packages.get(d).installed_dependents) + return tuple(all_deps) + + + @property + def all_dependents(self): + all_deps = list(self.dependents) + for pkg in self.dependents: + all_deps.extend(packages.get(pkg).all_dependents) + return tuple(all_deps) + + + @property + def stage_name(self): + return "%s-%s" % (self.name, self.version) + + + @property + def platform_path(self): + """Directory for binaries for the current platform.""" + return new_path(install_path, self.arch) + + + @property + def package_path(self): + """Directory for different versions of this package. Lives just above prefix.""" + return new_path(self.platform_path, self.name) + + + @property + def installed_versions(self): + return [ver for ver in os.listdir(self.package_path) + if os.path.isdir(new_path(self.package_path, ver))] + + + @property + def prefix(self): + """Packages are installed in $spack_prefix/opt///""" + return new_path(self.package_path, self.version) + + + def remove_prefix(self): + """Removes the prefix for a package along with any empty parent directories.""" + if os.path.exists(self.prefix): + shutil.rmtree(self.prefix, True) + + for dir in (self.package_path, self.platform_path): + if not os.path.isdir(dir): + continue + if not os.listdir(dir): + os.rmdir(dir) + else: + break + + + def do_fetch(self): + """Creates a stage directory and downloads the taball for this package. + Working directory will be set to the stage directory. + """ + stage = self.stage + stage.setup() + stage.fetch() + + archive_md5 = md5(stage.archive_file) + if archive_md5 != self.md5: + tty.die("MD5 Checksum failed for %s. Expected %s but got %s." + % (self.name, self.md5, archive_md5)) + + + def do_stage(self): + """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" + self.do_fetch() + stage = self.stage + + archive_dir = stage.expanded_archive_path + if not archive_dir: + tty.msg("Staging archive: %s" % stage.archive_file) + stage.expand_archive() + else: + tty.msg("Already staged %s" % self.name) + stage.chdir_to_archive() + + + def do_install(self): + """This class should call this version of the install method. + Package implementations should override install(). + """ + if os.path.exists(self.prefix): + tty.msg("%s is already installed." % self.name) + tty.pkg(self.prefix) + return + + if not self.ignore_dependencies: + self.do_install_dependencies() + + self.do_stage() + self.setup_install_environment() + + tty.msg("Building %s." % self.name) + try: + self.install(self.prefix) + if not os.path.isdir(self.prefix): + tty.die("Install failed for %s. No install dir created." % self.name) + except subprocess.CalledProcessError, e: + if not self.dirty: + self.remove_prefix() + tty.die("Install failed for %s" % self.name, e.message) + except Exception, e: + if not self.dirty: + self.remove_prefix() + raise + + tty.msg("Successfully installed %s" % self.name) + tty.pkg(self.prefix) + + # Once the install is done, destroy the stage where we built it, + # unless the user wants it kept around. + if not self.dirty: + self.stage.destroy() + + + def setup_install_environment(self): + """This ensures a clean install environment when we build packages.""" + pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") + + # Add spack environment at front of path and pass the + # lib location along so the compiler script can find spack + os.environ[SPACK_LIB] = lib_path + + # Fix for case-insensitive file systems. Conflicting links are + # in directories called "case*" within the env directory. + env_paths = [env_path] + for file in os.listdir(env_path): + path = new_path(env_path, file) + if file.startswith("case") and os.path.isdir(path): + env_paths.append(path) + path_put_first("PATH", env_paths) + path_set(SPACK_ENV_PATH, env_paths) + + # Pass along prefixes of dependencies here + path_set(SPACK_DEPENDENCIES, + [dep.package.prefix for dep in self.dependencies]) + + # Install location + os.environ[SPACK_PREFIX] = self.prefix + + # Build root for logging. + os.environ[SPACK_BUILD_ROOT] = self.stage.expanded_archive_path + + + def do_install_dependencies(self): + # Pass along paths of dependencies here + for dep in self.dependencies: + dep.package.do_install() + + + @property + def module(self): + """Use this to add variables to the class's module's scope. + This lets us use custom syntax in the install method. + """ + return __import__(self.__class__.__module__, + fromlist=[self.__class__.__name__]) + + + def install(self, prefix): + """Package implementations override this with their own build configuration.""" + tty.die("Packages must provide an install method!") + + + def do_uninstall(self): + if not os.path.exists(self.prefix): + tty.die(self.name + " is not installed.") + + if not self.ignore_dependencies: + deps = self.installed_dependents + if deps: tty.die( + "Cannot uninstall %s. The following installed packages depend on it:" + % self.name, " ".join(deps)) + + self.remove_prefix() + tty.msg("Successfully uninstalled %s." % self.name) + + + def do_clean(self): + if self.stage.expanded_archive_path: + self.stage.chdir_to_archive() + self.clean() + + + def clean(self): + """By default just runs make clean. Override if this isn't good.""" + try: + make = MakeExecutable('make', self.parallel) + make('clean') + tty.msg("Successfully cleaned %s" % self.name) + except subprocess.CalledProcessError, e: + tty.warn("Warning: 'make clean' didn't work. Consider 'spack clean --work'.") + + + def do_clean_work(self): + """By default just blows away the stage directory and re-stages.""" + self.stage.restage() + + + def do_clean_dist(self): + """Removes the stage directory where this package was built.""" + if os.path.exists(self.stage.path): + self.stage.destroy() + tty.msg("Successfully cleaned %s" % self.name) + + +class Dependency(object): + """Represents a dependency from one package to another.""" + def __init__(self, name, **kwargs): + self.name = name + for key in kwargs: + setattr(self, key, kwargs[key]) + + @property + def package(self): + return packages.get(self.name) + + def __repr__(self): + return "" % self.name + + def __str__(self): + return self.__repr__() + + +def depends_on(*args, **kwargs): + """Adds a depends_on local variable in the locals of + the calling class, based on args. + """ + # This gets the calling frame so we can pop variables into it + locals = sys._getframe(1).f_locals + + # Put deps into the dependencies variable + dependencies = locals.setdefault("dependencies", []) + for name in args: + dependencies.append(Dependency(name)) + + +class MakeExecutable(Executable): + """Special Executable for make so the user can specify parallel or + not on a per-invocation basis. Using 'parallel' as a kwarg will + override whatever the package's global setting is, so you can + either default to true or false and override particular calls. + + Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides + everything. + """ + def __init__(self, name, parallel): + super(MakeExecutable, self).__init__(name) + self.parallel = parallel + + def __call__(self, *args, **kwargs): + parallel = kwargs.get('parallel', self.parallel) + disable_parallel = env_flag(SPACK_NO_PARALLEL_MAKE) + + if parallel and not disable_parallel: + jobs = "-j%d" % multiprocessing.cpu_count() + args = (jobs,) + args + + super(MakeExecutable, self).__call__(*args, **kwargs) diff --git a/lib/spack/spack/packages/__init__.py b/lib/spack/spack/packages/__init__.py index 21b32b43cc..e571134538 100644 --- a/lib/spack/spack/packages/__init__.py +++ b/lib/spack/spack/packages/__init__.py @@ -74,7 +74,7 @@ def class_for(pkg): # If a class starts with a number, prefix it with Number_ to make it a valid # Python class name. if re.match(r'^[0-9]', class_name): - class_name = "Number_%s" % class_name + class_name = "Num_%s" % class_name return class_name diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 7b0840ae82..ed48a48758 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -5,47 +5,53 @@ import tempfile import getpass import spack -import packages import tty -def ensure_access(dir=spack.stage_path): - if not os.access(dir, os.R_OK|os.W_OK): - tty.die("Insufficient permissions on directory %s" % dir) - - -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the link. +class Stage(object): + """A Stage object manaages a directory where an archive is downloaded, + expanded, and built before being installed. A stage's lifecycle looks + like this: + + setup() Create the stage directory. + fetch() Fetch a source archive into the stage. + expand_archive() Expand the source archive. + Build and install the archive. This is handled + by the Package class. + destroy() Remove the stage once the package has been installed. + + If spack.use_tmp_stage is True, spack will attempt to create stages + in a tmp directory. Otherwise, stages are created directly in + spack.stage_path. """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - -def purge(): - """Remove any build directories in the stage path.""" - if os.path.isdir(spack.stage_path): - for stage_dir in os.listdir(spack.stage_path): - stage_path = spack.new_path(spack.stage_path, stage_dir) - remove_linked_tree(stage_path) - -class Stage(object): def __init__(self, stage_name, url): + """Create a stage object. + Parameters: + stage_name Name of the stage directory that will be created. + url URL of the archive to be downloaded into this stage. + """ self.stage_name = stage_name self.url = url @property def path(self): + """Absolute path to the stage directory.""" return spack.new_path(spack.stage_path, self.stage_name) def setup(self): - # If we're using a stag in tmp that has since been deleted, + """Creates the stage directory. + If spack.use_tmp_stage is False, the stage directory is created + directly under spack.stage_path. + + If spack.use_tmp_stage is True, this will attempt to create a + stage in a temporary directory and link it into spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs to + create a stage. If there is no valid location in tmp_dirs, fall + back to making the stage inside spack.stage_path. + """ + # If we're using a stage in tmp that has since been deleted, # remove the stale symbolic link. if os.path.islink(self.path): real_path = os.path.realpath(self.path) @@ -68,18 +74,23 @@ class Stage(object): if not os.path.isdir(self.path): tty.die("Stage path %s is not a directory!" % self.path) else: - # Now create the stage directory + # Create the top-level stage directory spack.mkdirp(spack.stage_path) - # And the stage for this build within it - if not spack.use_tmp_stage: - # non-tmp stage is just a directory in spack.stage_path - spack.mkdirp(self.path) - else: - # tmp stage is created in tmp but linked to spack.stage_path + # Find a tmp_dir if we're supposed to use one. + tmp_dir = None + if spack.use_tmp_stage: tmp_dir = next((tmp for tmp in spack.tmp_dirs - if os.access(tmp, os.R_OK|os.W_OK)), None) + if can_access(tmp)), None) + + if not tmp_dir: + # If we couldn't find a tmp dir or if we're not using tmp + # stages, create the stage directly in spack.stage_path. + spack.mkdirp(self.path) + else: + # Otherwise we found a tmp_dir, so create the stage there + # and link it back to the prefix. username = getpass.getuser() if username: tmp_dir = spack.new_path(tmp_dir, username) @@ -89,12 +100,13 @@ class Stage(object): os.symlink(tmp_dir, self.path) - # Finally make sure we can actually do something with the stage + # Make sure we can actually do something with the stage we made. ensure_access(self.path) @property def archive_file(self): + """Path to the source archive within this stage directory.""" path = os.path.join(self.path, os.path.basename(self.url)) if os.path.exists(path): return path @@ -155,6 +167,10 @@ class Stage(object): def expand_archive(self): + """Changes to the stage directory and attempt to expand the downloaded + archive. Fail if the stage is not set up or if the archive is not yet + downloaded. + """ self.chdir() if not self.archive_file: @@ -165,8 +181,8 @@ class Stage(object): def chdir_to_archive(self): - """Changes directory to the expanded archive directory if it exists. - Dies with an error otherwise. + """Changes directory to the expanded archive directory. + Dies with an error if there was no expanded archive. """ path = self.expanded_archive_path if not path: @@ -178,7 +194,9 @@ class Stage(object): def restage(self): - """Removes the expanded archive path if it exists, then re-expands the archive.""" + """Removes the expanded archive path if it exists, then re-expands + the archive. + """ if not self.archive_file: tty.die("Attempt to restage when not staged.") @@ -188,5 +206,38 @@ class Stage(object): def destroy(self): - """Blows away the stage directory. Can always call setup() again.""" + """Remove this stage directory.""" remove_linked_tree(self.path) + + + +def can_access(file=spack.stage_path): + """True if we have read/write access to the file.""" + return os.access(file, os.R_OK|os.W_OK) + + +def ensure_access(file=spack.stage_path): + """Ensure we can access a directory and die with an error if we can't.""" + if not can_access(file): + tty.die("Insufficient permissions for %s" % file) + + +def remove_linked_tree(path): + """Removes a directory and its contents. If the directory is a symlink, + follows the link and reamoves the real directory before removing the + link. + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) + + +def purge(): + """Remove all build directories in the top-level stage path.""" + if os.path.isdir(spack.stage_path): + for stage_dir in os.listdir(spack.stage_path): + stage_path = spack.new_path(spack.stage_path, stage_dir) + remove_linked_tree(stage_path) diff --git a/lib/spack/spack/test/test_versions.py b/lib/spack/spack/test/test_versions.py index e8a2295c1e..da3b61ec62 100755 --- a/lib/spack/spack/test/test_versions.py +++ b/lib/spack/spack/test/test_versions.py @@ -3,15 +3,15 @@ This file has a bunch of versions tests taken from the excellent version detection in Homebrew. """ -import spack.version as version import unittest +import spack.version as version +from spack.exception import * class VersionTest(unittest.TestCase): def assert_not_detected(self, string): - name, v = version.parse(string) - self.assertIsNone(v) + self.assertRaises(UndetectableVersionException, version.parse, string) def assert_detected(self, name, v, string): parsed_name, parsed_v = version.parse(string) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 792a2f6aa8..75b09c2f5a 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -2,6 +2,7 @@ import os import re import utils +from exception import * class Version(object): """Class to represent versions""" @@ -45,12 +46,13 @@ def canonical(v): return int(part) except: return part + return tuple(intify(v) for v in re.split(r'[_.-]+', v)) -def parse_version(spec): - """Try to extract a version from a filename or URL. This is taken - largely from Homebrew's Version class.""" +def parse_version_string_with_indices(spec): + """Try to extract a version string from a filename or URL. This is taken + largely from Homebrew's Version class.""" if os.path.isdir(spec): stem = os.path.basename(spec) @@ -76,7 +78,7 @@ def parse_version(spec): (r'[-_](R\d+[AB]\d*(-\d+)?)', spec), # e.g. boost_1_39_0 - (r'((\d+_)+\d+)$', stem, lambda s: s.replace('_', '.')), + (r'((\d+_)+\d+)$', stem), # e.g. foobar-4.5.1-1 # e.g. ruby-1.9.1-p243 @@ -119,11 +121,29 @@ def parse_version(spec): regex, match_string = vtype[:2] match = re.search(regex, match_string) if match and match.group(1) is not None: - if vtype[2:]: - return Version(vtype[2](match.group(1))) - else: - return Version(match.group(1)) - return None + return match.group(1), match.start(1), match.end(1) + + raise UndetectableVersionException(spec) + + +def parse_version(spec): + """Given a URL or archive name, extract a versino from it and return + a version object. + """ + ver, start, end = parse_version_string_with_indices(spec) + return Version(ver) + + +def create_version_format(spec): + """Given a URL or archive name, find the version and create a format string + that will allow another version to be substituted. + """ + ver, start, end = parse_version_string_with_indices(spec) + return spec[:start] + '%s' + spec[end:] + + +def replace_version(spec, new_version): + version = create_version_format(spec) def parse_name(spec, ver=None): @@ -142,7 +162,7 @@ def parse_name(spec, ver=None): match = re.search(nt, spec) if match: return match.group(1) - return None + raise UndetectableNameException(spec) def parse(spec): ver = parse_version(spec) -- cgit v1.2.3-70-g09d2