From a36f3764af424ce6da48adee2c09fe721c391942 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 7 Jul 2016 12:03:43 +0200 Subject: package : added hooks for generic phases --- lib/spack/spack/__init__.py | 4 +- lib/spack/spack/package.py | 162 +++++++++++++++++++++++++------------------- 2 files changed, 93 insertions(+), 73 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 20c9934704..0e811e5ee7 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -176,10 +176,10 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'StagedPackage', 'CMakePackage', \ +__all__ = ['Package', 'CMakePackage', \ 'Version', 'when', 'ver'] from spack.package import Package, ExtensionConflictError -from spack.package import StagedPackage, CMakePackage +from spack.package import CMakePackage from spack.version import Version, ver from spack.multimethod import when diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 74008c4dd9..e159bc59c5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -875,7 +875,7 @@ class Package(object): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - install_phases = set(['configure', 'build', 'install', 'provenance']) + _phases = ['install', 'create_spack_logs'] def do_install(self, keep_prefix=False, keep_stage=False, @@ -887,7 +887,7 @@ class Package(object): fake=False, explicit=False, dirty=False, - install_phases = install_phases): + allowed_phases=None): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -907,6 +907,10 @@ class Package(object): make_jobs -- Number of make jobs to use for install. Default is ncpus run_tests -- Runn tests within the package's install() """ + # FIXME : we need a better semantic + if allowed_phases is None: + allowed_phases = self._phases + if not self.spec.concrete: raise ValueError("Can only install concrete packages.") @@ -917,7 +921,8 @@ class Package(object): return # Ensure package is not already installed - if 'install' in install_phases and spack.install_layout.check_installed(self.spec): + # FIXME : This should be a pre-requisite to a phase + if 'install' in self._phases and spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) rec = spack.installed_db.get_record(self.spec) if (not rec.explicit) and explicit: @@ -960,7 +965,6 @@ class Package(object): tty.msg("Building %s" % self.name) self.stage.keep = keep_stage - self.install_phases = install_phases self.build_directory = join_path(self.stage.path, 'spack-build') self.source_directory = self.stage.source_path @@ -983,11 +987,25 @@ class Package(object): # the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') + # FIXME : refactor this assignment + self.log_path = log_path + self.env_path = env_path with log_output(log_file, verbose, sys.stdout.isatty(), True): dump_environment(env_path) - self.install(self.spec, self.prefix) - + try: + for phase in filter(lambda x: x in allowed_phases, self._phases): + # TODO : Log to screen the various phases + action = getattr(self, phase) + if getattr(action, 'preconditions', None): + action.preconditions() + action(self.spec, self.prefix) + if getattr(action, 'postconditions', None): + action.postconditions() + + except AttributeError as e: + # FIXME : improve error messages + raise ProcessError(e.message, long_message='') except ProcessError as e: # Annotate ProcessErrors with the location of # the build log @@ -995,29 +1013,9 @@ class Package(object): raise e # Ensure that something was actually installed. - if 'install' in self.install_phases: - self.sanity_check_prefix() - - - # Copy provenance into the install directory on success - if 'provenance' in self.install_phases: - log_install_path = spack.install_layout.build_log_path( - self.spec) - env_install_path = spack.install_layout.build_env_path( - self.spec) - packages_dir = spack.install_layout.build_packages_path( - self.spec) - - # Remove first if we're overwriting another build - # (can happen with spack setup) - try: - shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this - except: - pass - - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + # FIXME : This should be executed after 'install' as a postcondition + # if 'install' in self._phases: + # self.sanity_check_prefix() # Run post install hooks before build stage is removed. spack.hooks.post_install(self) @@ -1036,7 +1034,7 @@ class Package(object): # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) except directory_layout.InstallDirectoryAlreadyExistsError: - if 'install' in install_phases: + if 'install' in self._phases: # Abort install if install directory exists. # But do NOT remove it (you'd be overwriting someon else's stuff) tty.warn("Keeping existing install prefix in place.") @@ -1064,6 +1062,27 @@ class Package(object): # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix, explicit=explicit) + def create_spack_logs(self, spec, prefix): + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path( + self.spec) + env_install_path = spack.install_layout.build_env_path( + self.spec) + packages_dir = spack.install_layout.build_packages_path( + self.spec) + + # Remove first if we're overwriting another build + # (can happen with spack setup) + try: + shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this + except Exception: + # FIXME : this potentially catches too many things... + pass + + install(self.log_path, log_install_path) + install(self.env_path, env_install_path) + dump_packages(self.spec, packages_dir) + def sanity_check_prefix(self): """This function checks whether install succeeded.""" @@ -1529,42 +1548,42 @@ def _hms(seconds): parts.append("%.2fs" % s) return ' '.join(parts) -class StagedPackage(Package): - """A Package subclass where the install() is split up into stages.""" - - def install_setup(self): - """Creates an spack_setup.py script to configure the package later if we like.""" - raise InstallError("Package %s provides no install_setup() method!" % self.name) - - def install_configure(self): - """Runs the configure process.""" - raise InstallError("Package %s provides no install_configure() method!" % self.name) - - def install_build(self): - """Runs the build process.""" - raise InstallError("Package %s provides no install_build() method!" % self.name) - - def install_install(self): - """Runs the install process.""" - raise InstallError("Package %s provides no install_install() method!" % self.name) - - def install(self, spec, prefix): - if 'setup' in self.install_phases: - self.install_setup() - - if 'configure' in self.install_phases: - self.install_configure() - - if 'build' in self.install_phases: - self.install_build() - - if 'install' in self.install_phases: - self.install_install() - else: - # Create a dummy file so the build doesn't fail. - # That way, the module file will also be created. - with open(os.path.join(prefix, 'dummy'), 'w') as fout: - pass +#class StagedPackage(Package): +# """A Package subclass where the install() is split up into stages.""" +# _phases = ['configure'] +# def install_setup(self): +# """Creates an spack_setup.py script to configure the package later if we like.""" +# raise InstallError("Package %s provides no install_setup() method!" % self.name) +# +# def install_configure(self): +# """Runs the configure process.""" +# raise InstallError("Package %s provides no install_configure() method!" % self.name) +# +# def install_build(self): +# """Runs the build process.""" +# raise InstallError("Package %s provides no install_build() method!" % self.name) +# +# def install_install(self): +# """Runs the install process.""" +# raise InstallError("Package %s provides no install_install() method!" % self.name) +# +# def install(self, spec, prefix): +# if 'setup' in self._phases: +# self.install_setup() +# +# if 'configure' in self._phases: +# self.install_configure() +# +# if 'build' in self._phases: +# self.install_build() +# +# if 'install' in self._phases: +# self.install_install() +# else: +# # Create a dummy file so the build doesn't fail. +# # That way, the module file will also be created. +# with open(os.path.join(prefix, 'dummy'), 'w') as fout: +# pass # stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python def make_executable(path): @@ -1574,7 +1593,8 @@ def make_executable(path): -class CMakePackage(StagedPackage): +class CMakePackage(Package): + _phases = ['configure', 'build', 'install', 'provenance'] def make_make(self): import multiprocessing @@ -1602,7 +1622,7 @@ class CMakePackage(StagedPackage): for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) ) - def install_setup(self): + def setup(self): cmd = [str(which('cmake'))] + \ spack.build_environment.get_std_cmake_args(self) + \ ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], @@ -1657,7 +1677,7 @@ env = dict(os.environ) make_executable(setup_fname) - def install_configure(self): + def configure(self): cmake = which('cmake') with working_dir(self.build_directory, create=True): os.environ.update(self.configure_env()) @@ -1665,12 +1685,12 @@ env = dict(os.environ) options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) cmake(self.source_directory, *options) - def install_build(self): + def build(self): make = self.make_make() with working_dir(self.build_directory, create=False): make() - def install_install(self): + def install(self): make = self.make_make() with working_dir(self.build_directory, create=False): make('install') -- cgit v1.2.3-70-g09d2 From 8ed028e2d67f191a32751db8bd487c03f7687b50 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 8 Jul 2016 12:29:49 +0200 Subject: package : introduced InstallPhase, added decorators for prerequisites and sanity_checks of phases --- lib/spack/spack/package.py | 214 ++++++++++++++--------- var/spack/repos/builtin/packages/szip/package.py | 4 + 2 files changed, 133 insertions(+), 85 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e159bc59c5..4194f30827 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -35,10 +35,13 @@ README. """ import os import re +import string import textwrap import time -import glob -import string +import inspect +import functools +from StringIO import StringIO +from urlparse import urlparse import llnl.util.tty as tty import spack @@ -52,26 +55,95 @@ import spack.mirror import spack.repository import spack.url import spack.util.web - -from urlparse import urlparse -from StringIO import StringIO from llnl.util.filesystem import * from llnl.util.lang import * from llnl.util.link_tree import LinkTree from llnl.util.tty.log import log_output +from spack import directory_layout from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive from spack.util.environment import dump_environment -from spack.util.executable import ProcessError, Executable, which +from spack.util.executable import ProcessError, which from spack.version import * -from spack import directory_layout -from urlparse import urlparse """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] -class Package(object): +class InstallPhase(object): + """Manages a single phase of the installation + + This descriptor stores at creation time the name of the method it should search + for execution. The method is retrieved at get time, so that it can be overridden + by subclasses of whatever class declared the phases. + + It also provides hooks to execute prerequisite and sanity checks. + """ + def __init__(self, name): + self.name = name + self.preconditions = [] + self.sanity_checks = [] + + def __get__(self, instance, owner): + # The caller is a class that is trying to customize + # my behavior adding something + if instance is None: + return self + # If instance is there the caller wants to execute the + # install phase, thus return a properly set wrapper + phase = getattr(instance, self.name) + @functools.wraps(phase) + def phase_wrapper(spec, prefix): + # Execute phase pre-conditions, + # and give them the chance to fail + for check in self.preconditions: + check(instance) + # Do something sensible at some point + phase(spec, prefix) + # Execute phase sanity_checks, + # and give them the chance to fail + for check in self.sanity_checks: + check(instance) + + return phase_wrapper + + +class PackageMeta(type): + """Conveniently transforms attributes to permit extensible phases + + Iterates over the attribute 'phase' and creates / updates private + InstallPhase attributes in the class that is being initialized + """ + phase_fmt = '_InstallPhase_{0}' + + def __init__(cls, name, bases, attr_dict): + super(PackageMeta, cls).__init__(name, bases, attr_dict) + # Parse if phases is in attr dict, then set + # install phases wrappers + if 'phases' in attr_dict: + cls.phases = [PackageMeta.phase_fmt.format(name) for name in attr_dict['phases']] + for phase_name, callback_name in zip(cls.phases, attr_dict['phases']): + setattr(cls, phase_name, InstallPhase(callback_name)) + + def _transform_checks(check_name): + attr_name = PackageMeta.phase_fmt.format(check_name) + checks = getattr(cls, attr_name, None) + if checks: + for phase_name, funcs in checks.items(): + phase = getattr(cls, PackageMeta.phase_fmt.format(phase_name)) + getattr(phase, check_name).extend(funcs) + # TODO : this should delete the attribute, as it is just a placeholder + # TODO : to know what to do at class definition time. Clearing it is fine + # TODO : too, but it just leaves an empty dictionary in place + setattr(cls, attr_name, {}) + + # Preconditions + _transform_checks('preconditions') + # Sanity checks + _transform_checks('sanity_checks') + + +class PackageBase(object): """This is the superclass for all spack packages. ***The Package class*** @@ -304,6 +376,7 @@ class Package(object): clean() (some of them do this), and others to provide custom behavior. """ + __metaclass__ = PackageMeta # # These are default values for instance variables. # @@ -410,7 +483,6 @@ class Package(object): """Return the directory where the package.py file lives.""" return os.path.dirname(self.module.__file__) - @property def global_license_dir(self): """Returns the directory where global license files for all @@ -875,7 +947,6 @@ class Package(object): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - _phases = ['install', 'create_spack_logs'] def do_install(self, keep_prefix=False, keep_stage=False, @@ -909,7 +980,7 @@ class Package(object): """ # FIXME : we need a better semantic if allowed_phases is None: - allowed_phases = self._phases + allowed_phases = self.phases if not self.spec.concrete: raise ValueError("Can only install concrete packages.") @@ -921,8 +992,8 @@ class Package(object): return # Ensure package is not already installed - # FIXME : This should be a pre-requisite to a phase - if 'install' in self._phases and spack.install_layout.check_installed(self.spec): + # FIXME : skip condition : if any is True skip the installation + if 'install' in self.phases and spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) rec = spack.installed_db.get_record(self.spec) if (not rec.explicit) and explicit: @@ -994,15 +1065,9 @@ class Package(object): True): dump_environment(env_path) try: - for phase in filter(lambda x: x in allowed_phases, self._phases): + for phase in filter(lambda x: x in allowed_phases, self.phases): # TODO : Log to screen the various phases - action = getattr(self, phase) - if getattr(action, 'preconditions', None): - action.preconditions() - action(self.spec, self.prefix) - if getattr(action, 'postconditions', None): - action.postconditions() - + getattr(self, phase)(self.spec, self.prefix) except AttributeError as e: # FIXME : improve error messages raise ProcessError(e.message, long_message='') @@ -1012,11 +1077,6 @@ class Package(object): e.build_log = log_path raise e - # Ensure that something was actually installed. - # FIXME : This should be executed after 'install' as a postcondition - # if 'install' in self._phases: - # self.sanity_check_prefix() - # Run post install hooks before build stage is removed. spack.hooks.post_install(self) @@ -1034,7 +1094,7 @@ class Package(object): # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) except directory_layout.InstallDirectoryAlreadyExistsError: - if 'install' in self._phases: + if 'install' in self.phases: # Abort install if install directory exists. # But do NOT remove it (you'd be overwriting someon else's stuff) tty.warn("Keeping existing install prefix in place.") @@ -1062,7 +1122,7 @@ class Package(object): # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix, explicit=explicit) - def create_spack_logs(self, spec, prefix): + def log(self, spec, prefix): # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path( self.spec) @@ -1241,13 +1301,6 @@ class Package(object): """ pass - def install(self, spec, prefix): - """ - Package implementations override this with their own configuration - """ - raise InstallError("Package %s provides no install method!" % - self.name) - def do_uninstall(self, force=False): if not self.installed: raise InstallError(str(self.spec) + " is not installed.") @@ -1446,6 +1499,33 @@ class Package(object): """ return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) + @classmethod + def _register_checks(cls, check_type, *args): + def _register_sanity_checks(func): + attr_name = PackageMeta.phase_fmt.format(check_type) + sanity_checks = getattr(cls, attr_name, {}) + for item in args: + checks = sanity_checks.setdefault(item, []) + checks.append(func) + setattr(cls, attr_name, sanity_checks) + return func + return _register_sanity_checks + + @classmethod + def precondition(cls, *args): + return cls._register_checks('preconditions', *args) + + @classmethod + def sanity_check(cls, *args): + return cls._register_checks('sanity_checks', *args) + + +class Package(PackageBase): + phases = ['install', 'log'] + # This will be used as a registration decorator in user + # packages, if need be + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" @@ -1548,53 +1628,16 @@ def _hms(seconds): parts.append("%.2fs" % s) return ' '.join(parts) -#class StagedPackage(Package): -# """A Package subclass where the install() is split up into stages.""" -# _phases = ['configure'] -# def install_setup(self): -# """Creates an spack_setup.py script to configure the package later if we like.""" -# raise InstallError("Package %s provides no install_setup() method!" % self.name) -# -# def install_configure(self): -# """Runs the configure process.""" -# raise InstallError("Package %s provides no install_configure() method!" % self.name) -# -# def install_build(self): -# """Runs the build process.""" -# raise InstallError("Package %s provides no install_build() method!" % self.name) -# -# def install_install(self): -# """Runs the install process.""" -# raise InstallError("Package %s provides no install_install() method!" % self.name) -# -# def install(self, spec, prefix): -# if 'setup' in self._phases: -# self.install_setup() -# -# if 'configure' in self._phases: -# self.install_configure() -# -# if 'build' in self._phases: -# self.install_build() -# -# if 'install' in self._phases: -# self.install_install() -# else: -# # Create a dummy file so the build doesn't fail. -# # That way, the module file will also be created. -# with open(os.path.join(prefix, 'dummy'), 'w') as fout: -# pass - +# FIXME : remove this after checking that set_executable works the same way # stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python -def make_executable(path): - mode = os.stat(path).st_mode - mode |= (mode & 0o444) >> 2 # copy R bits to X - os.chmod(path, mode) +#def make_executable(path): +# mode = os.stat(path).st_mode +# mode |= (mode & 0o444) >> 2 # copy R bits to X +# os.chmod(path, mode) - -class CMakePackage(Package): - _phases = ['configure', 'build', 'install', 'provenance'] +class CMakePackage(PackageBase): + phases = ['setup', 'configure', 'build', 'install', 'provenance'] def make_make(self): import multiprocessing @@ -1614,6 +1657,7 @@ class CMakePackage(Package): def configure_env(self): """Returns package-specific environment under which the configure command should be run.""" + # FIXME : Why not EnvironmentModules return dict() def spack_transitive_include_path(self): @@ -1622,7 +1666,7 @@ class CMakePackage(Package): for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) ) - def setup(self): + def setup(self, spec, prefix): cmd = [str(which('cmake'))] + \ spack.build_environment.get_std_cmake_args(self) + \ ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], @@ -1674,10 +1718,10 @@ env = dict(os.environ) fout.write(' %s\n' % arg) fout.write('""") + sys.argv[1:]\n') fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') - make_executable(setup_fname) + set_executable(setup_fname) - def configure(self): + def configure(self, spec, prefix): cmake = which('cmake') with working_dir(self.build_directory, create=True): os.environ.update(self.configure_env()) @@ -1685,12 +1729,12 @@ env = dict(os.environ) options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) cmake(self.source_directory, *options) - def build(self): + def build(self, spec, prefix): make = self.make_make() with working_dir(self.build_directory, create=False): make() - def install(self): + def install(self, spec, prefix): make = self.make_make() with working_dir(self.build_directory, create=False): make('install') diff --git a/var/spack/repos/builtin/packages/szip/package.py b/var/spack/repos/builtin/packages/szip/package.py index fd3a2a209d..ec6d8239a9 100644 --- a/var/spack/repos/builtin/packages/szip/package.py +++ b/var/spack/repos/builtin/packages/szip/package.py @@ -34,6 +34,10 @@ class Szip(Package): version('2.1', '902f831bcefb69c6b635374424acbead') + @Package.sanity_check('install') + def always_raise(self): + raise RuntimeError('Precondition not respected') + def install(self, spec, prefix): configure('--prefix=%s' % prefix, '--enable-production', -- cgit v1.2.3-70-g09d2 From 8f75d343315a2bf7f85d613fdacb993a41a0f62b Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 8 Jul 2016 15:11:04 +0200 Subject: package : added a stub for AutotoolsPackage, examples in szip and swiftsim --- lib/spack/spack/__init__.py | 10 +- lib/spack/spack/package.py | 121 ++++++++++++++------- .../repos/builtin/packages/swiftsim/package.py | 18 ++- var/spack/repos/builtin/packages/szip/package.py | 32 +++--- 4 files changed, 109 insertions(+), 72 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0e811e5ee7..85a0a4e2f7 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -176,10 +176,14 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'CMakePackage', \ - 'Version', 'when', 'ver'] +__all__ = ['Package', + 'CMakePackage', + 'AutotoolsPackage', + 'Version', + 'when', + 'ver'] from spack.package import Package, ExtensionConflictError -from spack.package import CMakePackage +from spack.package import CMakePackage, AutotoolsPackage from spack.version import Version, ver from spack.multimethod import when diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4194f30827..dd5637b9bc 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -38,8 +38,8 @@ import re import string import textwrap import time -import inspect import functools +import inspect from StringIO import StringIO from urlparse import urlparse @@ -74,7 +74,7 @@ class InstallPhase(object): """Manages a single phase of the installation This descriptor stores at creation time the name of the method it should search - for execution. The method is retrieved at get time, so that it can be overridden + for execution. The method is retrieved at __get__ time, so that it can be overridden by subclasses of whatever class declared the phases. It also provides hooks to execute prerequisite and sanity checks. @@ -116,31 +116,63 @@ class PackageMeta(type): """ phase_fmt = '_InstallPhase_{0}' - def __init__(cls, name, bases, attr_dict): - super(PackageMeta, cls).__init__(name, bases, attr_dict) - # Parse if phases is in attr dict, then set + _InstallPhase_sanity_checks = {} + _InstallPhase_preconditions = {} + + def __new__(meta, name, bases, attr_dict): + # Check if phases is in attr dict, then set # install phases wrappers if 'phases' in attr_dict: - cls.phases = [PackageMeta.phase_fmt.format(name) for name in attr_dict['phases']] - for phase_name, callback_name in zip(cls.phases, attr_dict['phases']): - setattr(cls, phase_name, InstallPhase(callback_name)) + phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] + for phase_name, callback_name in zip(phases, attr_dict['phases']): + attr_dict[phase_name] = InstallPhase(callback_name) + attr_dict['phases'] = phases - def _transform_checks(check_name): + def _append_checks(check_name): + # Name of the attribute I am going to check it exists attr_name = PackageMeta.phase_fmt.format(check_name) - checks = getattr(cls, attr_name, None) + checks = getattr(meta, attr_name) if checks: for phase_name, funcs in checks.items(): - phase = getattr(cls, PackageMeta.phase_fmt.format(phase_name)) + phase = attr_dict.get(PackageMeta.phase_fmt.format(phase_name)) getattr(phase, check_name).extend(funcs) - # TODO : this should delete the attribute, as it is just a placeholder - # TODO : to know what to do at class definition time. Clearing it is fine - # TODO : too, but it just leaves an empty dictionary in place - setattr(cls, attr_name, {}) + # Clear the attribute for the next class + setattr(meta, attr_name, {}) + + @classmethod + def _register_checks(cls, check_type, *args): + def _register_sanity_checks(func): + attr_name = PackageMeta.phase_fmt.format(check_type) + sanity_checks = getattr(meta, attr_name) + for item in args: + checks = sanity_checks.setdefault(item, []) + checks.append(func) + setattr(meta, attr_name, sanity_checks) + return func + return _register_sanity_checks + + @classmethod + def precondition(cls, *args): + return cls._register_checks('preconditions', *args) + + @classmethod + def sanity_check(cls, *args): + return cls._register_checks('sanity_checks', *args) + + if all([not hasattr(x, '_register_checks') for x in bases]): + attr_dict['_register_checks'] = _register_checks + + if all([not hasattr(x, 'sanity_check') for x in bases]): + attr_dict['sanity_check'] = sanity_check + + if all([not hasattr(x, 'precondition') for x in bases]): + attr_dict['precondition'] = precondition # Preconditions - _transform_checks('preconditions') + _append_checks('preconditions') # Sanity checks - _transform_checks('sanity_checks') + _append_checks('sanity_checks') + return super(PackageMeta, meta).__new__(meta, name, bases, attr_dict) class PackageBase(object): @@ -993,7 +1025,7 @@ class PackageBase(object): # Ensure package is not already installed # FIXME : skip condition : if any is True skip the installation - if 'install' in self.phases and spack.install_layout.check_installed(self.spec): + if spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) rec = spack.installed_db.get_record(self.spec) if (not rec.explicit) and explicit: @@ -1499,29 +1531,39 @@ class PackageBase(object): """ return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) - @classmethod - def _register_checks(cls, check_type, *args): - def _register_sanity_checks(func): - attr_name = PackageMeta.phase_fmt.format(check_type) - sanity_checks = getattr(cls, attr_name, {}) - for item in args: - checks = sanity_checks.setdefault(item, []) - checks.append(func) - setattr(cls, attr_name, sanity_checks) - return func - return _register_sanity_checks - @classmethod - def precondition(cls, *args): - return cls._register_checks('preconditions', *args) +class Package(PackageBase): + phases = ['install', 'log'] + # This will be used as a registration decorator in user + # packages, if need be + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + - @classmethod - def sanity_check(cls, *args): - return cls._register_checks('sanity_checks', *args) +class AutotoolsPackage(PackageBase): + phases = ['autoreconf', 'configure', 'build', 'install', 'log'] + def autoreconf(self, spec, prefix): + """Not needed usually, configure should be already there""" + pass + + @PackageBase.sanity_check('autoreconf') + def is_configure_or_die(self): + if not os.path.exists('configure'): + raise RuntimeError('configure script not found in {0}'.format(os.getcwd())) + + def configure_args(self): + return list() + + def configure(self, spec, prefix): + options = ['--prefix={0}'.format(prefix)] + self.configure_args() + inspect.getmodule(self).configure(*options) + + def build(self, spec, prefix): + inspect.getmodule(self).make() + + def install(self, spec, prefix): + inspect.getmodule(self).make('install') -class Package(PackageBase): - phases = ['install', 'log'] # This will be used as a registration decorator in user # packages, if need be PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) @@ -1637,7 +1679,7 @@ def _hms(seconds): class CMakePackage(PackageBase): - phases = ['setup', 'configure', 'build', 'install', 'provenance'] + phases = ['configure', 'build', 'install', 'provenance'] def make_make(self): import multiprocessing @@ -1657,7 +1699,7 @@ class CMakePackage(PackageBase): def configure_env(self): """Returns package-specific environment under which the configure command should be run.""" - # FIXME : Why not EnvironmentModules + # FIXME : Why not EnvironmentModules and the hooks that PackageBase already provides ? return dict() def spack_transitive_include_path(self): @@ -1720,7 +1762,6 @@ env = dict(os.environ) fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') set_executable(setup_fname) - def configure(self, spec, prefix): cmake = which('cmake') with working_dir(self.build_directory, create=True): diff --git a/var/spack/repos/builtin/packages/swiftsim/package.py b/var/spack/repos/builtin/packages/swiftsim/package.py index 42e8fb466a..7c3204f96b 100644 --- a/var/spack/repos/builtin/packages/swiftsim/package.py +++ b/var/spack/repos/builtin/packages/swiftsim/package.py @@ -28,7 +28,7 @@ from spack import * import llnl.util.tty as tty -class Swiftsim(Package): +class Swiftsim(AutotoolsPackage): """ SPH With Inter-dependent Fine-grained Tasking (SWIFT) provides astrophysicists with a state of the art framework to perform @@ -59,19 +59,15 @@ class Swiftsim(Package): tty.warn('This is needed to clone SWIFT repository') spack_env.set('GIT_SSL_NO_VERIFY', 1) - def install(self, spec, prefix): - # Generate configure from configure.ac - # and Makefile.am + def autoreconf(self, spec, prefix): libtoolize() aclocal() autoconf() autogen = Executable('./autogen.sh') autogen() - # Configure and install - options = ['--prefix=%s' % prefix, - '--enable-mpi' if '+mpi' in spec else '--disable-mpi', - '--enable-optimization'] - configure(*options) - make() - make("install") + def config_args(self): + return ['--prefix=%s' % prefix, + '--enable-mpi' if '+mpi' in spec else '--disable-mpi', + '--with-metis={0}'.format(self.spec['metis'].prefix), + '--enable-optimization'] diff --git a/var/spack/repos/builtin/packages/szip/package.py b/var/spack/repos/builtin/packages/szip/package.py index ec6d8239a9..91934f7d03 100644 --- a/var/spack/repos/builtin/packages/szip/package.py +++ b/var/spack/repos/builtin/packages/szip/package.py @@ -24,26 +24,22 @@ ############################################################################## from spack import * -class Szip(Package): - """Szip is an implementation of the extended-Rice lossless compression algorithm. - It provides lossless compression of scientific data, and is provided with HDF - software products.""" - homepage = "https://www.hdfgroup.org/doc_resource/SZIP/" - url = "http://www.hdfgroup.org/ftp/lib-external/szip/2.1/src/szip-2.1.tar.gz" +class Szip(AutotoolsPackage): + """Szip is an implementation of the extended-Rice lossless + compression algorithm. - version('2.1', '902f831bcefb69c6b635374424acbead') + It provides lossless compression of scientific data, and is + provided with HDF software products. + """ - @Package.sanity_check('install') - def always_raise(self): - raise RuntimeError('Precondition not respected') + homepage = "https://www.hdfgroup.org/doc_resource/SZIP/" + url = "http://www.hdfgroup.org/ftp/lib-external/szip/2.1/src/szip-2.1.tar.gz" - def install(self, spec, prefix): - configure('--prefix=%s' % prefix, - '--enable-production', - '--enable-shared', - '--enable-static', - '--enable-encoding') + version('2.1', '902f831bcefb69c6b635374424acbead') - make() - make("install") + def configure_args(self): + return ['--enable-production', + '--enable-shared', + '--enable-static', + '--enable-encoding'] -- cgit v1.2.3-70-g09d2 From a43c63f14953ab105bf80e906dc212a62a0d3f67 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 11 Jul 2016 10:08:19 +0200 Subject: package : added EditableMakefile Modifications : - added EditableMakefile to PackageBase subclasses - astyle modified as an example - preliminary hook to stop at a certain phase of install --- lib/spack/spack/__init__.py | 3 +- lib/spack/spack/cmd/install.py | 7 ++- lib/spack/spack/package.py | 64 +++++++++++++++++----- var/spack/repos/builtin/packages/astyle/package.py | 24 ++++---- var/spack/repos/builtin/packages/blitz/package.py | 12 +--- var/spack/repos/builtin/packages/gmp/package.py | 10 +--- .../repos/builtin/packages/swiftsim/package.py | 6 +- 7 files changed, 78 insertions(+), 48 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 85a0a4e2f7..c924886b53 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -179,11 +179,12 @@ sys_type = None __all__ = ['Package', 'CMakePackage', 'AutotoolsPackage', + 'EditableMakefile', 'Version', 'when', 'ver'] from spack.package import Package, ExtensionConflictError -from spack.package import CMakePackage, AutotoolsPackage +from spack.package import CMakePackage, AutotoolsPackage, EditableMakefile from spack.version import Version, ver from spack.multimethod import when diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 4c076322a9..ae34e06c22 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -56,6 +56,9 @@ def setup_parser(subparser): subparser.add_argument( '--dirty', action='store_true', dest='dirty', help="Install a package *without* cleaning the environment.") + subparser.add_argument( + '--stop-at', help="Stop at a particular phase of installation" + ) subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") subparser.add_argument( @@ -88,4 +91,6 @@ def install(parser, args): verbose=args.verbose, fake=args.fake, dirty=args.dirty, - explicit=True) + explicit=True, + stop_at=args.stop_at + ) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index dd5637b9bc..2df6e6a433 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -104,6 +104,10 @@ class InstallPhase(object): # and give them the chance to fail for check in self.sanity_checks: check(instance) + # Permit instance to drive the execution + if self.name == instance.last_phase: + raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) + return phase_wrapper @@ -123,10 +127,10 @@ class PackageMeta(type): # Check if phases is in attr dict, then set # install phases wrappers if 'phases' in attr_dict: - phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] - for phase_name, callback_name in zip(phases, attr_dict['phases']): + _InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] + for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): attr_dict[phase_name] = InstallPhase(callback_name) - attr_dict['phases'] = phases + attr_dict['_InstallPhase_phases'] = _InstallPhase_phases def _append_checks(check_name): # Name of the attribute I am going to check it exists @@ -956,7 +960,8 @@ class PackageBase(object): return namespace def do_fake_install(self): - """Make a fake install directory contaiing a 'fake' file in bin.""" + """Make a fake install directory containing a 'fake' file in bin.""" + # FIXME : Make this part of the 'install' behavior ? mkdirp(self.prefix.bin) touch(join_path(self.prefix.bin, 'fake')) mkdirp(self.prefix.lib) @@ -990,7 +995,7 @@ class PackageBase(object): fake=False, explicit=False, dirty=False, - allowed_phases=None): + **kwargs): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -1010,9 +1015,13 @@ class PackageBase(object): make_jobs -- Number of make jobs to use for install. Default is ncpus run_tests -- Runn tests within the package's install() """ - # FIXME : we need a better semantic - if allowed_phases is None: - allowed_phases = self.phases + #if allowed_phases is None: + # allowed_phases = self.phases + # FIXME : Refine the error message + last_phase = kwargs.get('stop_at', None) + if last_phase is not None and last_phase not in self.phases: + raise KeyError('phase {0} is not among the allowed phases for package {1}'.format(last_phase, self)) + self.last_phase = last_phase if not self.spec.concrete: raise ValueError("Can only install concrete packages.") @@ -1097,9 +1106,10 @@ class PackageBase(object): True): dump_environment(env_path) try: - for phase in filter(lambda x: x in allowed_phases, self.phases): + for phase in self._InstallPhase_phases: # TODO : Log to screen the various phases getattr(self, phase)(self.spec, self.prefix) + self.log() except AttributeError as e: # FIXME : improve error messages raise ProcessError(e.message, long_message='') @@ -1126,9 +1136,10 @@ class PackageBase(object): # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) except directory_layout.InstallDirectoryAlreadyExistsError: + # FIXME : refactor this as a prerequisites to configure if 'install' in self.phases: # Abort install if install directory exists. - # But do NOT remove it (you'd be overwriting someon else's stuff) + # But do NOT remove it (you'd be overwriting someone else's stuff) tty.warn("Keeping existing install prefix in place.") raise else: @@ -1154,7 +1165,7 @@ class PackageBase(object): # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix, explicit=explicit) - def log(self, spec, prefix): + def log(self): # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path( self.spec) @@ -1533,14 +1544,41 @@ class PackageBase(object): class Package(PackageBase): - phases = ['install', 'log'] + phases = ['install'] # This will be used as a registration decorator in user # packages, if need be PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) +class EditableMakefile(PackageBase): + phases = ['edit', 'build', 'install'] + + def wdir(self): + return self.stage.source_path + + def build_args(self): + return list() + + def install_args(self): + return list() + + def edit(self, spec, prefix): + raise NotImplementedError('\'edit\' function not implemented') + + def build(self, spec, prefix): + args = self.build_args() + with working_dir(self.wdir()): + inspect.getmodule(self).make(*args) + + def install(self, spec, prefix): + args = self.install_args() + ['install'] + with working_dir(self.wdir()): + inspect.getmodule(self).make(*args) + + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + class AutotoolsPackage(PackageBase): - phases = ['autoreconf', 'configure', 'build', 'install', 'log'] + phases = ['autoreconf', 'configure', 'build', 'install'] def autoreconf(self, spec, prefix): """Not needed usually, configure should be already there""" diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index 815c184577..8f85d4ebbb 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -25,28 +25,24 @@ from spack import * -class Astyle(Package): +class Astyle(EditableMakefile): """ A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, Objective-C, C#, and Java Source Code. """ homepage = "http://astyle.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" + url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" version('2.04', '30b1193a758b0909d06e7ee8dd9627f6') - def install(self, spec, prefix): + parallel = False - with working_dir('src'): - # we need to edit the makefile in place to set compiler: - make_file = join_path(self.stage.source_path, - 'build', 'gcc', 'Makefile') - filter_file(r'^CXX\s*=.*', 'CXX=%s' % spack_cxx, make_file) + def wdir(self): + return join_path(self.stage.source_path, 'build', self.compiler.name) - make('-f', - make_file, - parallel=False) + def edit(self, spec, prefix): + makefile = join_path(self.wdir(), 'Makefile') + filter_file(r'^CXX\s*=.*', 'CXX=%s' % spack_cxx, makefile) - mkdirp(self.prefix.bin) - install(join_path(self.stage.source_path, 'src', 'bin', 'astyle'), - self.prefix.bin) + def install_args(self): + return ['prefix={0}'.format(prefix)] diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py index acc6ddcd07..16ad3bc2ab 100644 --- a/var/spack/repos/builtin/packages/blitz/package.py +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -24,16 +24,10 @@ ############################################################################## from spack import * -class Blitz(Package): + +class Blitz(AutotoolsPackage): """N-dimensional arrays for C++""" homepage = "http://github.com/blitzpp/blitz" - url = "https://github.com/blitzpp/blitz/tarball/1.0.0" + url = "https://github.com/blitzpp/blitz/tarball/1.0.0" version('1.0.0', '9f040b9827fe22228a892603671a77af') - - # No dependencies - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - make() - make("install") diff --git a/var/spack/repos/builtin/packages/gmp/package.py b/var/spack/repos/builtin/packages/gmp/package.py index 3933788425..9bfc8f9611 100644 --- a/var/spack/repos/builtin/packages/gmp/package.py +++ b/var/spack/repos/builtin/packages/gmp/package.py @@ -24,20 +24,16 @@ ############################################################################## from spack import * -class Gmp(Package): + +class Gmp(AutotoolsPackage): """GMP is a free library for arbitrary precision arithmetic, operating on signed integers, rational numbers, and floating-point numbers.""" homepage = "https://gmplib.org" - url = "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2" + url = "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2" version('6.1.0' , '86ee6e54ebfc4a90b643a65e402c4048') version('6.0.0a', 'b7ff2d88cae7f8085bd5006096eed470') version('6.0.0' , '6ef5869ae735db9995619135bd856b84') depends_on("m4") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/repos/builtin/packages/swiftsim/package.py b/var/spack/repos/builtin/packages/swiftsim/package.py index 7c3204f96b..f220335336 100644 --- a/var/spack/repos/builtin/packages/swiftsim/package.py +++ b/var/spack/repos/builtin/packages/swiftsim/package.py @@ -66,8 +66,8 @@ class Swiftsim(AutotoolsPackage): autogen = Executable('./autogen.sh') autogen() - def config_args(self): - return ['--prefix=%s' % prefix, - '--enable-mpi' if '+mpi' in spec else '--disable-mpi', + def configure_args(self): + return ['--prefix=%s' % self.prefix, + '--enable-mpi' if '+mpi' in self.spec else '--disable-mpi', '--with-metis={0}'.format(self.spec['metis'].prefix), '--enable-optimization'] -- cgit v1.2.3-70-g09d2 From 857d7bfe0ff1cd56d3c2b40dc8357ac729dcdb4e Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 11 Jul 2016 23:44:24 +0200 Subject: do_install : can stop at an arbitrary phase The mechanism would be simpler if we could leverage exceptions to raise signals. Unfortunately forking does not permit to do so. --- lib/spack/spack/package.py | 118 +++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 62 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2df6e6a433..a2a8998b86 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -104,10 +104,6 @@ class InstallPhase(object): # and give them the chance to fail for check in self.sanity_checks: check(instance) - # Permit instance to drive the execution - if self.name == instance.last_phase: - raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) - return phase_wrapper @@ -1015,14 +1011,6 @@ class PackageBase(object): make_jobs -- Number of make jobs to use for install. Default is ncpus run_tests -- Runn tests within the package's install() """ - #if allowed_phases is None: - # allowed_phases = self.phases - # FIXME : Refine the error message - last_phase = kwargs.get('stop_at', None) - if last_phase is not None and last_phase not in self.phases: - raise KeyError('phase {0} is not among the allowed phases for package {1}'.format(last_phase, self)) - self.last_phase = last_phase - if not self.spec.concrete: raise ValueError("Can only install concrete packages.") @@ -1059,6 +1047,16 @@ class PackageBase(object): # Set run_tests flag before starting build. self.run_tests = run_tests + last_phase = kwargs.get('stop_at', None) + phases_to_be_executed = self.phases + # We want to stop early + if last_phase is not None: + if last_phase not in self.phases: + raise KeyError('phase {0} is not among the allowed phases for package {1}'.format(last_phase, self)) + idx = self.phases.index(last_phase) + phases_to_be_executed = self.phases[:idx + 1] + keep_stage = True + # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1074,70 +1072,65 @@ class PackageBase(object): else: self.do_stage() - tty.msg("Building %s" % self.name) + tty.msg("Building {0} [{1}]".format(self.name, type(self).__base__ )) self.stage.keep = keep_stage self.build_directory = join_path(self.stage.path, 'spack-build') self.source_directory = self.stage.source_path - with self.stage: - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - if fake: - self.do_fake_install() - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() - - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') - - try: + try: + with self.stage: + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') # Redirect I/O to a build log (and optionally to # the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') # FIXME : refactor this assignment self.log_path = log_path self.env_path = env_path - with log_output(log_file, verbose, sys.stdout.isatty(), - True): - dump_environment(env_path) - try: - for phase in self._InstallPhase_phases: - # TODO : Log to screen the various phases - getattr(self, phase)(self.spec, self.prefix) - self.log() - except AttributeError as e: - # FIXME : improve error messages - raise ProcessError(e.message, long_message='') - except ProcessError as e: - # Annotate ProcessErrors with the location of - # the build log - e.build_log = log_path - raise e - - # Run post install hooks before build stage is removed. - spack.hooks.post_install(self) - - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time - - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." % - (_hms(self._fetch_time), _hms(build_time), - _hms(self._total_time))) - print_pkg(self.prefix) + dump_environment(env_path) + for phase_name, phase in zip(phases_to_be_executed, self._InstallPhase_phases): + log_file = open(log_path, 'a') + tty.msg('Executing phase : \'{0}\''.format(phase_name)) + with log_output(log_file, verbose, sys.stdout.isatty(), True): + getattr(self, phase)(self.spec, self.prefix) + if len(phases_to_be_executed) != len(self._InstallPhase_phases): + return + self.log() + # Run post install hooks before build stage is removed. + spack.hooks.post_install(self) + + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time + + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." % + (_hms(self._fetch_time), _hms(build_time), + _hms(self._total_time))) + print_pkg(self.prefix) + + except ProcessError as e: + # Annotate ProcessErrors with the location of + # the build log + e.build_log = log_path + raise e try: - # Create the install prefix and fork the build process. - spack.install_layout.create_install_directory(self.spec) + if len(phases_to_be_executed) == len(self.phases): + # Create the install prefix and fork the build process. + spack.install_layout.create_install_directory(self.spec) except directory_layout.InstallDirectoryAlreadyExistsError: # FIXME : refactor this as a prerequisites to configure - if 'install' in self.phases: + if 'install' in phases_to_be_executed: # Abort install if install directory exists. # But do NOT remove it (you'd be overwriting someone else's stuff) tty.warn("Keeping existing install prefix in place.") @@ -1163,7 +1156,8 @@ class PackageBase(object): # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix, explicit=explicit) + if len(phases_to_be_executed) == len(self.phases): + spack.installed_db.add(self.spec, self.prefix, explicit=explicit) def log(self): # Copy provenance into the install directory on success -- cgit v1.2.3-70-g09d2 From 440e71ff13d86856680257b282d016018101dd37 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jul 2016 10:39:20 +0200 Subject: build_environment : moved from os.fork to multiprocessing.Process --- lib/spack/spack/build_environment.py | 47 ++++++++++++------------------------ 1 file changed, 15 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index fe5186a7d7..839991deaa 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -499,41 +499,24 @@ def fork(pkg, function, dirty=False): well. If things go well, the child exits and the parent carries on. """ - try: - pid = os.fork() - except OSError as e: - raise InstallError("Unable to fork build process: %s" % e) - - if pid == 0: - # Give the child process the package's build environment. - setup_package(pkg, dirty=dirty) + def child_execution(child_connection): try: - # call the forked function. + setup_package(pkg, dirty=dirty) function() - - # Use os._exit here to avoid raising a SystemExit exception, - # which interferes with unit tests. - os._exit(0) - - except spack.error.SpackError as e: - e.die() - - except: - # Child doesn't raise or return to main spack code. - # Just runs default exception handler and exits. - sys.excepthook(*sys.exc_info()) - os._exit(1) - - else: - # Parent process just waits for the child to complete. If the - # child exited badly, assume it already printed an appropriate - # message. Just make the parent exit with an error code. - pid, returncode = os.waitpid(pid, 0) - if returncode != 0: - message = "Installation process had nonzero exit code : {code}" - strcode = str(returncode) - raise InstallError(message.format(code=strcode)) + child_connection.send([None, None, None]) + except Exception as e: + child_connection.send([type(e), e, None]) + finally: + child_connection.close() + + parent_connection, child_connection = multiprocessing.Pipe() + p = multiprocessing.Process(target=child_execution, args=(child_connection,)) + p.start() + exc_type, exception, traceback = parent_connection.recv() + p.join() + if exception is not None: + raise exception class InstallError(spack.error.SpackError): -- cgit v1.2.3-70-g09d2 From 9af964a6d6e87f62cfea73ababb489d27b272120 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jul 2016 17:13:41 +0200 Subject: log_output : moved from os.fork to multiprocessing.Process --- lib/spack/llnl/util/tty/log.py | 196 ++++++++++++++++------------------------- 1 file changed, 78 insertions(+), 118 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index ca82da7b17..9b9278b73c 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -29,6 +29,7 @@ import os import re import select import inspect +import multiprocessing import llnl.util.tty as tty import llnl.util.tty.color as color @@ -103,155 +104,114 @@ class log_output(object): """Redirects output and error of enclosed block to a file. Usage: - with log_output(open('logfile.txt', 'w')): + with log_output('logfile.txt', 'w'): # do things ... output will be logged. or: - with log_output(open('logfile.txt', 'w'), echo=True): + with log_output('logfile.txt', echo=True): # do things ... output will be logged # and also printed to stdout. - Closes the provided stream when done with the block. + Opens a stream in 'w' mode at instance creation and closes + it at instance deletion If echo is True, also prints the output to stdout. """ - def __init__(self, stream, echo=False, force_color=False, debug=False): - self.stream = stream - - # various output options + def __init__(self, filename, echo=False, force_color=False, debug=False): + self.filename = filename + # Various output options self.echo = echo self.force_color = force_color self.debug = debug - # Default is to try file-descriptor reassignment unless the system + # Default is to try file-descriptor reassignment unless the system # out/err streams do not have an associated file descriptor self.directAssignment = False + self.read, self.write = os.pipe() - def trace(self, frame, event, arg): - """Jumps to __exit__ on the child process.""" - raise _SkipWithBlock() - + # Spawn a daemon that writes what it reads from a pipe + self.p = multiprocessing.Process(target=self._forward_redirected_pipe, args=(self.read,), name='logger_daemon') + self.p.daemon = True + self.p.start() def __enter__(self): """Redirect output from the with block to a file. - This forks the with block as a separate process, with stdout - and stderr redirected back to the parent via a pipe. If - echo is set, also writes to standard out. - + Hijacks stdout / stderr and writes to the pipe + connected to the logger daemon """ # remember these values for later. self._force_color = color._force_color self._debug = tty._debug - - read, write = os.pipe() - - self.pid = os.fork() - if self.pid: - # Parent: read from child, skip the with block. - os.close(write) - - read_file = os.fdopen(read, 'r', 0) - with self.stream as log_file: - with keyboard_input(sys.stdin): - while True: - rlist, w, x = select.select([read_file, sys.stdin], [], []) - if not rlist: + # Redirect this output to a pipe + self._redirect_to_pipe(self.write) + + def _forward_redirected_pipe(self, read): + # Parent: read from child, skip the with block. + read_file = os.fdopen(read, 'r', 0) + with open(self.filename, 'w') as log_file: + with keyboard_input(sys.stdin): + while True: + rlist, _, _ = select.select([read_file, sys.stdin], [], []) + if not rlist: + break + + # Allow user to toggle echo with 'v' key. + # Currently ignores other chars. + if sys.stdin in rlist: + if sys.stdin.read(1) == 'v': + self.echo = not self.echo + + # Handle output from the with block process. + if read_file in rlist: + line = read_file.readline() + if not line: break - # Allow user to toggle echo with 'v' key. - # Currently ignores other chars. - if sys.stdin in rlist: - if sys.stdin.read(1) == 'v': - self.echo = not self.echo - - # handle output from the with block process. - if read_file in rlist: - line = read_file.readline() - if not line: - break - - # Echo to stdout if requested. - if self.echo: - sys.stdout.write(line) + # Echo to stdout if requested. + if self.echo: + sys.stdout.write(line) - # Stripped output to log file. - log_file.write(_strip(line)) - - read_file.flush() - read_file.close() - - # Set a trace function to skip the with block. - sys.settrace(lambda *args, **keys: None) - frame = inspect.currentframe(1) - frame.f_trace = self.trace - - else: - # Child: redirect output, execute the with block. - os.close(read) - - try: - # Save old stdout and stderr - self._stdout = os.dup(sys.stdout.fileno()) - self._stderr = os.dup(sys.stderr.fileno()) - - # redirect to the pipe. - os.dup2(write, sys.stdout.fileno()) - os.dup2(write, sys.stderr.fileno()) - except AttributeError: - self.directAssignment = True - self._stdout = sys.stdout - self._stderr = sys.stderr - output_redirect = os.fdopen(write, 'w') - sys.stdout = output_redirect - sys.stderr = output_redirect - - if self.force_color: - color._force_color = True - - if self.debug: - tty._debug = True + # Stripped output to log file. + log_file.write(_strip(line)) + log_file.flush() + def _redirect_to_pipe(self, write): + try: + # Save old stdout and stderr + self._stdout = os.dup(sys.stdout.fileno()) + self._stderr = os.dup(sys.stderr.fileno()) + + # redirect to the pipe. + os.dup2(write, sys.stdout.fileno()) + os.dup2(write, sys.stderr.fileno()) + except AttributeError: + self.directAssignment = True + self._stdout = sys.stdout + self._stderr = sys.stderr + output_redirect = os.fdopen(write, 'w') + sys.stdout = output_redirect + sys.stderr = output_redirect + if self.force_color: + color._force_color = True + if self.debug: + tty._debug = True def __exit__(self, exc_type, exception, traceback): - """Exits on child, handles skipping the with block on parent.""" - # Child should just exit here. - if self.pid == 0: - # Flush the log to disk. - sys.stdout.flush() - sys.stderr.flush() - - if exception: - # Restore stdout on the child if there's an exception, - # and let it be raised normally. - # - # This assumes that even if the exception is caught, - # the child will exit with a nonzero return code. If - # it doesn't, the child process will continue running. - # - # TODO: think about how this works outside install. - # TODO: ideally would propagate exception to parent... - if self.directAssignment: - sys.stdout = self._stdout - sys.stderr = self._stderr - else: - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) - - return False - - else: - # Die quietly if there was no exception. - os._exit(0) - - else: - # If the child exited badly, parent also should exit. - pid, returncode = os.waitpid(self.pid, 0) - if returncode != 0: - os._exit(1) + """Plugs back the original file descriptors + for stdout and stderr + """ + # Flush the log to disk. + sys.stdout.flush() + sys.stderr.flush() + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) # restore output options. color._force_color = self._force_color tty._debug = self._debug - # Suppresses exception if it's our own. - return exc_type is _SkipWithBlock + def __del__(self): + """Closes the pipes and joins the daemon""" + os.close(self.write) + self.p.join() + os.close(self.read) -- cgit v1.2.3-70-g09d2 From 513cdd580ea804c9861be07dd815b4f0745b8d4f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jul 2016 17:33:24 +0200 Subject: do_install : can stop at an arbitrary phase Now uses a StopIteration exception as a signal --- lib/spack/spack/package.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a2a8998b86..5ceb1ce2a2 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -104,7 +104,8 @@ class InstallPhase(object): # and give them the chance to fail for check in self.sanity_checks: check(instance) - + if getattr(instance, 'last_phase', None) == self.name: + raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) return phase_wrapper @@ -1047,15 +1048,9 @@ class PackageBase(object): # Set run_tests flag before starting build. self.run_tests = run_tests - last_phase = kwargs.get('stop_at', None) - phases_to_be_executed = self.phases - # We want to stop early - if last_phase is not None: - if last_phase not in self.phases: - raise KeyError('phase {0} is not among the allowed phases for package {1}'.format(last_phase, self)) - idx = self.phases.index(last_phase) - phases_to_be_executed = self.phases[:idx + 1] - keep_stage = True + self.last_phase = kwargs.get('stop_at', None) + if self.last_phase is not None and self.last_phase not in self.phases: + tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1097,13 +1092,11 @@ class PackageBase(object): self.log_path = log_path self.env_path = env_path dump_environment(env_path) - for phase_name, phase in zip(phases_to_be_executed, self._InstallPhase_phases): - log_file = open(log_path, 'a') + log_redirection = log_output(log_path, verbose, sys.stdout.isatty(), True) + for phase_name, phase in zip(self.phases, self._InstallPhase_phases): tty.msg('Executing phase : \'{0}\''.format(phase_name)) - with log_output(log_file, verbose, sys.stdout.isatty(), True): + with log_redirection: getattr(self, phase)(self.spec, self.prefix) - if len(phases_to_be_executed) != len(self._InstallPhase_phases): - return self.log() # Run post install hooks before build stage is removed. spack.hooks.post_install(self) @@ -1125,9 +1118,8 @@ class PackageBase(object): raise e try: - if len(phases_to_be_executed) == len(self.phases): - # Create the install prefix and fork the build process. - spack.install_layout.create_install_directory(self.spec) + # Create the install prefix and fork the build process. + spack.install_layout.create_install_directory(self.spec) except directory_layout.InstallDirectoryAlreadyExistsError: # FIXME : refactor this as a prerequisites to configure if 'install' in phases_to_be_executed: @@ -1142,6 +1134,13 @@ class PackageBase(object): try: spack.build_environment.fork(self, build_process, dirty=dirty) + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix, explicit=explicit) + except StopIteration as e: + tty.msg(e.message) + if not keep_prefix: + self.remove_prefix() except: # remove the install prefix if anything went wrong during install. if not keep_prefix: @@ -1154,11 +1153,6 @@ class PackageBase(object): wrap=False) raise - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - if len(phases_to_be_executed) == len(self.phases): - spack.installed_db.add(self.spec, self.prefix, explicit=explicit) - def log(self): # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path( @@ -1571,6 +1565,7 @@ class EditableMakefile(PackageBase): PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + class AutotoolsPackage(PackageBase): phases = ['autoreconf', 'configure', 'build', 'install'] -- cgit v1.2.3-70-g09d2 From 813cb032c47b7b8507fba28a3629bec0d2b244cd Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jul 2016 18:00:00 +0200 Subject: package.py : updated logic to log.py rework Conflicts: lib/spack/spack/package.py --- lib/spack/llnl/util/tty/log.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 9b9278b73c..cff7f22dcd 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -203,8 +203,13 @@ class log_output(object): # Flush the log to disk. sys.stdout.flush() sys.stderr.flush() - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) + if self.directAssignment: + # We seem to need this only to pass test/install.py + sys.stdout = self._stdout + sys.stderr = self._stderr + else: + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) # restore output options. color._force_color = self._force_color -- cgit v1.2.3-70-g09d2 From 97c2224cd6445c0e5395b4ae30a9f4d3bf5670e0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jul 2016 21:28:09 +0200 Subject: package.py : extra arguments, fixed inheritance issue - added attribute to hold extra arguments in PackageBase instances - fixed registration from within packages - examples : hdf5, lzo --- lib/spack/spack/package.py | 16 +++++++++++- var/spack/repos/builtin/packages/hdf5/package.py | 31 ++++++++++++++---------- var/spack/repos/builtin/packages/lzo/package.py | 16 ++++++------ 3 files changed, 41 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5ceb1ce2a2..c61f8262f7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -40,6 +40,8 @@ import textwrap import time import functools import inspect +import copy + from StringIO import StringIO from urlparse import urlparse @@ -135,7 +137,17 @@ class PackageMeta(type): checks = getattr(meta, attr_name) if checks: for phase_name, funcs in checks.items(): - phase = attr_dict.get(PackageMeta.phase_fmt.format(phase_name)) + try: + # Search for the phase in the attribute dictionary + phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] + except KeyError: + # If it is not there it's in the bases + # and we added a check. We need to copy + # and extend + for base in bases: + phase = getattr(base, PackageMeta.phase_fmt.format(phase_name), None) + attr_dict[PackageMeta.phase_fmt.format(phase_name)] = copy.deepcopy(phase) + phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] getattr(phase, check_name).extend(funcs) # Clear the attribute for the next class setattr(meta, attr_name, {}) @@ -511,6 +523,8 @@ class PackageBase(object): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() + self.extra_args = {} + @property def package_dir(self): """Return the directory where the package.py file lives.""" diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 51a5823aa5..54c74901f0 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -27,7 +27,7 @@ from spack import * import shutil -class Hdf5(Package): +class Hdf5(AutotoolsPackage): """HDF5 is a data model, library, and file format for storing and managing data. It supports an unlimited variety of datatypes, and is designed for flexible and efficient I/O and for high volume and complex data. @@ -58,13 +58,15 @@ class Hdf5(Package): depends_on("szip", when='+szip') depends_on("zlib") - def validate(self, spec): + @AutotoolsPackage.precondition('configure') + def validate(self): """ Checks if incompatible variants have been activated at the same time :param spec: spec of the package :raises RuntimeError: in case of inconsistencies """ + spec = self.spec if '+fortran' in spec and not self.compiler.fc: msg = 'cannot build a fortran variant without a fortran compiler' raise RuntimeError(msg) @@ -73,8 +75,8 @@ class Hdf5(Package): msg = 'cannot use variant +threadsafe with either +cxx or +fortran' raise RuntimeError(msg) - def install(self, spec, prefix): - self.validate(spec) + def configure_args(self): + spec = self.spec # Handle compilation after spec validation extra_args = [] @@ -137,16 +139,19 @@ class Hdf5(Package): '--disable-hl', ]) - configure( - "--prefix=%s" % prefix, - "--with-zlib=%s" % spec['zlib'].prefix, - *extra_args) - make() - make("install") - self.check_install(spec) - - def check_install(self, spec): + return ["--with-zlib=%s" % spec['zlib'].prefix] + extra_args + #configure( + # "--prefix=%s" % prefix, + # "--with-zlib=%s" % spec['zlib'].prefix, + # *extra_args) + #make() + #make("install") + #self.check_install(spec) + + @AutotoolsPackage.sanity_check('install') + def check_install(self): "Build and run a small program to test the installed HDF5 library" + spec = self.spec print "Checking HDF5 installation..." checkdir = "spack-check" with working_dir(checkdir, create=True): diff --git a/var/spack/repos/builtin/packages/lzo/package.py b/var/spack/repos/builtin/packages/lzo/package.py index 0961bbb58c..edf6dc1d4c 100644 --- a/var/spack/repos/builtin/packages/lzo/package.py +++ b/var/spack/repos/builtin/packages/lzo/package.py @@ -25,7 +25,7 @@ from spack import * -class Lzo(Package): +class Lzo(AutotoolsPackage): """Real-time data compression library""" homepage = 'https://www.oberhumer.com/opensource/lzo/' @@ -37,13 +37,13 @@ class Lzo(Package): version('2.06', '95380bd4081f85ef08c5209f4107e9f8') version('2.05', 'c67cda5fa191bab761c7cb06fe091e36') - def install(self, spec, prefix): - configure_args = [ - '--prefix={0}'.format(prefix), + def configure_args(self): + return [ '--disable-dependency-tracking', '--enable-shared' ] - configure(*configure_args) - make() - make('check') - make('install') + + @AutotoolsPackage.sanity_check('build') + def check(self): + if self.extra_args.get('build-tests', False): + make('check') -- cgit v1.2.3-70-g09d2 From 5cc59507f724950f8671803bb9ff616b17f255f7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jul 2016 09:21:49 +0200 Subject: package.py : hdf5 and lzo have examples of run_tests --- lib/spack/spack/package.py | 40 +++++++++++++++++++----- var/spack/repos/builtin/packages/hdf5/package.py | 18 ++++------- var/spack/repos/builtin/packages/lzo/package.py | 5 +-- 3 files changed, 41 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c61f8262f7..50f4e715b1 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -96,25 +96,35 @@ class InstallPhase(object): phase = getattr(instance, self.name) @functools.wraps(phase) def phase_wrapper(spec, prefix): + # Check instance attributes at the beginning of a phase + self._on_phase_start(instance) # Execute phase pre-conditions, # and give them the chance to fail for check in self.preconditions: - check(instance) - # Do something sensible at some point + check(instance) # Do something sensible at some point phase(spec, prefix) # Execute phase sanity_checks, # and give them the chance to fail for check in self.sanity_checks: check(instance) - if getattr(instance, 'last_phase', None) == self.name: - raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) + # Check instance attributes at the end of a phase + self._on_phase_exit(instance) return phase_wrapper + def _on_phase_start(self, instance): + pass + + def _on_phase_exit(self, instance): + # If a phase has a matching last_phase attribute, + # stop the installation process raising a StopIteration + if getattr(instance, 'last_phase', None) == self.name: + raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) + class PackageMeta(type): """Conveniently transforms attributes to permit extensible phases - Iterates over the attribute 'phase' and creates / updates private + Iterates over the attribute 'phases' and creates / updates private InstallPhase attributes in the class that is being initialized """ phase_fmt = '_InstallPhase_{0}' @@ -156,14 +166,25 @@ class PackageMeta(type): def _register_checks(cls, check_type, *args): def _register_sanity_checks(func): attr_name = PackageMeta.phase_fmt.format(check_type) - sanity_checks = getattr(meta, attr_name) + check_list = getattr(meta, attr_name) for item in args: - checks = sanity_checks.setdefault(item, []) + checks = check_list.setdefault(item, []) checks.append(func) - setattr(meta, attr_name, sanity_checks) + setattr(meta, attr_name, check_list) return func return _register_sanity_checks + @staticmethod + def on_package_attributes(**attrs): + def _execute_under_condition(func): + @functools.wraps(func) + def _wrapper(instance): + # If all the attributes have the value we require, then execute + if all([getattr(instance, key, None) == value for key, value in attrs.items()]): + func(instance) + return _wrapper + return _execute_under_condition + @classmethod def precondition(cls, *args): return cls._register_checks('preconditions', *args) @@ -181,6 +202,9 @@ class PackageMeta(type): if all([not hasattr(x, 'precondition') for x in bases]): attr_dict['precondition'] = precondition + if all([not hasattr(x, 'on_package_attributes') for x in bases]): + attr_dict['on_package_attributes'] = on_package_attributes + # Preconditions _append_checks('preconditions') # Sanity checks diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 54c74901f0..3b2600e671 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -29,8 +29,8 @@ import shutil class Hdf5(AutotoolsPackage): """HDF5 is a data model, library, and file format for storing and managing - data. It supports an unlimited variety of datatypes, and is designed for - flexible and efficient I/O and for high volume and complex data. + data. It supports an unlimited variety of datatypes, and is designed for + flexible and efficient I/O and for high volume and complex data. """ homepage = "http://www.hdfgroup.org/HDF5/" @@ -54,9 +54,9 @@ class Hdf5(AutotoolsPackage): variant('szip', default=False, description='Enable szip support') variant('threadsafe', default=False, description='Enable thread-safe capabilities') - depends_on("mpi", when='+mpi') - depends_on("szip", when='+szip') - depends_on("zlib") + depends_on('mpi', when='+mpi') + depends_on('szip', when='+szip') + depends_on('zlib') @AutotoolsPackage.precondition('configure') def validate(self): @@ -140,15 +140,9 @@ class Hdf5(AutotoolsPackage): ]) return ["--with-zlib=%s" % spec['zlib'].prefix] + extra_args - #configure( - # "--prefix=%s" % prefix, - # "--with-zlib=%s" % spec['zlib'].prefix, - # *extra_args) - #make() - #make("install") - #self.check_install(spec) @AutotoolsPackage.sanity_check('install') + @AutotoolsPackage.on_package_attributes(run_tests=True) def check_install(self): "Build and run a small program to test the installed HDF5 library" spec = self.spec diff --git a/var/spack/repos/builtin/packages/lzo/package.py b/var/spack/repos/builtin/packages/lzo/package.py index edf6dc1d4c..9232bbe4e6 100644 --- a/var/spack/repos/builtin/packages/lzo/package.py +++ b/var/spack/repos/builtin/packages/lzo/package.py @@ -44,6 +44,7 @@ class Lzo(AutotoolsPackage): ] @AutotoolsPackage.sanity_check('build') + @AutotoolsPackage.on_package_attributes(run_tests=True) def check(self): - if self.extra_args.get('build-tests', False): - make('check') + make('check') + make('test') -- cgit v1.2.3-70-g09d2 From 468a6431f95e963fe97f3ee49b908566375674bd Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jul 2016 11:08:19 +0200 Subject: package.py : workaround for a known bug that was not fixed in python 2.6 https://bugs.python.org/issue1515 --- lib/spack/spack/package.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 50f4e715b1..29d7e1dca7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -120,6 +120,17 @@ class InstallPhase(object): if getattr(instance, 'last_phase', None) == self.name: raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) + def copy(self): + try: + return copy.deepcopy(self) + except TypeError: + # This bug-fix was not back-ported in Python 2.6 + # http://bugs.python.org/issue1515 + other = InstallPhase(self.name) + other.preconditions.extend(self.preconditions) + other.sanity_checks.extend(self.sanity_checks) + return other + class PackageMeta(type): """Conveniently transforms attributes to permit extensible phases @@ -156,7 +167,7 @@ class PackageMeta(type): # and extend for base in bases: phase = getattr(base, PackageMeta.phase_fmt.format(phase_name), None) - attr_dict[PackageMeta.phase_fmt.format(phase_name)] = copy.deepcopy(phase) + attr_dict[PackageMeta.phase_fmt.format(phase_name)] = phase.copy() phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] getattr(phase, check_name).extend(funcs) # Clear the attribute for the next class @@ -901,7 +912,6 @@ class PackageBase(object): self.stage.cache_local() - def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" -- cgit v1.2.3-70-g09d2 From ad16830f71812cbe951244ec115e8b3c33f5cefc Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jul 2016 10:05:34 +0200 Subject: log : added timeout to avoid deadlocks on daemon join --- lib/spack/llnl/util/tty/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index cff7f22dcd..7f88261e34 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -218,5 +218,5 @@ class log_output(object): def __del__(self): """Closes the pipes and joins the daemon""" os.close(self.write) - self.p.join() + self.p.join(60.0) # 1 minute to join the daemonic child os.close(self.read) -- cgit v1.2.3-70-g09d2 From 7cedd620f1e233e559879dbb84d23db52a1a4a3c Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jul 2016 17:36:06 +0200 Subject: package.py : added CMakePackage, changed qhull, ibmisc, openjpeg to work as examples --- lib/spack/spack/package.py | 254 ++++++++++++--------- var/spack/repos/builtin/packages/ibmisc/package.py | 2 +- .../repos/builtin/packages/openjpeg/package.py | 14 +- var/spack/repos/builtin/packages/qhull/package.py | 12 +- 4 files changed, 159 insertions(+), 123 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 29d7e1dca7..525ff91e87 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -33,15 +33,14 @@ Homebrew makes it very easy to create packages. For a complete rundown on spack and how it differs from homebrew, look at the README. """ +import copy +import functools +import inspect import os +import platform import re -import string import textwrap import time -import functools -import inspect -import copy - from StringIO import StringIO from urlparse import urlparse @@ -65,7 +64,7 @@ from spack import directory_layout from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive from spack.util.environment import dump_environment -from spack.util.executable import ProcessError, which +from spack.util.executable import ProcessError from spack.version import * """Allowed URL schemes for spack packages.""" @@ -1644,6 +1643,49 @@ class AutotoolsPackage(PackageBase): PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) +class CMakePackage(PackageBase): + phases = ['cmake', 'build', 'install'] + + def build_type(self): + return 'RelWithDebInfo' + + @property + def std_cmake_args(self): + # standard CMake arguments + args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(self.prefix), + '-DCMAKE_BUILD_TYPE:STRING={0}'.format(self.build_type())] + if platform.mac_ver()[0]: + args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') + + # Set up CMake rpath + args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') + rpaths = ':'.join(spack.build_environment.get_rpaths(self)) + args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) + return args + + def wdir(self): + return join_path(self.stage.source_path, 'spack-build') + + def cmake_args(self): + return list() + + def cmake(self, spec, prefix): + options = [self.source_directory] + self.std_cmake_args + self.cmake_args() + create = not os.path.exists(self.wdir()) + with working_dir(self.wdir(), create=create): + inspect.getmodule(self).cmake(*options) + + def build(self, spec, prefix): + with working_dir(self.wdir()): + inspect.getmodule(self).make() + + def install(self, spec, prefix): + with working_dir(self.wdir()): + inspect.getmodule(self).make('install') + + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + + def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" flatten_dependencies(spec, prefix) @@ -1753,107 +1795,107 @@ def _hms(seconds): # os.chmod(path, mode) -class CMakePackage(PackageBase): - phases = ['configure', 'build', 'install', 'provenance'] - - def make_make(self): - import multiprocessing - # number of jobs spack will to build with. - jobs = multiprocessing.cpu_count() - if not self.parallel: - jobs = 1 - elif self.make_jobs: - jobs = self.make_jobs - - make = spack.build_environment.MakeExecutable('make', jobs) - return make - - def configure_args(self): - """Returns package-specific arguments to be provided to the configure command.""" - return list() - - def configure_env(self): - """Returns package-specific environment under which the configure command should be run.""" - # FIXME : Why not EnvironmentModules and the hooks that PackageBase already provides ? - return dict() - - def spack_transitive_include_path(self): - return ';'.join( - os.path.join(dep, 'include') - for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) - ) - - def setup(self, spec, prefix): - cmd = [str(which('cmake'))] + \ - spack.build_environment.get_std_cmake_args(self) + \ - ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], - '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], - '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], - '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] + \ - self.configure_args() - - env = dict() - env['PATH'] = os.environ['PATH'] - env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() - env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - - setup_fname = 'spconfig.py' - with open(setup_fname, 'w') as fout: - fout.write(\ -r"""#!%s +# class CMakePackage(PackageBase): +# phases = ['configure', 'build', 'install', 'provenance'] # - -import sys -import os -import subprocess - -def cmdlist(str): - return list(x.strip().replace("'",'') for x in str.split('\n') if x) -env = dict(os.environ) -""" % sys.executable) - - env_vars = sorted(list(env.keys())) - for name in env_vars: - val = env[name] - if string.find(name, 'PATH') < 0: - fout.write('env[%s] = %s\n' % (repr(name),repr(val))) - else: - if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': - sep = ';' - else: - sep = ':' - - fout.write('env[%s] = "%s".join(cmdlist("""\n' % (repr(name),sep)) - for part in string.split(val, sep): - fout.write(' %s\n' % part) - fout.write('"""))\n') - - fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") - fout.write('\ncmd = cmdlist("""\n') - fout.write('%s\n' % cmd[0]) - for arg in cmd[1:]: - fout.write(' %s\n' % arg) - fout.write('""") + sys.argv[1:]\n') - fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') - set_executable(setup_fname) - - def configure(self, spec, prefix): - cmake = which('cmake') - with working_dir(self.build_directory, create=True): - os.environ.update(self.configure_env()) - os.environ['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() - options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) - cmake(self.source_directory, *options) - - def build(self, spec, prefix): - make = self.make_make() - with working_dir(self.build_directory, create=False): - make() - - def install(self, spec, prefix): - make = self.make_make() - with working_dir(self.build_directory, create=False): - make('install') +# def make_make(self): +# import multiprocessing +# # number of jobs spack will to build with. +# jobs = multiprocessing.cpu_count() +# if not self.parallel: +# jobs = 1 +# elif self.make_jobs: +# jobs = self.make_jobs +# +# make = spack.build_environment.MakeExecutable('make', jobs) +# return make +# +# def configure_args(self): +# """Returns package-specific arguments to be provided to the configure command.""" +# return list() +# +# def configure_env(self): +# """Returns package-specific environment under which the configure command should be run.""" +# # FIXME : Why not EnvironmentModules and the hooks that PackageBase already provides ? +# return dict() +# +# def spack_transitive_include_path(self): +# return ';'.join( +# os.path.join(dep, 'include') +# for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) +# ) +# +# def setup(self, spec, prefix): +# cmd = [str(which('cmake'))] + \ +# spack.build_environment.get_std_cmake_args(self) + \ +# ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], +# '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], +# '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], +# '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] + \ +# self.configure_args() +# +# env = dict() +# env['PATH'] = os.environ['PATH'] +# env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() +# env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] +# +# setup_fname = 'spconfig.py' +# with open(setup_fname, 'w') as fout: +# fout.write(\ +# r"""#!%s +# # +# +# import sys +# import os +# import subprocess +# +# def cmdlist(str): +# return list(x.strip().replace("'",'') for x in str.split('\n') if x) +# env = dict(os.environ) +# """ % sys.executable) +# +# env_vars = sorted(list(env.keys())) +# for name in env_vars: +# val = env[name] +# if string.find(name, 'PATH') < 0: +# fout.write('env[%s] = %s\n' % (repr(name),repr(val))) +# else: +# if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': +# sep = ';' +# else: +# sep = ':' +# +# fout.write('env[%s] = "%s".join(cmdlist("""\n' % (repr(name),sep)) +# for part in string.split(val, sep): +# fout.write(' %s\n' % part) +# fout.write('"""))\n') +# +# fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") +# fout.write('\ncmd = cmdlist("""\n') +# fout.write('%s\n' % cmd[0]) +# for arg in cmd[1:]: +# fout.write(' %s\n' % arg) +# fout.write('""") + sys.argv[1:]\n') +# fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') +# set_executable(setup_fname) +# +# def configure(self, spec, prefix): +# cmake = which('cmake') +# with working_dir(self.build_directory, create=True): +# os.environ.update(self.configure_env()) +# os.environ['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() +# options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) +# cmake(self.source_directory, *options) +# +# def build(self, spec, prefix): +# make = self.make_make() +# with working_dir(self.build_directory, create=False): +# make() +# +# def install(self, spec, prefix): +# make = self.make_make() +# with working_dir(self.build_directory, create=False): +# make('install') class FetchError(spack.error.SpackError): diff --git a/var/spack/repos/builtin/packages/ibmisc/package.py b/var/spack/repos/builtin/packages/ibmisc/package.py index 8e6cf429a7..0da3871582 100644 --- a/var/spack/repos/builtin/packages/ibmisc/package.py +++ b/var/spack/repos/builtin/packages/ibmisc/package.py @@ -34,7 +34,7 @@ class Ibmisc(CMakePackage): depends_on('cmake') depends_on('doxygen') - def configure_args(self): + def cmake_args(self): spec = self.spec return [ '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py index 1d00edb06d..cf2c80817f 100644 --- a/var/spack/repos/builtin/packages/openjpeg/package.py +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -25,9 +25,9 @@ from spack import * -class Openjpeg(Package): - """ - OpenJPEG is an open-source JPEG 2000 codec written in C language. +class Openjpeg(CMakePackage): + """OpenJPEG is an open-source JPEG 2000 codec written in C language. + It has been developed in order to promote the use of JPEG 2000, a still-image compression standard from the Joint Photographic Experts Group (JPEG). @@ -35,7 +35,7 @@ class Openjpeg(Package): ITU-T as a JPEG 2000 Reference Software. """ homepage = "https://github.com/uclouvain/openjpeg" - url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" + url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" version('2.1', '3e1c451c087f8462955426da38aa3b3d') version('2.0.1', '105876ed43ff7dbb2f90b41b5a43cfa5') @@ -44,9 +44,3 @@ class Openjpeg(Package): version('1.5.1', 'd774e4b5a0db5f0f171c4fc0aabfa14e') depends_on('cmake') - - def install(self, spec, prefix): - cmake('.', *std_cmake_args) - - make() - make("install") diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index 280c9cc12c..9d8184c745 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -24,7 +24,8 @@ ############################################################################## from spack import * -class Qhull(Package): + +class Qhull(CMakePackage): """Qhull computes the convex hull, Delaunay triangulation, Voronoi diagram, halfspace intersection about a point, furt hest-site Delaunay triangulation, and furthest-site Voronoi diagram. The @@ -47,8 +48,7 @@ class Qhull(Package): depends_on('cmake') - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") + @CMakePackage.sanity_check('build') + @CMakePackage.on_package_attributes(run_tests=True) + def check(self): + make('test') -- cgit v1.2.3-70-g09d2 From 90b131260bc04b0cdd2caf3959c970bb9fbc69fb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jul 2016 20:49:16 +0200 Subject: log : changed semantic for start / join (now it's explicit) --- lib/spack/llnl/util/tty/log.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 7f88261e34..4738c8981a 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -131,8 +131,16 @@ class log_output(object): # Spawn a daemon that writes what it reads from a pipe self.p = multiprocessing.Process(target=self._forward_redirected_pipe, args=(self.read,), name='logger_daemon') self.p.daemon = True + # I just need this to communicate to un-summon the daemon + self.parent_pipe, self.child_pipe = multiprocessing.Pipe() + + def acquire(self): self.p.start() + def release(self): + self.parent_pipe.send(True) + self.p.join(60.0) # 1 minute to join the child + def __enter__(self): """Redirect output from the with block to a file. @@ -165,6 +173,7 @@ class log_output(object): if read_file in rlist: line = read_file.readline() if not line: + # For some reason we never reach this point... break # Echo to stdout if requested. @@ -175,6 +184,9 @@ class log_output(object): log_file.write(_strip(line)) log_file.flush() + if self.child_pipe.poll(): + break + def _redirect_to_pipe(self, write): try: # Save old stdout and stderr @@ -216,7 +228,6 @@ class log_output(object): tty._debug = self._debug def __del__(self): - """Closes the pipes and joins the daemon""" + """Closes the pipes""" os.close(self.write) - self.p.join(60.0) # 1 minute to join the daemonic child os.close(self.read) -- cgit v1.2.3-70-g09d2 From 9d66b8549295919cf1dba0958f54e35ada4d22c5 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jul 2016 22:51:12 +0200 Subject: log : changed semantic for start / join (now it's explicit) --- lib/spack/spack/package.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 525ff91e87..e7df3430fe 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1140,10 +1140,12 @@ class PackageBase(object): self.env_path = env_path dump_environment(env_path) log_redirection = log_output(log_path, verbose, sys.stdout.isatty(), True) + log_redirection.acquire() for phase_name, phase in zip(self.phases, self._InstallPhase_phases): tty.msg('Executing phase : \'{0}\''.format(phase_name)) with log_redirection: getattr(self, phase)(self.spec, self.prefix) + log_redirection.release() self.log() # Run post install hooks before build stage is removed. spack.hooks.post_install(self) -- cgit v1.2.3-70-g09d2 From 00b8e0b56782b815424c569eda41c4ef9810b858 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 14 Jul 2016 09:11:44 +0200 Subject: package.py : joined and simplified try/except blocks in do_install --- lib/spack/spack/package.py | 64 ++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 30 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e7df3430fe..7a849b358a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1057,7 +1057,7 @@ class PackageBase(object): verbose -- Display verbose build output (by default, suppresses it) dirty -- Don't clean the build environment before installing. make_jobs -- Number of make jobs to use for install. Default is ncpus - run_tests -- Runn tests within the package's install() + run_tests -- Run tests within the package's install() """ if not self.spec.concrete: raise ValueError("Can only install concrete packages.") @@ -1069,7 +1069,6 @@ class PackageBase(object): return # Ensure package is not already installed - # FIXME : skip condition : if any is True skip the installation if spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) rec = spack.installed_db.get_record(self.spec) @@ -1079,6 +1078,8 @@ class PackageBase(object): rec.explicit = True return + self._do_install_pop_kwargs(kwargs) + tty.msg("Installing %s" % self.name) # First, install dependencies recursively. @@ -1095,10 +1096,6 @@ class PackageBase(object): # Set run_tests flag before starting build. self.run_tests = run_tests - self.last_phase = kwargs.get('stop_at', None) - if self.last_phase is not None and self.last_phase not in self.phases: - tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) - # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1169,38 +1166,45 @@ class PackageBase(object): try: # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) - except directory_layout.InstallDirectoryAlreadyExistsError: - # FIXME : refactor this as a prerequisites to configure - if 'install' in phases_to_be_executed: - # Abort install if install directory exists. - # But do NOT remove it (you'd be overwriting someone else's stuff) - tty.warn("Keeping existing install prefix in place.") - raise - else: - # We're not installing anyway, so don't worry if someone - # else has already written in the install directory - pass - - try: + # Fork a child to do the actual installation spack.build_environment.fork(self, build_process, dirty=dirty) + # If we installed then we should keep the prefix + keep_prefix = True if self.last_phase is None else keep_prefix # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix, explicit=explicit) + except directory_layout.InstallDirectoryAlreadyExistsError: + # Abort install if install directory exists. + # But do NOT remove it (you'd be overwriting someone else's stuff) + tty.warn("Keeping existing install prefix in place.") + raise except StopIteration as e: + # A StopIteration exception means that do_install + # was asked to stop early from clients tty.msg(e.message) + except Exception: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed. " + + "Manually remove this directory to fix:", + self.prefix, + wrap=False) + raise + finally: + # Remove the install prefix if anything went wrong during install. if not keep_prefix: self.remove_prefix() - except: - # remove the install prefix if anything went wrong during install. - if not keep_prefix: - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, - wrap=False) - raise + + def _do_install_pop_kwargs(self, kwargs): + """Pops kwargs from do_install before starting the installation + + Args: + kwargs: + 'stop_at': last installation phase to be executed (or None) + + """ + self.last_phase = kwargs.pop('stop_at', None) + if self.last_phase is not None and self.last_phase not in self.phases: + tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) def log(self): # Copy provenance into the install directory on success -- cgit v1.2.3-70-g09d2 From 1ecea4c2f12f3f70bcfcdc46dc01acbf6d285f09 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 14 Jul 2016 09:34:01 +0200 Subject: log : refactored acquire and release semantic to meet the context manager protocol --- lib/spack/llnl/util/tty/log.py | 139 ++++++++++++++++++++++------------------- lib/spack/spack/package.py | 14 ++--- 2 files changed, 80 insertions(+), 73 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 4738c8981a..747315f41c 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -101,19 +101,23 @@ class keyboard_input(object): class log_output(object): - """Redirects output and error of enclosed block to a file. + """Spawns a daemon that reads from a pipe and writes to a file Usage: - with log_output('logfile.txt', 'w'): - # do things ... output will be logged. + # Spawns the daemon + with log_output('logfile.txt', 'w') as log_redirection: + # do things ... output is not redirected + with log_redirection: + # do things ... output will be logged or: - with log_output('logfile.txt', echo=True): - # do things ... output will be logged - # and also printed to stdout. + with log_output('logfile.txt', echo=True) as log_redirection: + # do things ... output is not redirected + with log_redirection: + # do things ... output will be logged + # and also printed to stdout. - Opens a stream in 'w' mode at instance creation and closes - it at instance deletion + Opens a stream in 'w' mode at daemon spawning and closes it at daemon joining. If echo is True, also prints the output to stdout. """ def __init__(self, filename, echo=False, force_color=False, debug=False): @@ -128,32 +132,21 @@ class log_output(object): self.directAssignment = False self.read, self.write = os.pipe() - # Spawn a daemon that writes what it reads from a pipe - self.p = multiprocessing.Process(target=self._forward_redirected_pipe, args=(self.read,), name='logger_daemon') + # Sets a daemon that writes to file what it reads from a pipe + self.p = multiprocessing.Process(target=self._spawn_writing_daemon, args=(self.read,), name='logger_daemon') self.p.daemon = True - # I just need this to communicate to un-summon the daemon + # Needed to un-summon the daemon self.parent_pipe, self.child_pipe = multiprocessing.Pipe() - def acquire(self): + def __enter__(self): self.p.start() + return log_output.OutputRedirection(self) - def release(self): + def __exit__(self, exc_type, exc_val, exc_tb): self.parent_pipe.send(True) self.p.join(60.0) # 1 minute to join the child - def __enter__(self): - """Redirect output from the with block to a file. - - Hijacks stdout / stderr and writes to the pipe - connected to the logger daemon - """ - # remember these values for later. - self._force_color = color._force_color - self._debug = tty._debug - # Redirect this output to a pipe - self._redirect_to_pipe(self.write) - - def _forward_redirected_pipe(self, read): + def _spawn_writing_daemon(self, read): # Parent: read from child, skip the with block. read_file = os.fdopen(read, 'r', 0) with open(self.filename, 'w') as log_file: @@ -187,47 +180,61 @@ class log_output(object): if self.child_pipe.poll(): break - def _redirect_to_pipe(self, write): - try: - # Save old stdout and stderr - self._stdout = os.dup(sys.stdout.fileno()) - self._stderr = os.dup(sys.stderr.fileno()) - - # redirect to the pipe. - os.dup2(write, sys.stdout.fileno()) - os.dup2(write, sys.stderr.fileno()) - except AttributeError: - self.directAssignment = True - self._stdout = sys.stdout - self._stderr = sys.stderr - output_redirect = os.fdopen(write, 'w') - sys.stdout = output_redirect - sys.stderr = output_redirect - if self.force_color: - color._force_color = True - if self.debug: - tty._debug = True - - def __exit__(self, exc_type, exception, traceback): - """Plugs back the original file descriptors - for stdout and stderr - """ - # Flush the log to disk. - sys.stdout.flush() - sys.stderr.flush() - if self.directAssignment: - # We seem to need this only to pass test/install.py - sys.stdout = self._stdout - sys.stderr = self._stderr - else: - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) - - # restore output options. - color._force_color = self._force_color - tty._debug = self._debug - def __del__(self): """Closes the pipes""" os.close(self.write) os.close(self.read) + + class OutputRedirection(object): + def __init__(self, other): + self.__dict__.update(other.__dict__) + + def __enter__(self): + """Redirect output from the with block to a file. + + Hijacks stdout / stderr and writes to the pipe + connected to the logger daemon + """ + # remember these values for later. + self._force_color = color._force_color + self._debug = tty._debug + # Redirect this output to a pipe + write = self.write + try: + # Save old stdout and stderr + self._stdout = os.dup(sys.stdout.fileno()) + self._stderr = os.dup(sys.stderr.fileno()) + + # redirect to the pipe. + os.dup2(write, sys.stdout.fileno()) + os.dup2(write, sys.stderr.fileno()) + except AttributeError: + self.directAssignment = True + self._stdout = sys.stdout + self._stderr = sys.stderr + output_redirect = os.fdopen(write, 'w') + sys.stdout = output_redirect + sys.stderr = output_redirect + if self.force_color: + color._force_color = True + if self.debug: + tty._debug = True + + def __exit__(self, exc_type, exception, traceback): + """Plugs back the original file descriptors + for stdout and stderr + """ + # Flush the log to disk. + sys.stdout.flush() + sys.stderr.flush() + if self.directAssignment: + # We seem to need this only to pass test/install.py + sys.stdout = self._stdout + sys.stderr = self._stderr + else: + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) + + # restore output options. + color._force_color = self._force_color + tty._debug = self._debug diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 7a849b358a..c90fd5808b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1136,13 +1136,13 @@ class PackageBase(object): self.log_path = log_path self.env_path = env_path dump_environment(env_path) - log_redirection = log_output(log_path, verbose, sys.stdout.isatty(), True) - log_redirection.acquire() - for phase_name, phase in zip(self.phases, self._InstallPhase_phases): - tty.msg('Executing phase : \'{0}\''.format(phase_name)) - with log_redirection: - getattr(self, phase)(self.spec, self.prefix) - log_redirection.release() + # Spawn a daemon that reads from a pipe and redirects everything to log_path + with log_output(log_path, verbose, sys.stdout.isatty(), True) as log_redirection: + for phase_name, phase in zip(self.phases, self._InstallPhase_phases): + tty.msg('Executing phase : \'{0}\''.format(phase_name)) + # Redirect stdout and stderr to daemon pipe + with log_redirection: + getattr(self, phase)(self.spec, self.prefix) self.log() # Run post install hooks before build stage is removed. spack.hooks.post_install(self) -- cgit v1.2.3-70-g09d2 From b8fccb5f614cd0550c69b3e7951e94bfa9e41fca Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 14 Jul 2016 12:04:24 +0200 Subject: CMakePackage : added hook for roo CmakeLists.txt, removed duplicated code from build_environment.py --- lib/spack/spack/build_environment.py | 12 ++---------- lib/spack/spack/package.py | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 839991deaa..ec03caef94 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -344,16 +344,8 @@ def set_module_variables_for_package(pkg, module): m.cmake = Executable('cmake') m.ctest = Executable('ctest') - # standard CMake arguments - m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, - '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] - if platform.mac_ver()[0]: - m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') - - # Set up CMake rpath - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % - ":".join(get_rpaths(pkg))) + # Standard CMake arguments + m.std_cmake_args = spack.CMakePackage._std_args(pkg) # Put spack compiler paths in module scope. link_dir = spack.build_env_path diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c90fd5808b..c22625e44e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1655,17 +1655,29 @@ class CMakePackage(PackageBase): def build_type(self): return 'RelWithDebInfo' + def root_cmakelists_dir(self): + return self.source_directory + @property def std_cmake_args(self): # standard CMake arguments - args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(self.prefix), - '-DCMAKE_BUILD_TYPE:STRING={0}'.format(self.build_type())] + return CMakePackage._std_args(self) + + @staticmethod + def _std_args(pkg): + try: + build_type = pkg.build_type() + except AttributeError: + build_type = 'RelWithDebInfo' + + args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), + '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type)] if platform.mac_ver()[0]: args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') # Set up CMake rpath args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') - rpaths = ':'.join(spack.build_environment.get_rpaths(self)) + rpaths = ':'.join(spack.build_environment.get_rpaths(pkg)) args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) return args @@ -1676,7 +1688,7 @@ class CMakePackage(PackageBase): return list() def cmake(self, spec, prefix): - options = [self.source_directory] + self.std_cmake_args + self.cmake_args() + options = [self.root_cmakelists_dir()] + self.std_cmake_args + self.cmake_args() create = not os.path.exists(self.wdir()) with working_dir(self.wdir(), create=create): inspect.getmodule(self).cmake(*options) -- cgit v1.2.3-70-g09d2 From 833b0ac282a4fa43d2cb963c0162605e221833f7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 15 Jul 2016 11:03:02 +0200 Subject: Merge branch 'develop' of https://github.com/LLNL/spack into features/install_with_phases Conflicts: lib/spack/spack/__init__.py var/spack/repos/builtin/packages/gmp/package.py var/spack/repos/builtin/packages/openjpeg/package.py --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 795e22186c..abbf3cc87b 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -184,7 +184,7 @@ __all__ = ['Package', 'when', 'ver', 'alldeps', - 'nolinks'] + 'nolink'] from spack.package import Package, ExtensionConflictError from spack.package import CMakePackage, AutotoolsPackage, EditableMakefile from spack.version import Version, ver -- cgit v1.2.3-70-g09d2 From b92deda41bf2995a8a7b717ea8149af7797e30d5 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jul 2016 22:27:01 +0200 Subject: spack setup : work as in documentation for openjpeg --- lib/spack/spack/cmd/setup.py | 92 ++++++++++++++++++++++++++++++++---- lib/spack/spack/package.py | 110 ------------------------------------------- 2 files changed, 82 insertions(+), 120 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index 04f3d663df..8cf05b5c8b 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -22,19 +22,22 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys -import os import argparse +import os +import string +import sys import llnl.util.tty as tty - import spack import spack.cmd +from spack import which from spack.cmd.edit import edit_package from spack.stage import DIYStage +from llnl.util.filesystem import set_executable description = "Create a configuration script and module, but don't build." + def setup_parser(subparser): subparser.add_argument( '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', @@ -47,6 +50,75 @@ def setup_parser(subparser): help="specs to use for install. Must contain package AND version.") +def spack_transitive_include_path(): + return ';'.join( + os.path.join(dep, 'include') + for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) + ) + + +def write_spconfig(package): + spec = package.spec + prefix = spec.prefix + # Set-up the environment + spack.build_environment.setup_package(package) + + cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args() + + env = dict() + + paths = os.environ['PATH'].split(':') + paths = [item for item in paths if 'spack/env' not in item] + env['PATH'] = ':'.join(paths) + env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path() + env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] + env['CC'] = os.environ['SPACK_CC'] + env['CXX'] = os.environ['SPACK_CXX'] + env['FC'] = os.environ['SPACK_FC'] + + setup_fname = 'spconfig.py' + with open(setup_fname, 'w') as fout: + fout.write( +r"""#!%s +# + +import sys +import os +import subprocess + +def cmdlist(str): + return list(x.strip().replace("'",'') for x in str.split('\n') if x) +env = dict(os.environ) +""" % sys.executable) + + env_vars = sorted(list(env.keys())) + for name in env_vars: + val = env[name] + if string.find(name, 'PATH') < 0: + fout.write('env[%s] = %s\n' % (repr(name), repr(val))) + else: + if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': + sep = ';' + else: + sep = ':' + + fout.write( + 'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep)) + for part in string.split(val, sep): + fout.write(' %s\n' % part) + fout.write('"""))\n') + + fout.write( + "env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") + fout.write('\ncmd = cmdlist("""\n') + fout.write('%s\n' % cmd[0]) + for arg in cmd[1:]: + fout.write(' %s\n' % arg) + fout.write('""") + sys.argv[1:]\n') + fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + set_executable(setup_fname) + + def setup(self, args): if not args.spec: tty.die("spack setup requires a package spec argument.") @@ -70,7 +142,8 @@ def setup(self, args): return if not spec.versions.concrete: - tty.die("spack setup spec must have a single, concrete version. Did you forget a package version number?") + tty.die( + "spack setup spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) @@ -83,9 +156,8 @@ def setup(self, args): # TODO: make this an argument, not a global. spack.do_checksum = False - package.do_install( - keep_prefix=True, # Don't remove install directory, even if you think you should - ignore_deps=args.ignore_deps, - verbose=args.verbose, - keep_stage=True, # don't remove source dir for SETUP. - install_phases = set(['setup', 'provenance'])) + if not isinstance(package, spack.CMakePackage): + raise RuntimeError( + 'Support for {0} not yet implemented'.format(type(package))) + + write_spconfig(package) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 68360ec532..ea798e8eaa 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1822,116 +1822,6 @@ def _hms(seconds): parts.append("%.2fs" % s) return ' '.join(parts) -# FIXME : remove this after checking that set_executable works the same way -# stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python -#def make_executable(path): -# mode = os.stat(path).st_mode -# mode |= (mode & 0o444) >> 2 # copy R bits to X -# os.chmod(path, mode) - - -# class CMakePackage(PackageBase): -# phases = ['configure', 'build', 'install', 'provenance'] -# -# def make_make(self): -# import multiprocessing -# # number of jobs spack will to build with. -# jobs = multiprocessing.cpu_count() -# if not self.parallel: -# jobs = 1 -# elif self.make_jobs: -# jobs = self.make_jobs -# -# make = spack.build_environment.MakeExecutable('make', jobs) -# return make -# -# def configure_args(self): -# """Returns package-specific arguments to be provided to the configure command.""" -# return list() -# -# def configure_env(self): -# """Returns package-specific environment under which the configure command should be run.""" -# # FIXME : Why not EnvironmentModules and the hooks that PackageBase already provides ? -# return dict() -# -# def spack_transitive_include_path(self): -# return ';'.join( -# os.path.join(dep, 'include') -# for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) -# ) -# -# def setup(self, spec, prefix): -# cmd = [str(which('cmake'))] + \ -# spack.build_environment.get_std_cmake_args(self) + \ -# ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], -# '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], -# '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], -# '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] + \ -# self.configure_args() -# -# env = dict() -# env['PATH'] = os.environ['PATH'] -# env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() -# env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] -# -# setup_fname = 'spconfig.py' -# with open(setup_fname, 'w') as fout: -# fout.write(\ -# r"""#!%s -# # -# -# import sys -# import os -# import subprocess -# -# def cmdlist(str): -# return list(x.strip().replace("'",'') for x in str.split('\n') if x) -# env = dict(os.environ) -# """ % sys.executable) -# -# env_vars = sorted(list(env.keys())) -# for name in env_vars: -# val = env[name] -# if string.find(name, 'PATH') < 0: -# fout.write('env[%s] = %s\n' % (repr(name),repr(val))) -# else: -# if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': -# sep = ';' -# else: -# sep = ':' -# -# fout.write('env[%s] = "%s".join(cmdlist("""\n' % (repr(name),sep)) -# for part in string.split(val, sep): -# fout.write(' %s\n' % part) -# fout.write('"""))\n') -# -# fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") -# fout.write('\ncmd = cmdlist("""\n') -# fout.write('%s\n' % cmd[0]) -# for arg in cmd[1:]: -# fout.write(' %s\n' % arg) -# fout.write('""") + sys.argv[1:]\n') -# fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') -# set_executable(setup_fname) -# -# def configure(self, spec, prefix): -# cmake = which('cmake') -# with working_dir(self.build_directory, create=True): -# os.environ.update(self.configure_env()) -# os.environ['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() -# options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) -# cmake(self.source_directory, *options) -# -# def build(self, spec, prefix): -# make = self.make_make() -# with working_dir(self.build_directory, create=False): -# make() -# -# def install(self, spec, prefix): -# make = self.make_make() -# with working_dir(self.build_directory, create=False): -# make('install') - class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" -- cgit v1.2.3-70-g09d2 From f5433477b9f7c98ca00947cf1b1fdc106cca1080 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Aug 2016 09:08:00 +0200 Subject: qa : flake8 issues --- lib/spack/llnl/util/tty/log.py | 17 +++--- lib/spack/spack/build_environment.py | 13 ++--- lib/spack/spack/cmd/setup.py | 7 +-- lib/spack/spack/package.py | 68 ++++++++++++++++-------- var/spack/repos/builtin/packages/hdf5/package.py | 20 +++---- 5 files changed, 76 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 25ed5254a5..a4ba2a9bdf 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -24,12 +24,11 @@ ############################################################################## """Utility classes for logging the output of blocks of code. """ -import sys +import multiprocessing import os import re import select -import inspect -import multiprocessing +import sys import llnl.util.tty as tty import llnl.util.tty.color as color @@ -117,9 +116,10 @@ class log_output(object): # do things ... output will be logged # and also printed to stdout. - Opens a stream in 'w' mode at daemon spawning and closes it at daemon joining. - If echo is True, also prints the output to stdout. + Opens a stream in 'w' mode at daemon spawning and closes it at + daemon joining. If echo is True, also prints the output to stdout. """ + def __init__(self, filename, echo=False, force_color=False, debug=False): self.filename = filename # Various output options @@ -133,7 +133,11 @@ class log_output(object): self.read, self.write = os.pipe() # Sets a daemon that writes to file what it reads from a pipe - self.p = multiprocessing.Process(target=self._spawn_writing_daemon, args=(self.read,), name='logger_daemon') + self.p = multiprocessing.Process( + target=self._spawn_writing_daemon, + args=(self.read,), + name='logger_daemon' + ) self.p.daemon = True # Needed to un-summon the daemon self.parent_pipe, self.child_pipe = multiprocessing.Pipe() @@ -186,6 +190,7 @@ class log_output(object): os.close(self.read) class OutputRedirection(object): + def __init__(self, other): self.__dict__.update(other.__dict__) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index f9d795845e..03b044567f 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -51,16 +51,14 @@ There are two parts to the build environment: Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ +import multiprocessing import os -import sys import shutil -import multiprocessing -import platform +import sys import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.environment import EnvironmentModifications, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -502,7 +500,10 @@ def fork(pkg, function, dirty=False): child_connection.close() parent_connection, child_connection = multiprocessing.Pipe() - p = multiprocessing.Process(target=child_execution, args=(child_connection,)) + p = multiprocessing.Process( + target=child_execution, + args=(child_connection,) + ) p.start() exc_type, exception, traceback = parent_connection.recv() p.join() diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index 9553942017..652c08354f 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -58,8 +58,6 @@ def spack_transitive_include_path(): def write_spconfig(package): - spec = package.spec - prefix = spec.prefix # Set-up the environment spack.build_environment.setup_package(package) @@ -79,7 +77,7 @@ def write_spconfig(package): setup_fname = 'spconfig.py' with open(setup_fname, 'w') as fout: fout.write( -r"""#!%s + r"""#!%s # import sys @@ -108,8 +106,7 @@ env = dict(os.environ) fout.write(' %s\n' % part) fout.write('"""))\n') - fout.write( - "env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") + fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") # NOQA: ignore=E501 fout.write('\ncmd = cmdlist("""\n') fout.write('%s\n' % cmd[0]) for arg in cmd[1:]: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0d9067a955..3e25dddb0e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -75,12 +75,13 @@ _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] class InstallPhase(object): """Manages a single phase of the installation - This descriptor stores at creation time the name of the method it should search - for execution. The method is retrieved at __get__ time, so that it can be overridden - by subclasses of whatever class declared the phases. + This descriptor stores at creation time the name of the method it should + search for execution. The method is retrieved at __get__ time, so that + it can be overridden by subclasses of whatever class declared the phases. It also provides hooks to execute prerequisite and sanity checks. """ + def __init__(self, name): self.name = name self.preconditions = [] @@ -94,6 +95,7 @@ class InstallPhase(object): # If instance is there the caller wants to execute the # install phase, thus return a properly set wrapper phase = getattr(instance, self.name) + @functools.wraps(phase) def phase_wrapper(spec, prefix): # Check instance attributes at the beginning of a phase @@ -101,7 +103,8 @@ class InstallPhase(object): # Execute phase pre-conditions, # and give them the chance to fail for check in self.preconditions: - check(instance) # Do something sensible at some point + # Do something sensible at some point + check(instance) phase(spec, prefix) # Execute phase sanity_checks, # and give them the chance to fail @@ -147,8 +150,8 @@ class PackageMeta(type): # Check if phases is in attr dict, then set # install phases wrappers if 'phases' in attr_dict: - _InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] - for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): + _InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] # NOQA: ignore=E501 + for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): # NOQA: ignore=E501 attr_dict[phase_name] = InstallPhase(callback_name) attr_dict['_InstallPhase_phases'] = _InstallPhase_phases @@ -160,15 +163,22 @@ class PackageMeta(type): for phase_name, funcs in checks.items(): try: # Search for the phase in the attribute dictionary - phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] + phase = attr_dict[ + PackageMeta.phase_fmt.format(phase_name)] except KeyError: # If it is not there it's in the bases # and we added a check. We need to copy # and extend for base in bases: - phase = getattr(base, PackageMeta.phase_fmt.format(phase_name), None) - attr_dict[PackageMeta.phase_fmt.format(phase_name)] = phase.copy() - phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] + phase = getattr( + base, + PackageMeta.phase_fmt.format(phase_name), + None + ) + attr_dict[PackageMeta.phase_fmt.format( + phase_name)] = phase.copy() + phase = attr_dict[ + PackageMeta.phase_fmt.format(phase_name)] getattr(phase, check_name).extend(funcs) # Clear the attribute for the next class setattr(meta, attr_name, {}) @@ -190,8 +200,9 @@ class PackageMeta(type): def _execute_under_condition(func): @functools.wraps(func) def _wrapper(instance): - # If all the attributes have the value we require, then execute - if all([getattr(instance, key, None) == value for key, value in attrs.items()]): + # If all the attributes have the value we require, then + # execute + if all([getattr(instance, key, None) == value for key, value in attrs.items()]): # NOQA: ignore=E501 func(instance) return _wrapper return _execute_under_condition @@ -1081,7 +1092,8 @@ class PackageBase(object): else: self.do_stage() - tty.msg("Building {0} [{1}]".format(self.name, type(self).__base__ )) + tty.msg("Building {0} [{1}]".format( + self.name, type(self).__base__)) self.stage.keep = keep_stage self.build_directory = join_path(self.stage.path, 'spack-build') @@ -1106,13 +1118,22 @@ class PackageBase(object): self.log_path = log_path self.env_path = env_path dump_environment(env_path) - # Spawn a daemon that reads from a pipe and redirects everything to log_path - with log_output(log_path, verbose, sys.stdout.isatty(), True) as log_redirection: - for phase_name, phase in zip(self.phases, self._InstallPhase_phases): - tty.msg('Executing phase : \'{0}\''.format(phase_name)) + # Spawn a daemon that reads from a pipe and redirects + # everything to log_path + redirection_context = log_output( + log_path, verbose, + sys.stdout.isatty(), + True + ) + with redirection_context as log_redirection: + for phase_name, phase in zip(self.phases, self._InstallPhase_phases): # NOQA: ignore=E501 + tty.msg( + 'Executing phase : \'{0}\''.format(phase_name) # NOQA: ignore=E501 + ) # Redirect stdout and stderr to daemon pipe with log_redirection: - getattr(self, phase)(self.spec, self.prefix) + getattr(self, phase)( + self.spec, self.prefix) self.log() # Run post install hooks before build stage is removed. spack.hooks.post_install(self) @@ -1174,7 +1195,7 @@ class PackageBase(object): """ self.last_phase = kwargs.pop('stop_at', None) if self.last_phase is not None and self.last_phase not in self.phases: - tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) + tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # NOQA: ignore=E501 def log(self): # Copy provenance into the install directory on success @@ -1188,7 +1209,8 @@ class PackageBase(object): # Remove first if we're overwriting another build # (can happen with spack setup) try: - shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this + # log_install_path and env_install_path are inside this + shutil.rmtree(packages_dir) except Exception: # FIXME : this potentially catches too many things... pass @@ -1609,7 +1631,8 @@ class AutotoolsPackage(PackageBase): @PackageBase.sanity_check('autoreconf') def is_configure_or_die(self): if not os.path.exists('configure'): - raise RuntimeError('configure script not found in {0}'.format(os.getcwd())) + raise RuntimeError( + 'configure script not found in {0}'.format(os.getcwd())) def configure_args(self): return list() @@ -1668,7 +1691,8 @@ class CMakePackage(PackageBase): return list() def cmake(self, spec, prefix): - options = [self.root_cmakelists_dir()] + self.std_cmake_args + self.cmake_args() + options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ + self.cmake_args() create = not os.path.exists(self.wdir()) with working_dir(self.wdir(), create=create): inspect.getmodule(self).cmake(*options) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index a7c4b2d85e..601e3278e5 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -149,7 +149,7 @@ class Hdf5(AutotoolsPackage): def check_install(self): "Build and run a small program to test the installed HDF5 library" spec = self.spec - print "Checking HDF5 installation..." + print("Checking HDF5 installation...") checkdir = "spack-check" with working_dir(checkdir, create=True): source = r""" @@ -186,15 +186,15 @@ HDF5 version {version} {version} output = "" success = output == expected if not success: - print "Produced output does not match expected output." - print "Expected output:" - print '-' * 80 - print expected - print '-' * 80 - print "Produced output:" - print '-' * 80 - print output - print '-' * 80 + print("Produced output does not match expected output.") + print("Expected output:") + print('-' * 80) + print(expected) + print('-' * 80) + print("Produced output:") + print('-' * 80) + print(output) + print('-' * 80) raise RuntimeError("HDF5 install check failed") shutil.rmtree(checkdir) -- cgit v1.2.3-70-g09d2 From dd56784d2861d866cc3b6e15a145331297d283f0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 11 Oct 2016 16:44:18 +0200 Subject: qa : flake8 issues --- lib/spack/spack/package.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a9b5b2069a..7a45988808 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -59,8 +59,6 @@ import spack.repository import spack.url import spack.util.web from llnl.util.filesystem import * -from llnl.util.filesystem import * -from llnl.util.lang import * from llnl.util.lang import * from llnl.util.link_tree import LinkTree from llnl.util.tty.log import log_output -- cgit v1.2.3-70-g09d2 From 5ce3071143f3d5542e136f723175e64110a717eb Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 21 Oct 2016 12:51:38 +0200 Subject: do_install : removed install_self from the list of arguments (leftover after rebasing #1956) --- lib/spack/spack/package.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3d1f87d599..4872b24984 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1094,7 +1094,6 @@ class PackageBase(object): keep_prefix=False, keep_stage=False, install_deps=True, - install_self=True, skip_patch=False, verbose=False, make_jobs=None, -- cgit v1.2.3-70-g09d2 From 2251428f0a92a33567edd68a04c03ebc356d04c0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 21 Oct 2016 13:01:55 +0200 Subject: CMakePackage : changed `list()` to [] --- lib/spack/spack/package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4872b24984..a0cc8e68c8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1088,8 +1088,6 @@ class PackageBase(object): finally: self.prefix_lock.release_write() - install_phases = set(['configure', 'build', 'install', 'provenance']) - def do_install(self, keep_prefix=False, keep_stage=False, @@ -1171,7 +1169,6 @@ class PackageBase(object): # Set parallelism before starting build. self.make_jobs = make_jobs - # ------------------- BEGIN def build_process() # Then install the package itself. def build_process(): """Forked for each build. Has its own process and python @@ -1778,7 +1775,7 @@ class CMakePackage(PackageBase): return join_path(self.stage.source_path, 'spack-build') def cmake_args(self): - return list() + return [] def cmake(self, spec, prefix): options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ -- cgit v1.2.3-70-g09d2 From 04821c7be88a8a8333f5aa4df310f30e92c741a9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 21 Oct 2016 22:12:21 +0200 Subject: spack create : now creates packages that are derived from AutotoolsPackage and CMakePackage --- lib/spack/spack/cmd/create.py | 243 ++++++++++++++++++++++++++---------------- lib/spack/spack/package.py | 8 +- 2 files changed, 155 insertions(+), 96 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 741a320ea7..7f8cf1aa5c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -22,25 +22,24 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import string +from __future__ import print_function + import os import re +import string -from ordereddict_backport import OrderedDict import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp - import spack import spack.cmd import spack.cmd.checksum import spack.url import spack.util.web -from spack.spec import Spec -from spack.util.naming import * +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import Repo, RepoError - +from spack.spec import Spec from spack.util.executable import which - +from spack.util.naming import * description = "Create a new package file from an archive URL" @@ -87,7 +86,7 @@ package_template = string.Template("""\ from spack import * -class ${class_name}(Package): +class ${class_name}(${base_class_name}): ""\"FIXME: Put a proper description of your package here.""\" # FIXME: Add a proper url for your package's homepage here. @@ -98,109 +97,158 @@ ${versions} ${dependencies} - def install(self, spec, prefix): -${install} +${body} """) -# Build dependencies and extensions -dependencies_dict = { - 'autotools': """\ - # FIXME: Add dependencies if required. - # depends_on('foo')""", - 'cmake': """\ - # FIXME: Add additional dependencies if required. - depends_on('cmake', type='build')""", +class DefaultGuess(object): - 'scons': """\ - # FIXME: Add additional dependencies if required. - depends_on('scons', type='build')""", + base_class_name = 'Package' - 'bazel': """\ - # FIXME: Add additional dependencies if required. - depends_on('bazel', type='build')""", + dependencies = """\ + # FIXME: Add dependencies if required. + # depends_on('foo')""" - 'python': """\ - extends('python') + body = """\ + def install(self, spec, prefix): + # FIXME: Unknown build system + make() + make('install')""" - # FIXME: Add additional dependencies if required. - # depends_on('py-setuptools', type='build') - # depends_on('py-foo', type=nolink)""", + def __init__(self, name, url, version_hash_tuples): + self.name = name + self.class_name = mod_to_class(name) + self.url = url + self.version_hash_tuples = version_hash_tuples - 'R': """\ - extends('R') + @property + def versions(self): + """Adds a version() call to the package for each version found.""" + max_len = max(len(str(v)) for v, h in self.version_hash_tuples) + format = " version(%%-%ds, '%%s')" % (max_len + 2) + return '\n'.join(format % ("'%s'" % v, h) for v, h in self.version_hash_tuples) - # FIXME: Add additional dependencies if required. - # depends_on('r-foo', type=nolink)""", - 'octave': """\ - extends('octave') +class AutotoolsGuess(DefaultGuess): - # FIXME: Add additional dependencies if required. - # depends_on('octave-foo', type=nolink)""", + base_class_name = 'AutotoolsPackage' - 'unknown': """\ + dependencies = """\ # FIXME: Add dependencies if required. + # depends_on('m4', type='build') + # depends_on('autoconf', type='build') + # depends_on('automake', type='build') + # depends_on('libtool', type='build') # depends_on('foo')""" -} -# Default installation instructions -install_dict = { - 'autotools': """\ - # FIXME: Modify the configure line to suit your build system here. - configure('--prefix={0}'.format(prefix)) + body = """\ + def configure_args(self): + # FIXME : Add arguments other than --prefix + # FIXME : If not needed delete the function + args = [] + return args""" - # FIXME: Add logic to build and install here. - make() - make('install')""", - 'cmake': """\ - with working_dir('spack-build', create=True): - # FIXME: Modify the cmake line to suit your build system here. - cmake('..', *std_cmake_args) +class CMakeGuess(DefaultGuess): + + base_class_name = 'CMakePackage' + + dependencies = """\ + # FIXME: Add additional dependencies if required. + depends_on('cmake', type='build')""" + + body = """\ + def cmake_args(self): + # FIXME : Add arguments other than + # FIXME : CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE + # FIXME : If not needed delete the function + args = [] + return args""" + - # FIXME: Add logic to build and install here. - make() - make('install')""", +class SconsGuess(DefaultGuess): - 'scons': """\ + dependencies = """\ + # FIXME: Add additional dependencies if required. + depends_on('scons', type='build')""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. scons('prefix={0}'.format(prefix)) - scons('install')""", + scons('install')""" - 'bazel': """\ + +class BazelGuess(DefaultGuess): + + dependencies = """\ + # FIXME: Add additional dependencies if required. + depends_on('bazel', type='build')""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. - bazel()""", + bazel()""" + - 'python': """\ +class PythonGuess(DefaultGuess): + + dependencies = """\ + extends('python') + + # FIXME: Add additional dependencies if required. + # depends_on('py-setuptools', type='build') + # depends_on('py-foo', type=nolink)""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. - setup_py('install', '--prefix={0}'.format(prefix))""", + setup_py('install', '--prefix={0}'.format(prefix))""" + + def __init__(self, name, *args): + name = 'py-{0}'.format(name) + super(PythonGuess, self).__init__(name, *args) + + +class RGuess(DefaultGuess): + + dependencies = """\ + extends('R') + + # FIXME: Add additional dependencies if required. + # depends_on('r-foo', type=nolink)""" - 'R': """\ + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), - self.stage.source_path)""", + self.stage.source_path)""" + + def __init__(self, name, *args): + name = 'r-{0}'.format(name) + super(RGuess, self).__init__(name, *args) - 'octave': """\ + +class OctaveGuess(DefaultGuess): + + dependencies = """\ + extends('octave') + + # FIXME: Add additional dependencies if required. + # depends_on('octave-foo', type=nolink)""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. octave('--quiet', '--norc', '--built-in-docstrings-file=/dev/null', '--texi-macros-file=/dev/null', '--eval', 'pkg prefix {0}; pkg install {1}'.format( - prefix, self.stage.archive_file))""", + prefix, self.stage.archive_file))""" - 'unknown': """\ - # FIXME: Unknown build system - make() - make('install')""" -} - - -def make_version_calls(ver_hash_tuples): - """Adds a version() call to the package for each version found.""" - max_len = max(len(str(v)) for v, h in ver_hash_tuples) - format = " version(%%-%ds, '%%s')" % (max_len + 2) - return '\n'.join(format % ("'%s'" % v, h) for v, h in ver_hash_tuples) + def __init__(self, name, *args): + name = 'octave-{0}'.format(name) + super(OctaveGuess, self).__init__(name, *args) def setup_parser(subparser): @@ -227,6 +275,16 @@ def setup_parser(subparser): class BuildSystemGuesser(object): + _choiches = { + 'autotools': AutotoolsGuess, + 'cmake': CMakeGuess, + 'scons': SconsGuess, + 'bazel': BazelGuess, + 'python': PythonGuess, + 'R': RGuess, + 'octave': OctaveGuess + } + def __call__(self, stage, url): """Try to guess the type of build system used by a project based on the contents of its archive or the URL it was downloaded from.""" @@ -275,6 +333,10 @@ class BuildSystemGuesser(object): self.build_system = build_system + def make_guess(self, name, url, ver_hash_tuples): + cls = self._choiches.get(self.build_system, DefaultGuess) + return cls(name, url, ver_hash_tuples) + def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -348,7 +410,7 @@ def fetch_tarballs(url, name, version): tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( ["%-10s%s" % (v, u) for v, u in versions.iteritems()])) - print + print('') archives_to_fetch = tty.get_number( "Include how many checksums in the package file?", default=5, abort='q') @@ -389,16 +451,10 @@ def create(parser, args): if not ver_hash_tuples: tty.die("Could not fetch any tarballs for %s" % name) - # Add prefix to package name if it is an extension. - if guesser.build_system == 'python': - name = 'py-{0}'.format(name) - if guesser.build_system == 'R': - name = 'r-{0}'.format(name) - if guesser.build_system == 'octave': - name = 'octave-{0}'.format(name) + guess = guesser.make_guess(name, url, ver_hash_tuples) # Create a directory for the new package. - pkg_path = repo.filename_for_package_name(name) + pkg_path = repo.filename_for_package_name(guess.name) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: @@ -408,12 +464,15 @@ def create(parser, args): with open(pkg_path, "w") as pkg_file: pkg_file.write( package_template.substitute( - name=name, - class_name=mod_to_class(name), - url=url, - versions=make_version_calls(ver_hash_tuples), - dependencies=dependencies_dict[guesser.build_system], - install=install_dict[guesser.build_system])) + name=guess.name, + class_name=guess.class_name, + base_class_name=guess.base_class_name, + url=guess.url, + versions=guess.versions, + dependencies=guess.dependencies, + body=guess.body + ) + ) # If everything checks out, go ahead and edit. spack.editor(pkg_path) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a0cc8e68c8..91e6b74dbd 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1686,10 +1686,10 @@ class EditableMakefile(PackageBase): return self.stage.source_path def build_args(self): - return list() + return [] def install_args(self): - return list() + return [] def edit(self, spec, prefix): raise NotImplementedError('\'edit\' function not implemented') @@ -1721,7 +1721,7 @@ class AutotoolsPackage(PackageBase): 'configure script not found in {0}'.format(os.getcwd())) def configure_args(self): - return list() + return [] def configure(self, spec, prefix): options = ['--prefix={0}'.format(prefix)] + self.configure_args() @@ -1761,7 +1761,7 @@ class CMakePackage(PackageBase): args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type), - '-DCMAKE_VERBOSE_MAKEFILE=ON'] + '-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON'] if platform.mac_ver()[0]: args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') -- cgit v1.2.3-70-g09d2 From fd2b72fd0fb0ea1ac86c6f0eef9e0d132fb95f06 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 21 Oct 2016 22:15:29 +0200 Subject: qa : flake8 issues --- lib/spack/spack/cmd/create.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7f8cf1aa5c..9ec9cfe98c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -126,7 +126,9 @@ class DefaultGuess(object): """Adds a version() call to the package for each version found.""" max_len = max(len(str(v)) for v, h in self.version_hash_tuples) format = " version(%%-%ds, '%%s')" % (max_len + 2) - return '\n'.join(format % ("'%s'" % v, h) for v, h in self.version_hash_tuples) + return '\n'.join( + format % ("'%s'" % v, h) for v, h in self.version_hash_tuples + ) class AutotoolsGuess(DefaultGuess): -- cgit v1.2.3-70-g09d2 From 715e029361207f28b2834d4f1c1b38de5802405c Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 21 Oct 2016 22:44:41 +0200 Subject: spack info : added phases --- lib/spack/spack/cmd/info.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 2fa3a07525..0171d28fde 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -84,6 +84,13 @@ def print_text_info(pkg): print " " + fmt % (name, default, desc) + print + print "Installation Phases:" + phase_str = '' + for phase in pkg.phases: + phase_str += " {0}".format(phase) + print phase_str + for deptype in ('build', 'link', 'run'): print print "%s Dependencies:" % deptype.capitalize() @@ -94,7 +101,7 @@ def print_text_info(pkg): print " None" print - print "Virtual packages: " + print "Virtual Packages: " if pkg.provided: for spec, when in pkg.provided.items(): print " %s provides %s" % (when, spec) -- cgit v1.2.3-70-g09d2 From e8dafd1090f1322673a5017f19bfce2a6cf071ed Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 21 Oct 2016 23:54:11 +0200 Subject: Removed space before colon in `FIXME:`, added one line description of guess classes --- lib/spack/spack/cmd/create.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 9ec9cfe98c..225285e077 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -102,7 +102,7 @@ ${body} class DefaultGuess(object): - + """Provides the default values to be used for the package file template""" base_class_name = 'Package' dependencies = """\ @@ -132,7 +132,7 @@ class DefaultGuess(object): class AutotoolsGuess(DefaultGuess): - + """Provides appropriate overrides for autotools-based packages""" base_class_name = 'AutotoolsPackage' dependencies = """\ @@ -145,14 +145,14 @@ class AutotoolsGuess(DefaultGuess): body = """\ def configure_args(self): - # FIXME : Add arguments other than --prefix - # FIXME : If not needed delete the function + # FIXME: Add arguments other than --prefix + # FIXME: If not needed delete the function args = [] return args""" class CMakeGuess(DefaultGuess): - + """Provides appropriate overrides for cmake-based packages""" base_class_name = 'CMakePackage' dependencies = """\ @@ -161,15 +161,15 @@ class CMakeGuess(DefaultGuess): body = """\ def cmake_args(self): - # FIXME : Add arguments other than - # FIXME : CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE - # FIXME : If not needed delete the function + # FIXME: Add arguments other than + # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE + # FIXME: If not needed delete the function args = [] return args""" class SconsGuess(DefaultGuess): - + """Provides appropriate overrides for scons-based packages""" dependencies = """\ # FIXME: Add additional dependencies if required. depends_on('scons', type='build')""" @@ -182,7 +182,7 @@ class SconsGuess(DefaultGuess): class BazelGuess(DefaultGuess): - + """Provides appropriate overrides for bazel-based packages""" dependencies = """\ # FIXME: Add additional dependencies if required. depends_on('bazel', type='build')""" @@ -194,7 +194,7 @@ class BazelGuess(DefaultGuess): class PythonGuess(DefaultGuess): - + """Provides appropriate overrides for python extensions""" dependencies = """\ extends('python') @@ -213,7 +213,7 @@ class PythonGuess(DefaultGuess): class RGuess(DefaultGuess): - + """Provides appropriate overrides for R extensions""" dependencies = """\ extends('R') @@ -232,7 +232,7 @@ class RGuess(DefaultGuess): class OctaveGuess(DefaultGuess): - + """Provides appropriate overrides for octave packages""" dependencies = """\ extends('octave') -- cgit v1.2.3-70-g09d2 From 012da99644e54dd80a328fe3b44aa10700d8e83e Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 22 Oct 2016 00:10:37 +0200 Subject: spack create : fixed typo --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 225285e077..5db0601d44 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -277,7 +277,7 @@ def setup_parser(subparser): class BuildSystemGuesser(object): - _choiches = { + _choices = { 'autotools': AutotoolsGuess, 'cmake': CMakeGuess, 'scons': SconsGuess, -- cgit v1.2.3-70-g09d2 From 482f60d1d37d8d18ff62318c23a72808ceacd99e Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 22 Oct 2016 01:03:07 +0200 Subject: packages : moved decorators into AutotoolsPackage and CMakePackage --- lib/spack/spack/package.py | 20 ++++++++++++++++++++ var/spack/repos/builtin/packages/autoconf/package.py | 11 ++--------- var/spack/repos/builtin/packages/hdf5/package.py | 4 +--- var/spack/repos/builtin/packages/lzo/package.py | 2 -- var/spack/repos/builtin/packages/qhull/package.py | 2 -- 5 files changed, 23 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 91e6b74dbd..75a708de9c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1733,6 +1733,16 @@ class AutotoolsPackage(PackageBase): def install(self, spec, prefix): inspect.getmodule(self).make('install') + @PackageBase.sanity_check('build') + @PackageBase.on_package_attributes(run_tests=True) + def _run_default_function(self): + try: + fn = getattr(self, 'check') + tty.msg('Trying default sanity checks [check]') + fn() + except AttributeError: + tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + # This will be used as a registration decorator in user # packages, if need be PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) @@ -1792,6 +1802,16 @@ class CMakePackage(PackageBase): with working_dir(self.wdir()): inspect.getmodule(self).make('install') + @PackageBase.sanity_check('build') + @PackageBase.on_package_attributes(run_tests=True) + def _run_default_function(self): + try: + fn = getattr(self, 'check') + tty.msg('Trying default sanity checks [check]') + fn() + except AttributeError: + tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/var/spack/repos/builtin/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py index 5eb7f3347b..b6aba8c03f 100644 --- a/var/spack/repos/builtin/packages/autoconf/package.py +++ b/var/spack/repos/builtin/packages/autoconf/package.py @@ -25,10 +25,8 @@ from spack import * -class Autoconf(Package): - """ - Autoconf -- system configuration part of autotools - """ +class Autoconf(AutotoolsPackage): + """Autoconf -- system configuration part of autotools""" homepage = 'https://www.gnu.org/software/autoconf/' url = 'http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz' @@ -54,8 +52,3 @@ class Autoconf(Package): 'ifnames'] for name in executables: setattr(module, name, self._make_executable(name)) - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 6b18aa4ab8..c92ed284bb 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -143,9 +143,7 @@ class Hdf5(AutotoolsPackage): return ["--with-zlib=%s" % spec['zlib'].prefix] + extra_args - @AutotoolsPackage.sanity_check('install') - @AutotoolsPackage.on_package_attributes(run_tests=True) - def check_install(self): + def check(self): "Build and run a small program to test the installed HDF5 library" spec = self.spec print("Checking HDF5 installation...") diff --git a/var/spack/repos/builtin/packages/lzo/package.py b/var/spack/repos/builtin/packages/lzo/package.py index dc8f316a72..05229b6a62 100644 --- a/var/spack/repos/builtin/packages/lzo/package.py +++ b/var/spack/repos/builtin/packages/lzo/package.py @@ -43,8 +43,6 @@ class Lzo(AutotoolsPackage): '--enable-shared' ] - @AutotoolsPackage.sanity_check('build') - @AutotoolsPackage.on_package_attributes(run_tests=True) def check(self): make('check') make('test') diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index e221bf1552..3816b377eb 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -45,7 +45,5 @@ class Qhull(CMakePackage): depends_on('cmake@2.6:', type='build') - @CMakePackage.sanity_check('build') - @CMakePackage.on_package_attributes(run_tests=True) def check(self): make('test') -- cgit v1.2.3-70-g09d2 From c84123dce5a576c0f677eb5918858591851b3921 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 22 Oct 2016 10:39:33 +0200 Subject: spack info : shows the build-system class used --- lib/spack/spack/cmd/info.py | 7 +++++-- lib/spack/spack/package.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 0171d28fde..5366ad4aa8 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -48,8 +48,11 @@ def setup_parser(subparser): def print_text_info(pkg): """Print out a plain text description of a package.""" - print "Package: ", pkg.name - print "Homepage: ", pkg.homepage + header = "{0}: ".format(pkg.build_system_class) + + print header, pkg.name + whitespaces = ''.join([' '] * (len(header) - len("Homepage: "))) + print "Homepage:", whitespaces, pkg.homepage print print "Safe versions: " diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 75a708de9c..bc39c6400d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1674,6 +1674,9 @@ class PackageBase(object): class Package(PackageBase): phases = ['install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'Package' # This will be used as a registration decorator in user # packages, if need be PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) @@ -1681,6 +1684,9 @@ class Package(PackageBase): class EditableMakefile(PackageBase): phases = ['edit', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'EditableMakefile' def wdir(self): return self.stage.source_path @@ -1709,6 +1715,9 @@ class EditableMakefile(PackageBase): class AutotoolsPackage(PackageBase): phases = ['autoreconf', 'configure', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'AutotoolsPackage' def autoreconf(self, spec, prefix): """Not needed usually, configure should be already there""" @@ -1750,6 +1759,9 @@ class AutotoolsPackage(PackageBase): class CMakePackage(PackageBase): phases = ['cmake', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'CMakePackage' def build_type(self): return 'RelWithDebInfo' -- cgit v1.2.3-70-g09d2 From 8091a3d6cb10de640a7581142c59bcdeefd07ee6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 22 Oct 2016 10:48:16 +0200 Subject: do_install : use build_system_class attribute instead of `type(self).__base__` --- lib/spack/spack/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index bc39c6400d..a883f8646e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1181,8 +1181,9 @@ class PackageBase(object): else: self.do_stage() - tty.msg("Building {0} [{1}]".format( - self.name, type(self).__base__)) + tty.msg( + 'Building {0} [{1}]'.format(self.name, self.build_system_class) + ) self.stage.keep = keep_stage self.build_directory = join_path(self.stage.path, 'spack-build') -- cgit v1.2.3-70-g09d2 From 484aaf50cce64ed11b6e7c7ec74cd82c5fac7efb Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 22 Oct 2016 14:54:26 +0200 Subject: CMakePackage : changed method name from `wdir` to `build_directory` --- lib/spack/spack/package.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a883f8646e..7eb5446a30 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1186,8 +1186,6 @@ class PackageBase(object): ) self.stage.keep = keep_stage - self.build_directory = join_path(self.stage.path, 'spack-build') - self.source_directory = self.stage.source_path try: with contextlib.nested(self.stage, self._prefix_write_lock()): @@ -1768,7 +1766,7 @@ class CMakePackage(PackageBase): return 'RelWithDebInfo' def root_cmakelists_dir(self): - return self.source_directory + return self.stage.source_path @property def std_cmake_args(self): @@ -1794,7 +1792,7 @@ class CMakePackage(PackageBase): args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) return args - def wdir(self): + def build_directory(self): return join_path(self.stage.source_path, 'spack-build') def cmake_args(self): @@ -1803,16 +1801,16 @@ class CMakePackage(PackageBase): def cmake(self, spec, prefix): options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ self.cmake_args() - create = not os.path.exists(self.wdir()) - with working_dir(self.wdir(), create=create): + create = not os.path.exists(self.build_directory()) + with working_dir(self.build_directory(), create=create): inspect.getmodule(self).cmake(*options) def build(self, spec, prefix): - with working_dir(self.wdir()): + with working_dir(self.build_directory()): inspect.getmodule(self).make() def install(self, spec, prefix): - with working_dir(self.wdir()): + with working_dir(self.build_directory()): inspect.getmodule(self).make('install') @PackageBase.sanity_check('build') -- cgit v1.2.3-70-g09d2 From bdf48322696290d4e3d00ed12b7c7fe6ca213478 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 22 Oct 2016 17:08:52 +0200 Subject: spack build, spack configure : added commands --- lib/spack/spack/cmd/build.py | 43 +++++++++++++++++++ lib/spack/spack/cmd/configure.py | 90 ++++++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/install.py | 6 +-- lib/spack/spack/package.py | 10 ++--- 4 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 lib/spack/spack/cmd/build.py create mode 100644 lib/spack/spack/cmd/configure.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/build.py b/lib/spack/spack/cmd/build.py new file mode 100644 index 0000000000..1c43acc2b3 --- /dev/null +++ b/lib/spack/spack/cmd/build.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack.cmd.configure as cfg + +from spack import * + +description = 'Stops at build stage when installing a package, if possible' + +build_system_to_phase = { + CMakePackage: 'build', + AutotoolsPackage: 'build' +} + + +def setup_parser(subparser): + cfg.setup_parser(subparser) + + +def build(parser, args): + cfg._stop_at_phase_during_install(args, build, build_system_to_phase) diff --git a/lib/spack/spack/cmd/configure.py b/lib/spack/spack/cmd/configure.py new file mode 100644 index 0000000000..3eebe2584b --- /dev/null +++ b/lib/spack/spack/cmd/configure.py @@ -0,0 +1,90 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import argparse + +import llnl.util.tty as tty +import spack.cmd +import spack.cmd.install as inst + +from spack import * + +description = 'Stops at configuration stage when installing a package, if possible' # NOQA: ignore=E501 + + +build_system_to_phase = { + CMakePackage: 'cmake', + AutotoolsPackage: 'configure' +} + + +def setup_parser(subparser): + subparser.add_argument( + 'package', + nargs=argparse.REMAINDER, + help="spec of the package to install" + ) + subparser.add_argument( + '-v', '--verbose', + action='store_true', + help="Print additional output during builds" + ) + + +def _stop_at_phase_during_install(args, calling_fn, phase_mapping): + if not args.package: + tty.die("configure requires at least one package argument") + + # TODO: to be refactored with code in install + specs = spack.cmd.parse_specs(args.package, concretize=True) + if len(specs) != 1: + tty.error('only one spec can be installed at a time.') + spec = specs.pop() + pkg = spec.package + try: + key = [cls for cls in phase_mapping if isinstance(pkg, cls)].pop() + phase = phase_mapping[key] + # Install package dependencies if needed + parser = argparse.ArgumentParser() + inst.setup_parser(parser) + tty.msg('Checking dependencies for {0}'.format(args.package)) + cli_args = ['-v'] if args.verbose else [] + install_args = parser.parse_args(cli_args + ['--only=dependencies']) + install_args.package = args.package + inst.install(parser, install_args) + # Install package and stop at the given phase + cli_args = ['-v'] if args.verbose else [] + install_args = parser.parse_args(cli_args + ['--only=package']) + install_args.package = args.package + inst.install(parser, install_args, stop_at=phase) + except IndexError: + tty.error( + 'Package {0} has no {1} phase, or its {1} phase is not separated from install'.format( # NOQA: ignore=E501 + spec.name, calling_fn.__name__) + ) + + +def configure(parser, args): + _stop_at_phase_during_install(args, configure, build_system_to_phase) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 6ca75d7999..aab7c0abc7 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -74,7 +74,7 @@ the dependencies.""" help="Run tests during installation of a package.") -def install(parser, args): +def install(parser, args, **kwargs): if not args.package: tty.die("install requires at least one package argument") @@ -87,7 +87,7 @@ def install(parser, args): # Parse cli arguments and construct a dictionary # that will be passed to Package.do_install API - kwargs = { + kwargs.update({ 'keep_prefix': args.keep_prefix, 'keep_stage': args.keep_stage, 'install_deps': 'dependencies' in args.things_to_install, @@ -96,7 +96,7 @@ def install(parser, args): 'verbose': args.verbose, 'fake': args.fake, 'dirty': args.dirty - } + }) # Spec from cli specs = spack.cmd.parse_specs(args.package, concretize=True) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 7eb5446a30..99796104a5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1263,13 +1263,9 @@ class PackageBase(object): # A StopIteration exception means that do_install # was asked to stop early from clients tty.msg(e.message) - except Exception: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, - wrap=False) - raise + tty.msg( + 'Package stage directory : {0}'.format(self.stage.source_path) + ) finally: # Remove the install prefix if anything went wrong during install. if not keep_prefix: -- cgit v1.2.3-70-g09d2 From 284ed13fa658f90226f6b26bc4f0529722401992 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 23 Oct 2016 13:34:29 +0200 Subject: spack.error : fixed pickling and representation to permit to pass FetchErrors --- lib/spack/spack/error.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index c94875e91a..5e5c1b1c7e 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -26,6 +26,7 @@ import os import sys import llnl.util.tty as tty import spack +import inspect class SpackError(Exception): @@ -49,7 +50,7 @@ class SpackError(Exception): else: tty.error(self.message) if self.long_message: - print self.long_message + print(self.long_message) os._exit(1) def __str__(self): @@ -58,6 +59,16 @@ class SpackError(Exception): msg += "\n %s" % self._long_message return msg + def __repr__(self): + args = [repr(self.message), repr(self.long_message)] + args = ','.join(args) + qualified_name = inspect.getmodule( + self).__name__ + '.' + type(self).__name__ + return qualified_name + '(' + args + ')' + + def __reduce__(self): + return type(self), (self.message, self.long_message) + class UnsupportedPlatformError(SpackError): """Raised by packages when a platform is not supported""" -- cgit v1.2.3-70-g09d2 From fa3f07c0929ef2beedb3746226d3f826f613e9cc Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 23 Oct 2016 19:02:46 +0200 Subject: CMakePackage, AutotoolsPackage : added default behavior on check --- lib/spack/spack/package.py | 33 +++++++++++++++++++++-- var/spack/repos/builtin/packages/hdf5/package.py | 3 ++- var/spack/repos/builtin/packages/lzo/package.py | 4 --- var/spack/repos/builtin/packages/qhull/package.py | 3 --- 4 files changed, 33 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 99796104a5..9483f370dd 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1055,6 +1055,27 @@ class PackageBase(object): mkdirp(self.prefix.lib) mkdirp(self.prefix.man1) + def _if_make_target_execute(self, target): + try: + # Check if we have a makefile + file = [x for x in ('Makefile', 'makefile') if os.path.exists(x)] + file = file.pop() + except IndexError: + tty.msg('No Makefile found in the build directory') + return + + # Check if 'target' is in the makefile + regex = re.compile('^' + target + ':') + with open(file, 'r') as f: + matches = [line for line in f.readlines() if regex.match(line)] + + if not matches: + tty.msg('Target \'' + target + ':\' not found in Makefile') + return + + # Execute target + inspect.getmodule(self).make(target) + def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build @@ -1747,6 +1768,10 @@ class AutotoolsPackage(PackageBase): except AttributeError: tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + def check(self): + self._if_make_target_execute('test') + self._if_make_target_execute('check') + # This will be used as a registration decorator in user # packages, if need be PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) @@ -1814,10 +1839,14 @@ class CMakePackage(PackageBase): def _run_default_function(self): try: fn = getattr(self, 'check') - tty.msg('Trying default sanity checks [check]') + tty.msg('Trying default build sanity checks [check]') fn() except AttributeError: - tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + tty.msg('Skipping default build sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + + def check(self): + with working_dir(self.build_directory()): + self._if_make_target_execute('test') PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index c92ed284bb..cbb7501034 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -144,7 +144,8 @@ class Hdf5(AutotoolsPackage): return ["--with-zlib=%s" % spec['zlib'].prefix] + extra_args def check(self): - "Build and run a small program to test the installed HDF5 library" + super(Hdf5, self).check() + # Build and run a small program to test the installed HDF5 library spec = self.spec print("Checking HDF5 installation...") checkdir = "spack-check" diff --git a/var/spack/repos/builtin/packages/lzo/package.py b/var/spack/repos/builtin/packages/lzo/package.py index 05229b6a62..e9c98842f4 100644 --- a/var/spack/repos/builtin/packages/lzo/package.py +++ b/var/spack/repos/builtin/packages/lzo/package.py @@ -42,7 +42,3 @@ class Lzo(AutotoolsPackage): '--disable-dependency-tracking', '--enable-shared' ] - - def check(self): - make('check') - make('test') diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index 3816b377eb..4456c16bd2 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -44,6 +44,3 @@ class Qhull(CMakePackage): url="http://www.qhull.org/download/qhull-2012.1-src.tgz") depends_on('cmake@2.6:', type='build') - - def check(self): - make('test') -- cgit v1.2.3-70-g09d2 From e0f3188970f7b5896517b8e879d477d44f73c088 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 23 Oct 2016 19:10:07 +0200 Subject: spack setup : improved error message --- lib/spack/spack/cmd/setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index f39a827a8d..50bc031330 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -148,6 +148,12 @@ def setup(self, args): spec.concretize() package = spack.repo.get(spec) + if not isinstance(package, spack.CMakePackage): + tty.die( + 'Support for {0} derived packages not yet implemented'.format( + package.build_system_class + ) + ) # It's OK if the package is already installed. @@ -157,8 +163,4 @@ def setup(self, args): # TODO: make this an argument, not a global. spack.do_checksum = False - if not isinstance(package, spack.CMakePackage): - raise RuntimeError( - 'Support for {0} not yet implemented'.format(type(package))) - write_spconfig(package) -- cgit v1.2.3-70-g09d2 From 7bd735416dcf8235efa77bf21f7e92beacf7d433 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 23 Oct 2016 22:38:19 +0200 Subject: package.py : moved each specialized package to its own module file --- lib/spack/spack/__init__.py | 4 +- lib/spack/spack/build_systems/__init__.py | 0 lib/spack/spack/build_systems/autotools.py | 106 ++++++++++++++ lib/spack/spack/build_systems/cmake.py | 143 +++++++++++++++++++ lib/spack/spack/build_systems/editable_makefile.py | 77 ++++++++++ lib/spack/spack/package.py | 155 --------------------- var/spack/repos/builtin/packages/astyle/package.py | 4 +- 7 files changed, 331 insertions(+), 158 deletions(-) create mode 100644 lib/spack/spack/build_systems/__init__.py create mode 100644 lib/spack/spack/build_systems/autotools.py create mode 100644 lib/spack/spack/build_systems/cmake.py create mode 100644 lib/spack/spack/build_systems/editable_makefile.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c7d592befb..918e17323b 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -196,7 +196,9 @@ __all__ = ['Package', 'alldeps', 'nolink'] from spack.package import Package, ExtensionConflictError -from spack.package import CMakePackage, AutotoolsPackage, EditableMakefile +from spack.build_systems.editable_makefile import EditableMakefile +from spack.build_systems.autotools import AutotoolsPackage +from spack.build_systems.cmake import CMakePackage from spack.version import Version, ver from spack.spec import DependencySpec, alldeps, nolink from spack.multimethod import when diff --git a/lib/spack/spack/build_systems/__init__.py b/lib/spack/spack/build_systems/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py new file mode 100644 index 0000000000..0bb5576708 --- /dev/null +++ b/lib/spack/spack/build_systems/autotools.py @@ -0,0 +1,106 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os.path + +import llnl.util.tty as tty +from spack.package import PackageBase + + +class AutotoolsPackage(PackageBase): + """Specialized class for packages that are built using GNU Autotools + + This class provides four phases that can be overridden: + - autoreconf + - configure + - build + - install + + They all have sensible defaults and for many packages the only thing + necessary will be to override `configure_args` + """ + phases = ['autoreconf', 'configure', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'AutotoolsPackage' + + def autoreconf(self, spec, prefix): + """Not needed usually, configure should be already there""" + pass + + @PackageBase.sanity_check('autoreconf') + def is_configure_or_die(self): + """Checks the presence of a `configure` file after the + autoreconf phase""" + if not os.path.exists('configure'): + raise RuntimeError( + 'configure script not found in {0}'.format(os.getcwd())) + + def configure_args(self): + """Method to be overridden. Should return an iterable containing + all the arguments that must be passed to configure, except --prefix + """ + return [] + + def configure(self, spec, prefix): + """Runs configure with the arguments specified in `configure_args` + and an appropriately set prefix + """ + options = ['--prefix={0}'.format(prefix)] + self.configure_args() + inspect.getmodule(self).configure(*options) + + def build(self, spec, prefix): + """The usual `make` after configure""" + inspect.getmodule(self).make() + + def install(self, spec, prefix): + """...and the final `make install` after configure""" + inspect.getmodule(self).make('install') + + @PackageBase.sanity_check('build') + @PackageBase.on_package_attributes(run_tests=True) + def _run_default_function(self): + """This function is run after build if self.run_tests == True + + It will search for a method named `check` and run it. A sensible + default is provided in the base class. + """ + try: + fn = getattr(self, 'check') + tty.msg('Trying default sanity checks [check]') + fn() + except AttributeError: + tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + + def check(self): + """Default test : search the Makefile for targets `test` and `check` + and run them if found. + """ + self._if_make_target_execute('test') + self._if_make_target_execute('check') + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py new file mode 100644 index 0000000000..cb1076d7b7 --- /dev/null +++ b/lib/spack/spack/build_systems/cmake.py @@ -0,0 +1,143 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os +import platform + +import llnl.util.tty as tty +import spack.build_environment +from llnl.util.filesystem import working_dir, join_path +from spack.package import PackageBase + + +class CMakePackage(PackageBase): + """Specialized class for packages that are built using cmake + + This class provides three phases that can be overridden: + - cmake + - build + - install + + They all have sensible defaults and for many packages the only thing + necessary will be to override `cmake_args` + """ + phases = ['cmake', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'CMakePackage' + + def build_type(self): + """Override to provide the correct build_type in case a complex + logic is needed + """ + return 'RelWithDebInfo' + + def root_cmakelists_dir(self): + """Directory where to find the root CMakeLists.txt""" + return self.stage.source_path + + @property + def std_cmake_args(self): + """Standard cmake arguments provided as a property for + convenience of package writers + """ + # standard CMake arguments + return CMakePackage._std_args(self) + + @staticmethod + def _std_args(pkg): + """Computes the standard cmake arguments for a generic package""" + try: + build_type = pkg.build_type() + except AttributeError: + build_type = 'RelWithDebInfo' + + args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), + '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type), + '-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON'] + if platform.mac_ver()[0]: + args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') + + # Set up CMake rpath + args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') + rpaths = ':'.join(spack.build_environment.get_rpaths(pkg)) + args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) + return args + + def build_directory(self): + """Override to provide another place to build the package""" + return join_path(self.stage.source_path, 'spack-build') + + def cmake_args(self): + """Method to be overridden. Should return an iterable containing + all the arguments that must be passed to configure, except: + - CMAKE_INSTALL_PREFIX + - CMAKE_BUILD_TYPE + """ + return [] + + def cmake(self, spec, prefix): + """Run cmake in the build directory""" + options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ + self.cmake_args() + create = not os.path.exists(self.build_directory()) + with working_dir(self.build_directory(), create=create): + inspect.getmodule(self).cmake(*options) + + def build(self, spec, prefix): + """The usual `make` after cmake""" + with working_dir(self.build_directory()): + inspect.getmodule(self).make() + + def install(self, spec, prefix): + """...and the final `make install` after cmake""" + with working_dir(self.build_directory()): + inspect.getmodule(self).make('install') + + @PackageBase.sanity_check('build') + @PackageBase.on_package_attributes(run_tests=True) + def _run_default_function(self): + """This function is run after build if self.run_tests == True + + It will search for a method named `check` and run it. A sensible + default is provided in the base class. + """ + try: + fn = getattr(self, 'check') + tty.msg('Trying default build sanity checks [check]') + fn() + except AttributeError: + tty.msg('Skipping default build sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + + def check(self): + """Default test : search the Makefile for the target `test` + and run them if found. + """ + with working_dir(self.build_directory()): + self._if_make_target_execute('test') + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/build_systems/editable_makefile.py b/lib/spack/spack/build_systems/editable_makefile.py new file mode 100644 index 0000000000..e3adea8363 --- /dev/null +++ b/lib/spack/spack/build_systems/editable_makefile.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect + +from llnl.util.filesystem import working_dir +from spack.package import PackageBase + + +class EditableMakefile(PackageBase): + """Specialized class for packages that are built using editable Makefiles + + This class provides three phases that can be overridden: + - edit + - build + - install + + It is necessary to override the 'edit' phase, while 'build' and 'install' + have sensible defaults. + """ + phases = ['edit', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'EditableMakefile' + + def build_directory(self): + """Directory where the main Makefile is located""" + return self.stage.source_path + + def build_args(self): + """List of arguments that should be passed to make at build time""" + return [] + + def install_args(self): + """List of arguments that should be passed to make at install time""" + return [] + + def edit(self, spec, prefix): + """This phase cannot be defaulted for obvious reasons...""" + raise NotImplementedError('\'edit\' function not implemented') + + def build(self, spec, prefix): + """Default build phase : call make passing build_args""" + args = self.build_args() + with working_dir(self.build_directory()): + inspect.getmodule(self).make(*args) + + def install(self, spec, prefix): + """Default install phase : call make passing install_args""" + args = self.install_args() + ['install'] + with working_dir(self.build_directory()): + inspect.getmodule(self).make(*args) + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9483f370dd..52dbd40f6f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -38,7 +38,6 @@ import copy import functools import inspect import os -import platform import re import sys import textwrap @@ -48,7 +47,6 @@ from StringIO import StringIO import llnl.util.lock import llnl.util.tty as tty import spack -import spack.build_environment import spack.compilers import spack.directives import spack.error @@ -1698,159 +1696,6 @@ class Package(PackageBase): PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) -class EditableMakefile(PackageBase): - phases = ['edit', 'build', 'install'] - # To be used in UI queries that require to know which - # build-system class we are using - build_system_class = 'EditableMakefile' - - def wdir(self): - return self.stage.source_path - - def build_args(self): - return [] - - def install_args(self): - return [] - - def edit(self, spec, prefix): - raise NotImplementedError('\'edit\' function not implemented') - - def build(self, spec, prefix): - args = self.build_args() - with working_dir(self.wdir()): - inspect.getmodule(self).make(*args) - - def install(self, spec, prefix): - args = self.install_args() + ['install'] - with working_dir(self.wdir()): - inspect.getmodule(self).make(*args) - - PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) - - -class AutotoolsPackage(PackageBase): - phases = ['autoreconf', 'configure', 'build', 'install'] - # To be used in UI queries that require to know which - # build-system class we are using - build_system_class = 'AutotoolsPackage' - - def autoreconf(self, spec, prefix): - """Not needed usually, configure should be already there""" - pass - - @PackageBase.sanity_check('autoreconf') - def is_configure_or_die(self): - if not os.path.exists('configure'): - raise RuntimeError( - 'configure script not found in {0}'.format(os.getcwd())) - - def configure_args(self): - return [] - - def configure(self, spec, prefix): - options = ['--prefix={0}'.format(prefix)] + self.configure_args() - inspect.getmodule(self).configure(*options) - - def build(self, spec, prefix): - inspect.getmodule(self).make() - - def install(self, spec, prefix): - inspect.getmodule(self).make('install') - - @PackageBase.sanity_check('build') - @PackageBase.on_package_attributes(run_tests=True) - def _run_default_function(self): - try: - fn = getattr(self, 'check') - tty.msg('Trying default sanity checks [check]') - fn() - except AttributeError: - tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 - - def check(self): - self._if_make_target_execute('test') - self._if_make_target_execute('check') - - # This will be used as a registration decorator in user - # packages, if need be - PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) - - -class CMakePackage(PackageBase): - phases = ['cmake', 'build', 'install'] - # To be used in UI queries that require to know which - # build-system class we are using - build_system_class = 'CMakePackage' - - def build_type(self): - return 'RelWithDebInfo' - - def root_cmakelists_dir(self): - return self.stage.source_path - - @property - def std_cmake_args(self): - # standard CMake arguments - return CMakePackage._std_args(self) - - @staticmethod - def _std_args(pkg): - try: - build_type = pkg.build_type() - except AttributeError: - build_type = 'RelWithDebInfo' - - args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), - '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type), - '-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON'] - if platform.mac_ver()[0]: - args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') - - # Set up CMake rpath - args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') - rpaths = ':'.join(spack.build_environment.get_rpaths(pkg)) - args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) - return args - - def build_directory(self): - return join_path(self.stage.source_path, 'spack-build') - - def cmake_args(self): - return [] - - def cmake(self, spec, prefix): - options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ - self.cmake_args() - create = not os.path.exists(self.build_directory()) - with working_dir(self.build_directory(), create=create): - inspect.getmodule(self).cmake(*options) - - def build(self, spec, prefix): - with working_dir(self.build_directory()): - inspect.getmodule(self).make() - - def install(self, spec, prefix): - with working_dir(self.build_directory()): - inspect.getmodule(self).make('install') - - @PackageBase.sanity_check('build') - @PackageBase.on_package_attributes(run_tests=True) - def _run_default_function(self): - try: - fn = getattr(self, 'check') - tty.msg('Trying default build sanity checks [check]') - fn() - except AttributeError: - tty.msg('Skipping default build sanity checks [method `check` not implemented]') # NOQA: ignore=E501 - - def check(self): - with working_dir(self.build_directory()): - self._if_make_target_execute('test') - - PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) - - def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" flatten_dependencies(spec, prefix) diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index 7acb77b304..fdd9c2111e 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -37,11 +37,11 @@ class Astyle(EditableMakefile): parallel = False - def wdir(self): + def build_directory(self): return join_path(self.stage.source_path, 'build', self.compiler.name) def edit(self, spec, prefix): - makefile = join_path(self.wdir(), 'Makefile') + makefile = join_path(self.build_directory(), 'Makefile') filter_file(r'^CXX\s*=.*', 'CXX=%s' % spack_cxx, makefile) def install_args(self): -- cgit v1.2.3-70-g09d2 From c1ad4bde28a09f39dbae0f6488dc7b7182d11f93 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 24 Oct 2016 16:41:20 -0700 Subject: Rename EditableMakefile to MakefilePackage --- lib/spack/spack/__init__.py | 4 +- lib/spack/spack/build_systems/editable_makefile.py | 77 ---------------------- lib/spack/spack/build_systems/makefile.py | 77 ++++++++++++++++++++++ var/spack/repos/builtin/packages/astyle/package.py | 2 +- 4 files changed, 80 insertions(+), 80 deletions(-) delete mode 100644 lib/spack/spack/build_systems/editable_makefile.py create mode 100644 lib/spack/spack/build_systems/makefile.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 918e17323b..67c64276ee 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -189,14 +189,14 @@ sys_type = None __all__ = ['Package', 'CMakePackage', 'AutotoolsPackage', - 'EditableMakefile', + 'MakefilePackage', 'Version', 'when', 'ver', 'alldeps', 'nolink'] from spack.package import Package, ExtensionConflictError -from spack.build_systems.editable_makefile import EditableMakefile +from spack.build_systems.makefile import MakefilePackage from spack.build_systems.autotools import AutotoolsPackage from spack.build_systems.cmake import CMakePackage from spack.version import Version, ver diff --git a/lib/spack/spack/build_systems/editable_makefile.py b/lib/spack/spack/build_systems/editable_makefile.py deleted file mode 100644 index e3adea8363..0000000000 --- a/lib/spack/spack/build_systems/editable_makefile.py +++ /dev/null @@ -1,77 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - -import inspect - -from llnl.util.filesystem import working_dir -from spack.package import PackageBase - - -class EditableMakefile(PackageBase): - """Specialized class for packages that are built using editable Makefiles - - This class provides three phases that can be overridden: - - edit - - build - - install - - It is necessary to override the 'edit' phase, while 'build' and 'install' - have sensible defaults. - """ - phases = ['edit', 'build', 'install'] - # To be used in UI queries that require to know which - # build-system class we are using - build_system_class = 'EditableMakefile' - - def build_directory(self): - """Directory where the main Makefile is located""" - return self.stage.source_path - - def build_args(self): - """List of arguments that should be passed to make at build time""" - return [] - - def install_args(self): - """List of arguments that should be passed to make at install time""" - return [] - - def edit(self, spec, prefix): - """This phase cannot be defaulted for obvious reasons...""" - raise NotImplementedError('\'edit\' function not implemented') - - def build(self, spec, prefix): - """Default build phase : call make passing build_args""" - args = self.build_args() - with working_dir(self.build_directory()): - inspect.getmodule(self).make(*args) - - def install(self, spec, prefix): - """Default install phase : call make passing install_args""" - args = self.install_args() + ['install'] - with working_dir(self.build_directory()): - inspect.getmodule(self).make(*args) - - # Check that self.prefix is there after installation - PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/build_systems/makefile.py b/lib/spack/spack/build_systems/makefile.py new file mode 100644 index 0000000000..dcddadeedc --- /dev/null +++ b/lib/spack/spack/build_systems/makefile.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect + +from llnl.util.filesystem import working_dir +from spack.package import PackageBase + + +class MakefilePackage(PackageBase): + """Specialized class for packages that are built using editable Makefiles + + This class provides three phases that can be overridden: + - edit + - build + - install + + It is necessary to override the 'edit' phase, while 'build' and 'install' + have sensible defaults. + """ + phases = ['edit', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'MakefilePackage' + + def build_directory(self): + """Directory where the main Makefile is located""" + return self.stage.source_path + + def build_args(self): + """List of arguments that should be passed to make at build time""" + return [] + + def install_args(self): + """List of arguments that should be passed to make at install time""" + return [] + + def edit(self, spec, prefix): + """This phase cannot be defaulted for obvious reasons...""" + raise NotImplementedError('\'edit\' function not implemented') + + def build(self, spec, prefix): + """Default build phase : call make passing build_args""" + args = self.build_args() + with working_dir(self.build_directory()): + inspect.getmodule(self).make(*args) + + def install(self, spec, prefix): + """Default install phase : call make passing install_args""" + args = self.install_args() + ['install'] + with working_dir(self.build_directory()): + inspect.getmodule(self).make(*args) + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index fdd9c2111e..31e1efb591 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -25,7 +25,7 @@ from spack import * -class Astyle(EditableMakefile): +class Astyle(MakefilePackage): """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, Objective-C, C#, and Java Source Code. """ -- cgit v1.2.3-70-g09d2