diff options
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 32 | ||||
| -rw-r--r-- | lib/spack/docs/environments.rst | 47 | ||||
| -rw-r--r-- | lib/spack/docs/packaging_guide.rst | 49 | ||||
| -rwxr-xr-x | lib/spack/env/cc | 2 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/install.py | 61 | ||||
| -rw-r--r-- | lib/spack/spack/environment.py | 60 | ||||
| -rw-r--r-- | lib/spack/spack/installer.py | 802 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/report.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/test/buildrequest.py | 55 | ||||
| -rw-r--r-- | lib/spack/spack/test/buildtask.py | 20 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/dev_build.py | 16 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/env.py | 55 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/install.py | 33 | ||||
| -rw-r--r-- | lib/spack/spack/test/install.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/test/installer.py | 477 | ||||
| -rw-r--r-- | var/spack/repos/builtin.mock/packages/depb/package.py | 22 |
17 files changed, 1222 insertions, 556 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 221516238e..f846cdabe4 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -132,32 +132,28 @@ If ``mpileaks`` depends on other packages, Spack will install the dependencies first. It then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``. You'll see -a number of messages from spack, a lot of build output, and a message -that the packages is installed: +a number of messages from Spack, a lot of build output, and a message +that the package is installed. Add one or more debug options (``-d``) +to get increasingly detailed output. .. code-block:: console $ spack install mpileaks - ==> Installing mpileaks - ==> mpich is already installed in ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4. - ==> callpath is already installed in ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/callpath@1.0.2-5dce4318. - ==> adept-utils is already installed in ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/adept-utils@1.0-5adef8da. - ==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz - ######################################################################## 100.0% - ==> Staging archive: ~/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-debian7-x86_64-59f6ad23/mpileaks-1.0.tar.gz - ==> Created stage in ~/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-debian7-x86_64-59f6ad23. - ==> No patches needed for mpileaks. - ==> Building mpileaks. - - ... build output ... - - ==> Successfully installed mpileaks. - Fetch: 2.16s. Build: 9.82s. Total: 11.98s. - [+] ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpileaks@1.0-59f6ad23 + ... dependency build output ... + ==> Installing mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 + ==> No binary for mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 found: installing from source + ==> mpileaks: Executing phase: 'autoreconf' + ==> mpileaks: Executing phase: 'configure' + ==> mpileaks: Executing phase: 'build' + ==> mpileaks: Executing phase: 'install' + [+] ~/spack/opt/linux-rhel7-broadwell/gcc-8.1.0/mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 The last line, with the ``[+]``, indicates where the package is installed. +Add the debug option -- ``spack install -d mpileaks`` -- to get additional +output. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Building a specific version ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 55ac458a3c..65eec36581 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -191,44 +191,24 @@ Environment has been activated. Similarly, the ``install`` and ==> 0 installed packages $ spack install zlib@1.2.11 - ==> Installing zlib - ==> Searching for binary cache of zlib - ==> Warning: No Spack mirrors are currently configured - ==> No binary for zlib found: installing from source - ==> Fetching http://zlib.net/fossils/zlib-1.2.11.tar.gz - ######################################################################## 100.0% - ==> Staging archive: /spack/var/spack/stage/zlib-1.2.11-3r4cfkmx3wwfqeof4bc244yduu2mz4ur/zlib-1.2.11.tar.gz - ==> Created stage in /spack/var/spack/stage/zlib-1.2.11-3r4cfkmx3wwfqeof4bc244yduu2mz4ur - ==> No patches needed for zlib - ==> Building zlib [Package] - ==> Executing phase: 'install' - ==> Successfully installed zlib - Fetch: 0.36s. Build: 11.58s. Total: 11.93s. - [+] /spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/zlib-1.2.11-3r4cfkmx3wwfqeof4bc244yduu2mz4ur + ==> Installing zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv + ==> No binary for zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv found: installing from source + ==> zlib: Executing phase: 'install' + [+] ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv $ spack env activate myenv $ spack find ==> In environment myenv ==> No root specs - ==> 0 installed packages $ spack install zlib@1.2.8 - ==> Installing zlib - ==> Searching for binary cache of zlib - ==> Warning: No Spack mirrors are currently configured - ==> No binary for zlib found: installing from source - ==> Fetching http://zlib.net/fossils/zlib-1.2.8.tar.gz - ######################################################################## 100.0% - ==> Staging archive: /spack/var/spack/stage/zlib-1.2.8-y2t6kq3s23l52yzhcyhbpovswajzi7f7/zlib-1.2.8.tar.gz - ==> Created stage in /spack/var/spack/stage/zlib-1.2.8-y2t6kq3s23l52yzhcyhbpovswajzi7f7 - ==> No patches needed for zlib - ==> Building zlib [Package] - ==> Executing phase: 'install' - ==> Successfully installed zlib - Fetch: 0.26s. Build: 2.08s. Total: 2.35s. - [+] /spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/zlib-1.2.8-y2t6kq3s23l52yzhcyhbpovswajzi7f7 + ==> Installing zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x + ==> No binary for zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x found: installing from source + ==> zlib: Executing phase: 'install' + [+] ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x + ==> Updating view at ~/spack/var/spack/environments/myenv/.spack-env/view $ spack find ==> In environment myenv @@ -236,15 +216,17 @@ Environment has been activated. Similarly, the ``install`` and zlib@1.2.8 ==> 1 installed package - -- linux-rhel7-x86_64 / gcc@4.9.3 ------------------------------- + -- linux-rhel7-broadwell / gcc@8.1.0 ---------------------------- zlib@1.2.8 $ despacktivate + $ spack find ==> 2 installed packages - -- linux-rhel7-x86_64 / gcc@4.9.3 ------------------------------- + -- linux-rhel7-broadwell / gcc@8.1.0 ---------------------------- zlib@1.2.8 zlib@1.2.11 + Note that when we installed the abstract spec ``zlib@1.2.8``, it was presented as a root of the Environment. All explicitly installed packages will be listed as roots of the Environment. @@ -349,6 +331,9 @@ installed specs using the ``-c`` (``--concretized``) flag. ==> 0 installed packages + +.. _installing-environment: + ^^^^^^^^^^^^^^^^^^^^^^^^^ Installing an Environment ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 66caddeb0b..836fc12b83 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1778,8 +1778,18 @@ RPATHs in Spack are handled in one of three ways: Parallel builds --------------- +Spack supports parallel builds on an individual package and at the +installation level. Package-level parallelism is established by the +``--jobs`` option and its configuration and package recipe equivalents. +Installation-level parallelism is driven by the DAG(s) of the requested +package or packages. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Package-level build parallelism +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + By default, Spack will invoke ``make()``, or any other similar tool, -with a ``-j <njobs>`` argument, so that builds run in parallel. +with a ``-j <njobs>`` argument, so those builds run in parallel. The parallelism is determined by the value of the ``build_jobs`` entry in ``config.yaml`` (see :ref:`here <build-jobs>` for more details on how this value is computed). @@ -1827,6 +1837,43 @@ you set ``parallel`` to ``False`` at the package level, then each call to ``make()`` will be sequential by default, but packagers can call ``make(parallel=True)`` to override it. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Install-level build parallelism +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack supports the concurrent installation of packages within a Spack +instance across multiple processes using file system locks. This +parallelism is separate from the package-level achieved through build +systems' use of the ``-j <njobs>`` option. With install-level parallelism, +processes coordinate the installation of the dependencies of specs +provided on the command line and as part of an environment build with +only **one process** being allowed to install a given package at a time. +Refer to :ref:`Dependencies` for more information on dependencies and +:ref:`installing-environment` for how to install an environment. + +Concurrent processes may be any combination of interactive sessions and +batch jobs. Which means a ``spack install`` can be running in a terminal +window while a batch job is running ``spack install`` on the same or +overlapping dependencies without any process trying to re-do the work of +another. + +For example, if you are using SLURM, you could launch an installation +of ``mpich`` using the following command: + +.. code-block:: console + + $ srun -N 2 -n 8 spack install -j 4 mpich@3.3.2 + +This will create eight concurrent four-job installation on two different +nodes. + +.. note:: + + The effective parallelism will be based on the maximum number of + packages that can be installed at the same time, which will limited + by the number of packages with no (remaining) uninstalled dependencies. + + .. _dependencies: ------------ diff --git a/lib/spack/env/cc b/lib/spack/env/cc index f23b22775b..5efe015c9e 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -22,7 +22,7 @@ # This is an array of environment variables that need to be set before # the script runs. They are set by routines in spack.build_environment -# as part of spack.package.Package.do_install(). +# as part of the package installation process. parameters=( SPACK_ENV_PATH SPACK_DEBUG_LOG_DIR diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index c2875e495b..8803a9f312 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -20,6 +20,7 @@ import spack.fetch_strategy import spack.paths import spack.report from spack.error import SpackError +from spack.installer import PackageInstaller description = "build and install packages" @@ -29,7 +30,7 @@ level = "short" def update_kwargs_from_args(args, kwargs): """Parse cli arguments and construct a dictionary - that will be passed to Package.do_install API""" + that will be passed to the package installer.""" kwargs.update({ 'fail_fast': args.fail_fast, @@ -238,23 +239,29 @@ def default_log_file(spec): return fs.os.path.join(dirname, basename) -def install_spec(cli_args, kwargs, abstract_spec, spec): - """Do the actual installation.""" +def install_specs(cli_args, kwargs, specs): + """Do the actual installation. + + Args: + cli_args (Namespace): argparse namespace with command arguments + kwargs (dict): keyword arguments + specs (list of tuples): list of (abstract, concrete) spec tuples + """ + + # handle active environment, if any + env = ev.get_env(cli_args, 'install') try: - # handle active environment, if any - env = ev.get_env(cli_args, 'install') if env: - with env.write_transaction(): - concrete = env.concretize_and_add( - abstract_spec, spec) - env.write(regenerate_views=False) - env._install(concrete, **kwargs) - with env.write_transaction(): - env.regenerate_views() + for abstract, concrete in specs: + with env.write_transaction(): + concrete = env.concretize_and_add(abstract, concrete) + env.write(regenerate_views=False) + env.install_all(cli_args, **kwargs) else: - spec.package.do_install(**kwargs) - + installs = [(concrete.package, kwargs) for _, concrete in specs] + builder = PackageInstaller(installs) + builder.install() except spack.build_environment.InstallError as e: if cli_args.show_log_on_error: e.print_context() @@ -280,6 +287,10 @@ environment variables: parser.print_help() return + reporter = spack.report.collect_info(args.log_format, args) + if args.log_file: + reporter.filename = args.log_file + if not args.spec and not args.specfiles: # if there are no args but an active environment # then install the packages from it. @@ -294,8 +305,17 @@ environment variables: # once, as it can be slow. env.write(regenerate_views=False) - tty.msg("Installing environment %s" % env.name) - env.install_all(args) + specs = env.all_specs() + if not args.log_file and not reporter.filename: + reporter.filename = default_log_file(specs[0]) + reporter.specs = specs + + tty.msg("Installing environment {0}".format(env.name)) + with reporter: + env.install_all(args, **kwargs) + + tty.debug("Regenerating environment views for {0}" + .format(env.name)) with env.write_transaction(): # It is not strictly required to synchronize view regeneration # but doing so can prevent redundant work in the filesystem. @@ -319,17 +339,13 @@ environment variables: spack.config.set('config:checksum', False, scope='command_line') # Parse cli arguments and construct a dictionary - # that will be passed to Package.do_install API + # that will be passed to the package installer update_kwargs_from_args(args, kwargs) if args.run_tests: tty.warn("Deprecated option: --run-tests: use --test=all instead") # 1. Abstract specs from cli - reporter = spack.report.collect_info(args.log_format, args) - if args.log_file: - reporter.filename = args.log_file - abstract_specs = spack.cmd.parse_specs(args.spec) tests = False if args.test == 'all' or args.run_tests: @@ -401,5 +417,4 @@ environment variables: # overwrite all concrete explicit specs from this build kwargs['overwrite'] = [spec.dag_hash() for spec in specs] - for abstract, concrete in zip(abstract_specs, specs): - install_spec(args, kwargs, abstract, concrete) + install_specs(args, kwargs, zip(abstract_specs, specs)) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 42b4cb819a..0b63bad6a7 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -39,6 +39,7 @@ from spack.spec_list import SpecList, InvalidSpecConstraintError from spack.variant import UnknownVariantError import spack.util.lock as lk from spack.util.path import substitute_path_variables +from spack.installer import PackageInstaller #: environment variable used to indicate the active environment spack_env_var = 'SPACK_ENV' @@ -1371,24 +1372,7 @@ class Environment(object): return ret - def install(self, user_spec, concrete_spec=None, **install_args): - """Install a single spec into an environment. - - This will automatically concretize the single spec, but it won't - affect other as-yet unconcretized specs. - """ - concrete = self.concretize_and_add(user_spec, concrete_spec) - - self._install(concrete, **install_args) - - def _install(self, spec, **install_args): - # "spec" must be concrete - package = spec.package - - install_args['overwrite'] = install_args.get( - 'overwrite', []) + self._get_overwrite_specs() - package.do_install(**install_args) - + def _install_log_links(self, spec): if not spec.external: # Make sure log directory exists log_path = self.log_path @@ -1402,19 +1386,22 @@ class Environment(object): os.remove(build_log_link) os.symlink(spec.package.build_log_path, build_log_link) - def install_all(self, args=None): + def install_all(self, args=None, **install_args): """Install all concretized specs in an environment. Note: this does not regenerate the views for the environment; that needs to be done separately with a call to write(). + Args: + args (Namespace): argparse namespace with command arguments + install_args (dict): keyword install arguments """ - # If "spack install" is invoked repeatedly for a large environment # where all specs are already installed, the operation can take # a large amount of time due to repeatedly acquiring and releasing # locks, this does an initial check across all specs within a single # DB read transaction to reduce time spent in this case. + tty.debug('Assessing installation status of environment packages') specs_to_install = [] with spack.store.db.read_transaction(): for concretized_hash in self.concretized_order: @@ -1426,14 +1413,43 @@ class Environment(object): # If it's a dev build it could need to be reinstalled specs_to_install.append(spec) + if not specs_to_install: + tty.msg('All of the packages are already installed') + return + + tty.debug('Processing {0} uninstalled specs' + .format(len(specs_to_install))) + + install_args['overwrite'] = install_args.get( + 'overwrite', []) + self._get_overwrite_specs() + + installs = [] for spec in specs_to_install: # Parse cli arguments and construct a dictionary - # that will be passed to Package.do_install API + # that will be passed to the package installer kwargs = dict() + if install_args: + kwargs.update(install_args) if args: spack.cmd.install.update_kwargs_from_args(args, kwargs) - self._install(spec, **kwargs) + installs.append((spec.package, kwargs)) + + try: + builder = PackageInstaller(installs) + builder.install() + finally: + # Ensure links are set appropriately + for spec in specs_to_install: + if spec.package.installed: + try: + self._install_log_links(spec) + except OSError as e: + tty.warn('Could not install log links for {0}: {1}' + .format(spec.name, str(e))) + + with self.write_transaction(): + self.regenerate_views() def all_specs(self): """Return all specs, even those a user spec would shadow.""" diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index 51919f3729..cf7b5bfb99 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -27,6 +27,7 @@ This module supports the coordination of local and distributed concurrent installations of packages in a Spack instance. """ +import copy import glob import heapq import itertools @@ -36,6 +37,8 @@ import six import sys import time +from collections import defaultdict + import llnl.util.filesystem as fs import llnl.util.lock as lk import llnl.util.tty as tty @@ -80,6 +83,31 @@ STATUS_DEQUEUED = 'dequeued' STATUS_REMOVED = 'removed' +def _check_last_phase(pkg): + """ + Ensures the specified package has a valid last phase before proceeding + with its installation. + + The last phase is also set to None if it is the last phase of the + package already. + + Args: + pkg (PackageBase): the package being installed + + Raises: + ``BadInstallPhase`` if stop_before or last phase is invalid + """ + if pkg.stop_before_phase and pkg.stop_before_phase not in pkg.phases: + raise BadInstallPhase(pkg.name, pkg.stop_before_phase) + + if pkg.last_phase and pkg.last_phase not in pkg.phases: + raise BadInstallPhase(pkg.name, pkg.last_phase) + + # If we got a last_phase, make sure it's not already last + if pkg.last_phase and pkg.last_phase == pkg.phases[-1]: + pkg.last_phase = None + + def _handle_external_and_upstream(pkg, explicit): """ Determine if the package is external or upstream and register it in the @@ -96,6 +124,8 @@ def _handle_external_and_upstream(pkg, explicit): # consists in module file generation and registration in the DB. if pkg.spec.external: _process_external_package(pkg, explicit) + _print_installed_pkg('{0} (external {1})' + .format(pkg.prefix, package_id(pkg))) return True if pkg.installed_upstream: @@ -279,8 +309,8 @@ def _process_external_package(pkg, explicit): tty.debug('{0} is actually installed in {1}' .format(pre, spec.external_path)) else: - tty.msg('{0} externally installed in {1}' - .format(pre, spec.external_path)) + tty.debug('{0} externally installed in {1}' + .format(pre, spec.external_path)) try: # Check if the package was already registered in the DB. @@ -452,6 +482,19 @@ def dump_packages(spec, path): fs.install_tree(source_pkg_dir, dest_pkg_dir) +def get_dependent_ids(spec): + """ + Return a list of package ids for the spec's dependents + + Args: + spec (Spec): Concretized spec + + Returns: + (list of str): list of package ids + """ + return [package_id(d.package) for d in spec.dependents()] + + def install_msg(name, pid): """ Colorize the name/id of the package being installed @@ -472,7 +515,7 @@ def log(pkg): Copy provenance into the install directory on success Args: - pkg (Package): the package that was installed and built + pkg (Package): the package that was built and installed """ packages_dir = spack.store.layout.build_packages_path(pkg.spec) @@ -544,61 +587,17 @@ def package_id(pkg): The identifier is used to track build tasks, locks, install, and failure statuses. + The identifier needs to distinguish between combinations of compilers + and packages for combinatorial environments. + Args: pkg (PackageBase): the package from which the identifier is derived """ if not pkg.spec.concrete: raise ValueError("Cannot provide a unique, readable id when " "the spec is not concretized.") - # TODO: Restore use of the dag hash once resolve issues with different - # TODO: hashes being associated with dependents of different packages - # TODO: within the same install, such as the hash for callpath being - # TODO: different for mpich and dyninst in the - # TODO: test_force_uninstall_and_reinstall_by_hash` test. - - # TODO: Is the extra "readability" of the version worth keeping? - # return "{0}-{1}-{2}".format(pkg.name, pkg.version, pkg.spec.dag_hash()) - - # TODO: Including the version causes some installs to fail. Specifically - # TODO: failures occur when the version of a dependent of one of the - # TODO: packages does not match the version that is installed. - # return "{0}-{1}".format(pkg.name, pkg.version) - - return pkg.name - - -install_args_docstring = """ - cache_only (bool): Fail if binary package unavailable. - dirty (bool): Don't clean the build environment before installing. - explicit (bool): True if package was explicitly installed, False - if package was implicitly installed (as a dependency). - fail_fast (bool): Fail if any dependency fails to install; - otherwise, the default is to install as many dependencies as - possible (i.e., best effort installation). - fake (bool): Don't really build; install fake stub files instead. - install_deps (bool): Install dependencies before installing this - package - install_source (bool): By default, source is not installed, but - for debugging it might be useful to keep it around. - keep_prefix (bool): Keep install prefix on failure. By default, - destroys it. - keep_stage (bool): By default, stage is destroyed only if there - are no exceptions during build. Set to True to keep the stage - even with exceptions. - overwrite (list): list of hashes for packages to do overwrite - installs. Default empty list. - restage (bool): Force spack to restage the package source. - skip_patch (bool): Skip patch stage of build if True. - stop_before (InstallPhase): stop execution before this - installation phase (or None) - stop_at (InstallPhase): last installation phase to be executed - (or None) - tests (bool or list or set): False to run no tests, True to test - all packages, or a list of package names to run tests for some - use_cache (bool): Install from binary package, if available. - verbose (bool): Display verbose build output (by default, - suppresses it) - """ + + return "{0}-{1}-{2}".format(pkg.name, pkg.version, pkg.spec.dag_hash()) class PackageInstaller(object): @@ -611,29 +610,19 @@ class PackageInstaller(object): instance. ''' - def __init__(self, pkg): - """ - Initialize and set up the build specs. + def __init__(self, installs=[]): + """ Initialize the installer. Args: - pkg (PackageBase): the package being installed, whose spec is - concrete - + installs (list of (pkg, install_args)): list of tuples, where each + tuple consists of a package (PackageBase) and its associated + install arguments (dict) Return: (PackageInstaller) instance """ - if not isinstance(pkg, spack.package.PackageBase): - raise ValueError("{0} must be a package".format(str(pkg))) - - if not pkg.spec.concrete: - raise ValueError("{0}: Can only install concrete packages." - .format(pkg.spec.name)) - - # Spec of the package to be built - self.pkg = pkg - - # The identifier used for the explicit package being built - self.pkg_id = package_id(pkg) + # List of build requests + self.build_requests = [BuildRequest(pkg, install_args) + for pkg, install_args in installs] # Priority queue of build tasks self.build_pq = [] @@ -650,16 +639,16 @@ class PackageInstaller(object): # Cache of installed packages' unique ids self.installed = set() - # Cache of overwrite information - self.overwrite = set() - self.overwrite_time = time.time() - # Data store layout self.layout = spack.store.layout # Locks on specs being built, keyed on the package's unique id self.locks = {} + # Cache fail_fast option to ensure if one build request asks to fail + # fast then that option applies to all build requests. + self.fail_fast = False + def __repr__(self): """Returns a formal representation of the package installer.""" rep = '{0}('.format(self.__class__.__name__) @@ -669,24 +658,48 @@ class PackageInstaller(object): def __str__(self): """Returns a printable version of the package installer.""" + requests = '#requests={0}'.format(len(self.build_requests)) tasks = '#tasks={0}'.format(len(self.build_tasks)) failed = 'failed ({0}) = {1}'.format(len(self.failed), self.failed) installed = 'installed ({0}) = {1}'.format( len(self.installed), self.installed) - return '{0} ({1}): {2}; {3}; {4}'.format( - self.pkg_id, self.pid, tasks, installed, failed) + return '{0}: {1}; {2}; {3}; {4}'.format( + self.pid, requests, tasks, installed, failed) - def _add_bootstrap_compilers(self, pkg): + def _add_bootstrap_compilers(self, pkg, request, all_deps): """ Add bootstrap compilers and dependencies to the build queue. Args: pkg (PackageBase): the package with possible compiler dependencies + request (BuildRequest): the associated install request + all_deps (defaultdict(set)): dictionary of all dependencies and + associated dependents """ packages = _packages_needed_to_bootstrap_compiler(pkg) for (comp_pkg, is_compiler) in packages: if package_id(comp_pkg) not in self.build_tasks: - self._push_task(comp_pkg, is_compiler, 0, 0, STATUS_ADDED) + self._add_init_task(comp_pkg, request, is_compiler, all_deps) + + def _add_init_task(self, pkg, request, is_compiler, all_deps): + """ + Creates and queus the initial build task for the package. + + Args: + pkg (Package): the package to be built and installed + request (BuildRequest or None): the associated install request + where ``None`` can be used to indicate the package was + explicitly requested by the user + is_compiler (bool): whether task is for a bootstrap compiler + all_deps (defaultdict(set)): dictionary of all dependencies and + associated dependents + """ + task = BuildTask(pkg, request, is_compiler, 0, 0, STATUS_ADDED, + self.installed) + for dep_id in task.dependencies: + all_deps[dep_id].add(package_id(pkg)) + + self._push_task(task) def _check_db(self, spec): """Determine if the spec is flagged as installed in the database @@ -710,11 +723,14 @@ class PackageInstaller(object): installed_in_db = False return rec, installed_in_db - def _check_deps_status(self): - """Check the install status of the explicit spec's dependencies""" + def _check_deps_status(self, request): + """Check the install status of the requested package + Args: + request (BuildRequest): the associated install request + """ err = 'Cannot proceed with {0}: {1}' - for dep in self.spec.traverse(order='post', root=False): + for dep in request.traverse_dependencies(): dep_pkg = dep.package dep_id = package_id(dep_pkg) @@ -723,7 +739,7 @@ class PackageInstaller(object): action = "'spack install' the dependency" msg = '{0} is marked as an install failure: {1}' \ .format(dep_id, action) - raise InstallError(err.format(self.pkg_id, msg)) + raise InstallError(err.format(request.pkg_id, msg)) # Attempt to get a write lock to ensure another process does not # uninstall the dependency while the requested spec is being @@ -731,27 +747,24 @@ class PackageInstaller(object): ltype, lock = self._ensure_locked('write', dep_pkg) if lock is None: msg = '{0} is write locked by another process'.format(dep_id) - raise InstallError(err.format(self.pkg_id, msg)) + raise InstallError(err.format(request.pkg_id, msg)) # Flag external and upstream packages as being installed if dep_pkg.spec.external or dep_pkg.installed_upstream: - form = 'external' if dep_pkg.spec.external else 'upstream' - tty.debug('Flagging {0} {1} as installed'.format(form, dep_id)) - self.installed.add(dep_id) + self._flag_installed(dep_pkg) continue # Check the database to see if the dependency has been installed # and flag as such if appropriate rec, installed_in_db = self._check_db(dep) if installed_in_db and ( - dep.dag_hash() not in self.overwrite or - rec.installation_time > self.overwrite_time): + dep.dag_hash() not in request.overwrite or + rec.installation_time > request.overwrite_time): tty.debug('Flagging {0} as installed per the database' .format(dep_id)) - self.installed.add(dep_id) + self._flag_installed(dep_pkg) - def _prepare_for_install(self, task, keep_prefix, keep_stage, - restage=False): + def _prepare_for_install(self, task): """ Check the database and leftover installation directories/files and prepare for a new install attempt for an uninstalled package. @@ -762,13 +775,12 @@ class PackageInstaller(object): Args: task (BuildTask): the build task whose associated package is being checked - keep_prefix (bool): ``True`` if the prefix is to be kept on - failure, otherwise ``False`` - keep_stage (bool): ``True`` if the stage is to be kept even if - there are exceptions, otherwise ``False`` - restage (bool): ``True`` if forcing Spack to restage the package - source, otherwise ``False`` """ + install_args = task.request.install_args + keep_prefix = install_args.get('keep_prefix') + keep_stage = install_args.get('keep_stage') + restage = install_args.get('restage') + # Make sure the package is ready to be locally installed. self._ensure_install_ready(task.pkg) @@ -797,13 +809,13 @@ class PackageInstaller(object): task.pkg.stage.destroy() if not partial and self.layout.check_installed(task.pkg.spec) and ( - rec.spec.dag_hash() not in self.overwrite or - rec.installation_time > self.overwrite_time + rec.spec.dag_hash() not in task.request.overwrite or + rec.installation_time > task.request.overwrite_time ): self._update_installed(task) # Only update the explicit entry once for the explicit package - if task.pkg_id == self.pkg_id: + if task.explicit: _update_explicit_entry_in_db(task.pkg, rec, True) # In case the stage directory has already been created, this @@ -812,38 +824,6 @@ class PackageInstaller(object): if not keep_stage: task.pkg.stage.destroy() - def _check_last_phase(self, **kwargs): - """ - Ensures the package being installed has a valid last phase before - proceeding with the installation. - - The ``stop_before`` or ``stop_at`` arguments are removed from the - installation arguments. - - The last phase is also set to None if it is the last phase of the - package already - - Args: - kwargs: - ``stop_before``': stop before execution of this phase (or None) - ``stop_at``': last installation phase to be executed (or None) - """ - self.pkg.stop_before_phase = kwargs.pop('stop_before', None) - if self.pkg.stop_before_phase is not None and \ - self.pkg.stop_before_phase not in self.pkg.phases: - tty.die('\'{0}\' is not an allowed phase for package {1}' - .format(self.pkg.stop_before_phase, self.pkg.name)) - - self.pkg.last_phase = kwargs.pop('stop_at', None) - if self.pkg.last_phase is not None and \ - self.pkg.last_phase not in self.pkg.phases: - tty.die('\'{0}\' is not an allowed phase for package {1}' - .format(self.pkg.last_phase, self.pkg.name)) - # If we got a last_phase, make sure it's not already last - if self.pkg.last_phase and \ - self.pkg.last_phase == self.pkg.phases[-1]: - self.pkg.last_phase = None - def _cleanup_all_tasks(self): """Cleanup all build tasks to include releasing their locks.""" for pkg_id in self.locks: @@ -1000,30 +980,53 @@ class PackageInstaller(object): self.locks[pkg_id] = (lock_type, lock) return self.locks[pkg_id] - def _init_queue(self, install_deps, install_package): - """ - Initialize the build task priority queue and spec state. + def _add_tasks(self, request, all_deps): + """Add tasks to the priority queue for the given build request. + + It also tracks all dependents associated with each dependency in + order to ensure proper tracking of uninstalled dependencies. Args: - install_deps (bool): ``True`` if installing package dependencies, - otherwise ``False`` - install_package (bool): ``True`` if installing the package, - otherwise ``False`` + request (BuildRequest): the associated install request + all_deps (defaultdict(set)): dictionary of all dependencies and + associated dependents """ - tty.debug('Initializing the build queue for {0}'.format(self.pkg.name)) + tty.debug('Initializing the build queue for {0}' + .format(request.pkg.name)) + + # Ensure not attempting to perform an installation when user didn't + # want to go that far for the requested package. + try: + _check_last_phase(request.pkg) + except BadInstallPhase as err: + tty.warn('Installation request refused: {0}'.format(str(err))) + return + + # Skip out early if the spec is not being installed locally (i.e., if + # external or upstream). + # + # External and upstream packages need to get flagged as installed to + # ensure proper status tracking for environment build. + not_local = _handle_external_and_upstream(request.pkg, True) + if not_local: + self._flag_installed(request.pkg) + return + install_compilers = spack.config.get( 'config:install_missing_compilers', False) + install_deps = request.install_args.get('install_deps') if install_deps: - for dep in self.spec.traverse(order='post', root=False): + for dep in request.traverse_dependencies(): dep_pkg = dep.package # First push any missing compilers (if requested) if install_compilers: - self._add_bootstrap_compilers(dep_pkg) + self._add_bootstrap_compilers(dep_pkg, request, all_deps) - if package_id(dep_pkg) not in self.build_tasks: - self._push_task(dep_pkg, False, 0, 0, STATUS_ADDED) + dep_id = package_id(dep_pkg) + if dep_id not in self.build_tasks: + self._add_init_task(dep_pkg, request, False, all_deps) # Clear any persistent failure markings _unless_ they are # associated with another process in this parallel build @@ -1033,21 +1036,26 @@ class PackageInstaller(object): # Push any missing compilers (if requested) as part of the # package dependencies. if install_compilers: - self._add_bootstrap_compilers(self.pkg) + self._add_bootstrap_compilers(request.pkg, request, all_deps) - if install_package and self.pkg_id not in self.build_tasks: + install_package = request.install_args.get('install_package') + if install_package and request.pkg_id not in self.build_tasks: # Be sure to clear any previous failure - spack.store.db.clear_failure(self.pkg.spec, force=True) + spack.store.db.clear_failure(request.spec, force=True) # If not installing dependencies, then determine their # installation status before proceeding if not install_deps: - self._check_deps_status() + self._check_deps_status(request) # Now add the package itself, if appropriate - self._push_task(self.pkg, False, 0, 0, STATUS_ADDED) + self._add_init_task(request.pkg, request, False, all_deps) - def _install_task(self, task, **kwargs): + # Ensure if one request is to fail fast then all requests will. + fail_fast = request.install_args.get('fail_fast') + self.fail_fast = self.fail_fast or fail_fast + + def _install_task(self, task): """ Perform the installation of the requested spec and/or dependency represented by the build task. @@ -1055,15 +1063,15 @@ class PackageInstaller(object): Args: task (BuildTask): the installation build task for a package""" - cache_only = kwargs.get('cache_only', False) - tests = kwargs.get('tests', False) - unsigned = kwargs.get('unsigned', False) - use_cache = kwargs.get('use_cache', True) - full_hash_match = kwargs.get('full_hash_match', False) + install_args = task.request.install_args + cache_only = install_args.get('cache_only') + explicit = task.explicit + full_hash_match = install_args.get('full_hash_match') + tests = install_args.get('tests') + unsigned = install_args.get('unsigned') + use_cache = install_args.get('use_cache') - pkg = task.pkg - pkg_id = package_id(pkg) - explicit = pkg_id == self.pkg_id + pkg, pkg_id = task.pkg, task.pkg_id tty.msg(install_msg(pkg_id, self.pid)) task.start = task.start or time.time() @@ -1093,7 +1101,7 @@ class PackageInstaller(object): # Preserve verbosity settings across installs. spack.package.PackageBase._verbose = ( spack.build_environment.start_build_process( - pkg, build_process, kwargs) + pkg, build_process, install_args) ) # Note: PARENT of the build process adds the new package to @@ -1113,8 +1121,6 @@ class PackageInstaller(object): tty.debug('Package stage directory: {0}' .format(pkg.stage.source_path)) - _install_task.__doc__ += install_args_docstring - def _next_is_pri0(self): """ Determine if the next build task has priority 0 @@ -1141,32 +1147,39 @@ class PackageInstaller(object): return task return None - def _push_task(self, pkg, compiler, start, attempts, status): + def _push_task(self, task): """ - Create and push (or queue) a build task for the package. + Push (or queue) the specified build task for the package. Source: Customization of "add_task" function at docs.python.org/2/library/heapq.html + + Args: + task (BuildTask): the installation build task for a package """ msg = "{0} a build task for {1} with status '{2}'" - pkg_id = package_id(pkg) + skip = 'Skipping requeue of task for {0}: {1}' - # Ensure do not (re-)queue installed or failed packages. - assert pkg_id not in self.installed - assert pkg_id not in self.failed + # Ensure do not (re-)queue installed or failed packages whose status + # may have been determined by a separate process. + if task.pkg_id in self.installed: + tty.debug(skip.format(task.pkg_id, 'installed')) + return + + if task.pkg_id in self.failed: + tty.debug(skip.format(task.pkg_id, 'failed')) + return # Remove any associated build task since its sequence will change - self._remove_task(pkg_id) - desc = 'Queueing' if attempts == 0 else 'Requeueing' - tty.verbose(msg.format(desc, pkg_id, status)) + self._remove_task(task.pkg_id) + desc = 'Queueing' if task.attempts == 0 else 'Requeueing' + tty.verbose(msg.format(desc, task.pkg_id, task.status)) # Now add the new task to the queue with a new sequence number to # ensure it is the last entry popped with the same priority. This # is necessary in case we are re-queueing a task whose priority # was decremented due to the installation of one of its dependencies. - task = BuildTask(pkg, compiler, start, attempts, status, - self.installed) - self.build_tasks[pkg_id] = task + self.build_tasks[task.pkg_id] = task heapq.heappush(self.build_pq, (task.key, task)) def _release_lock(self, pkg_id): @@ -1221,16 +1234,16 @@ class PackageInstaller(object): tty.msg('{0} {1}'.format(install_msg(task.pkg_id, self.pid), 'in progress by another process')) - start = task.start or time.time() - self._push_task(task.pkg, task.compiler, start, task.attempts, - STATUS_INSTALLING) + new_task = task.next_attempt(self.installed) + new_task.status = STATUS_INSTALLING + self._push_task(new_task) def _setup_install_dir(self, pkg): """ Create and ensure proper access controls for the install directory. Args: - pkg (Package): the package to be installed and built + pkg (Package): the package to be built and installed """ if not os.path.exists(pkg.spec.prefix): tty.verbose('Creating the installation directory {0}' @@ -1268,7 +1281,7 @@ class PackageInstaller(object): err = '' if exc is None else ': {0}'.format(str(exc)) tty.debug('Flagging {0} as failed{1}'.format(pkg_id, err)) if mark: - self.failed[pkg_id] = spack.store.db.mark_failed(task.spec) + self.failed[pkg_id] = spack.store.db.mark_failed(task.pkg.spec) else: self.failed[pkg_id] = None task.status = STATUS_FAILED @@ -1288,77 +1301,90 @@ class PackageInstaller(object): def _update_installed(self, task): """ - Mark the task's spec as installed and update the dependencies of its - dependents. + Mark the task as installed and ensure dependent build tasks are aware. Args: task (BuildTask): the build task for the installed package """ - pkg_id = task.pkg_id + task.status = STATUS_INSTALLED + self._flag_installed(task.pkg, task.dependents) + + def _flag_installed(self, pkg, dependent_ids=None): + """ + Flag the package as installed and ensure known by all build tasks of + known dependents. + + Args: + pkg (Package): Package that has been installed locally, externally + or upstream + dependent_ids (list of str or None): list of the package's + dependent ids, or None if the dependent ids are limited to + those maintained in the package (dependency DAG) + """ + pkg_id = package_id(pkg) + + if pkg_id in self.installed: + # Already determined the package has been installed + return + tty.debug('Flagging {0} as installed'.format(pkg_id)) self.installed.add(pkg_id) - task.status = STATUS_INSTALLED - for dep_id in task.dependents: + + # Update affected dependents + dependent_ids = dependent_ids or get_dependent_ids(pkg.spec) + for dep_id in set(dependent_ids): tty.debug('Removing {0} from {1}\'s uninstalled dependencies.' .format(pkg_id, dep_id)) if dep_id in self.build_tasks: # Ensure the dependent's uninstalled dependencies are # up-to-date. This will require requeueing the task. dep_task = self.build_tasks[dep_id] - dep_task.flag_installed(self.installed) - self._push_task(dep_task.pkg, dep_task.compiler, - dep_task.start, dep_task.attempts, - dep_task.status) + self._push_task(dep_task.next_attempt(self.installed)) else: tty.debug('{0} has no build task to update for {1}\'s success' .format(dep_id, pkg_id)) - def install(self, **kwargs): - """ - Install the package and/or associated dependencies. - - Args:""" - - fail_fast = kwargs.get('fail_fast', False) - install_deps = kwargs.get('install_deps', True) - keep_prefix = kwargs.get('keep_prefix', False) - keep_stage = kwargs.get('keep_stage', False) - restage = kwargs.get('restage', False) - - fail_fast_err = 'Terminating after first install failure' + def _init_queue(self): + """Initialize the build queue from the list of build requests.""" + all_dependencies = defaultdict(set) - # install_package defaults True and is popped so that dependencies are - # always installed regardless of whether the root was installed - install_package = kwargs.pop('install_package', True) + tty.debug('Initializing the build queue from the build requests') + for request in self.build_requests: + self._add_tasks(request, all_dependencies) - # take a timestamp with the overwrite argument to check whether another - # process has already overridden the package. - self.overwrite = set(kwargs.get('overwrite', [])) - if self.overwrite: - self.overwrite_time = time.time() + # Add any missing dependents to ensure proper uninstalled dependency + # tracking when installing multiple specs + tty.debug('Ensure all dependencies know all dependents across specs') + for dep_id in all_dependencies: + if dep_id in self.build_tasks: + dependents = all_dependencies[dep_id] + task = self.build_tasks[dep_id] + for dependent_id in dependents.difference(task.dependents): + task.add_dependent(dependent_id) - # Ensure not attempting to perform an installation when user didn't - # want to go that far. - self._check_last_phase(**kwargs) + def install(self): + """ + Install the requested package(s) and or associated dependencies. - # Skip out early if the spec is not being installed locally (i.e., if - # external or upstream). - not_local = _handle_external_and_upstream(self.pkg, True) - if not_local: - return + Args: + pkg (Package): the package to be built and installed""" + self._init_queue() - # Initialize the build task queue - self._init_queue(install_deps, install_package) + fail_fast_err = 'Terminating after first install failure' + single_explicit_spec = len(self.build_requests) == 1 + failed_explicits = [] + exists_errors = [] - # Proceed with the installation while self.build_pq: task = self._pop_task() if task is None: continue - pkg, spec = task.pkg, task.pkg.spec - pkg_id = package_id(pkg) + install_args = task.request.install_args + keep_prefix = install_args.get('keep_prefix') + + pkg, pkg_id, spec = task.pkg, task.pkg_id, task.pkg.spec tty.verbose('Processing {0}: task={1}'.format(pkg_id, task)) # Ensure that the current spec has NO uninstalled dependencies, @@ -1372,6 +1398,11 @@ class PackageInstaller(object): if task.priority != 0: tty.error('Detected uninstalled dependencies for {0}: {1}' .format(pkg_id, task.uninstalled_deps)) + left = [dep_id for dep_id in task.uninstalled_deps if + dep_id not in self.installed] + if not left: + tty.warn('{0} does NOT actually have any uninstalled deps' + ' left'.format(pkg_id)) dep_str = 'dependencies' if task.priority > 1 else 'dependency' raise InstallError( 'Cannot proceed with {0}: {1} uninstalled {2}: {3}' @@ -1381,11 +1412,9 @@ class PackageInstaller(object): # Skip the installation if the spec is not being installed locally # (i.e., if external or upstream) BUT flag it as installed since # some package likely depends on it. - if pkg_id != self.pkg_id: - not_local = _handle_external_and_upstream(pkg, False) - if not_local: - self._update_installed(task) - _print_installed_pkg(pkg.prefix) + if not task.explicit: + if _handle_external_and_upstream(pkg, False): + self._flag_installed(pkg, task.dependents) continue # Flag a failed spec. Do not need an (install) prefix lock since @@ -1394,7 +1423,7 @@ class PackageInstaller(object): tty.warn('{0} failed to install'.format(pkg_id)) self._update_failed(task) - if fail_fast: + if self.fail_fast: raise InstallError(fail_fast_err) continue @@ -1417,9 +1446,13 @@ class PackageInstaller(object): self._requeue_task(task) continue + # Take a timestamp with the overwrite argument to allow checking + # whether another process has already overridden the package. + if task.request.overwrite and task.explicit: + task.request.overwrite_time = time.time() + # Determine state of installation artifacts and adjust accordingly. - self._prepare_for_install(task, keep_prefix, keep_stage, - restage) + self._prepare_for_install(task) # Flag an already installed package if pkg_id in self.installed: @@ -1464,23 +1497,24 @@ class PackageInstaller(object): # Proceed with the installation since we have an exclusive write # lock on the package. try: - if pkg.spec.dag_hash() in self.overwrite: + if pkg.spec.dag_hash() in task.request.overwrite: rec, _ = self._check_db(pkg.spec) if rec and rec.installed: - if rec.installation_time < self.overwrite_time: + if rec.installation_time < task.request.overwrite_time: # If it's actually overwriting, do a fs transaction if os.path.exists(rec.path): with fs.replace_directory_transaction( rec.path): - self._install_task(task, **kwargs) + self._install_task(task) else: tty.debug("Missing installation to overwrite") - self._install_task(task, **kwargs) + self._install_task(task) else: # overwriting nothing - self._install_task(task, **kwargs) + self._install_task(task) else: - self._install_task(task, **kwargs) + self._install_task(task) + self._update_installed(task) # If we installed then we should keep the prefix @@ -1489,41 +1523,55 @@ class PackageInstaller(object): keep_prefix = keep_prefix or \ (stop_before_phase is None and last_phase is None) - except spack.directory_layout.InstallDirectoryAlreadyExistsError: - tty.debug("Keeping existing install prefix in place.") + except spack.directory_layout.InstallDirectoryAlreadyExistsError \ + as err: + tty.debug('Install prefix for {0} exists, keeping {1} in ' + 'place.'.format(pkg.name, pkg.prefix)) self._update_installed(task) - raise + + # Only terminate at this point if a single build request was + # made. + if task.explicit and single_explicit_spec: + raise + + if task.explicit: + exists_errors.append((pkg_id, str(err))) except KeyboardInterrupt as exc: - # The build has been terminated with a Ctrl-C so terminate. + # The build has been terminated with a Ctrl-C so terminate + # regardless of the number of remaining specs. err = 'Failed to install {0} due to {1}: {2}' tty.error(err.format(pkg.name, exc.__class__.__name__, str(exc))) raise except (Exception, SystemExit) as exc: + self._update_failed(task, True, exc) + # Best effort installs suppress the exception and mark the - # package as a failure UNLESS this is the explicit package. + # package as a failure. if (not isinstance(exc, spack.error.SpackError) or not exc.printed): # SpackErrors can be printed by the build process or at # lower levels -- skip printing if already printed. - # TODO: sort out this and SpackEror.print_context() - err = 'Failed to install {0} due to {1}: {2}' - tty.error( - err.format(pkg.name, exc.__class__.__name__, str(exc))) - - self._update_failed(task, True, exc) - - if fail_fast: - # The user requested the installation to terminate on - # failure. + # TODO: sort out this and SpackError.print_context() + tty.error('Failed to install {0} due to {1}: {2}' + .format(pkg.name, exc.__class__.__name__, + str(exc))) + # Terminate if requested to do so on the first failure. + if self.fail_fast: raise InstallError('{0}: {1}' .format(fail_fast_err, str(exc))) - if pkg_id == self.pkg_id: + # Terminate at this point if the single explicit spec has + # failed to install. + if single_explicit_spec and task.explicit: raise + # Track explicit spec id and error to summarize when done + if task.explicit: + failed_explicits.append((pkg_id, str(exc))) + finally: # Remove the install prefix if anything went wrong during # install. @@ -1542,20 +1590,16 @@ class PackageInstaller(object): # Cleanup, which includes releasing all of the read locks self._cleanup_all_tasks() - # Ensure we properly report if the original/explicit pkg is failed - if self.pkg_id in self.failed: - msg = ('Installation of {0} failed. Review log for details' - .format(self.pkg_id)) - raise InstallError(msg) + # Ensure we properly report if one or more explicit specs failed + if exists_errors or failed_explicits: + for pkg_id, err in exists_errors: + tty.error('{0}: {1}'.format(pkg_id, err)) - install.__doc__ += install_args_docstring + for pkg_id, err in failed_explicits: + tty.error('{0}: {1}'.format(pkg_id, err)) - # Helper method to "smooth" the transition from the - # spack.package.PackageBase class - @property - def spec(self): - """The specification associated with the package.""" - return self.pkg.spec + raise InstallError('Installation request failed. Refer to ' + 'recent errors for specific package(s).') def build_process(pkg, kwargs): @@ -1672,14 +1716,17 @@ def build_process(pkg, kwargs): class BuildTask(object): """Class for representing the build task for a package.""" - def __init__(self, pkg, compiler, start, attempts, status, installed): + def __init__(self, pkg, request, compiler, start, attempts, status, + installed): """ Instantiate a build task for a package. Args: - pkg (Package): the package to be installed and built - compiler (bool): ``True`` if the task is for a bootstrap compiler, - otherwise, ``False`` + pkg (Package): the package to be built and installed + request (BuildRequest or None): the associated install request + where ``None`` can be used to indicate the package was + explicitly requested by the user + compiler (bool): whether task is for a bootstrap compiler start (int): the initial start time for the package, in seconds attempts (int): the number of attempts to install the package status (str): the installation status @@ -1699,6 +1746,12 @@ class BuildTask(object): # The "unique" identifier for the task's package self.pkg_id = package_id(self.pkg) + # The explicit build request associated with the package + if not isinstance(request, BuildRequest): + raise ValueError("{0} must have a build request".format(str(pkg))) + + self.request = request + # Initialize the status to an active state. The status is used to # ensure priority queue invariants when tasks are "removed" from the # queue. @@ -1714,28 +1767,26 @@ class BuildTask(object): # The initial start time for processing the spec self.start = start - # Number of times the task has been queued - self.attempts = attempts + 1 - - # Set of dependents - self.dependents = set(package_id(d.package) for d - in self.spec.dependents()) + # Set of dependents, which needs to include the requesting package + # to support tracking of parallel, multi-spec, environment installs. + self.dependents = set(get_dependent_ids(self.pkg.spec)) # Set of dependencies # # Be consistent wrt use of dependents and dependencies. That is, # if use traverse for transitive dependencies, then must remove # transitive dependents on failure. + deptypes = self.request.get_deptypes(self.pkg) self.dependencies = set(package_id(d.package) for d in - self.spec.dependencies() if - package_id(d.package) != self.pkg_id) + self.pkg.spec.dependencies(deptype=deptypes) + if package_id(d.package) != self.pkg_id) # Handle bootstrapped compiler # # The bootstrapped compiler is not a dependency in the spec, but it is # a dependency of the build task. Here we add it to self.dependencies - compiler_spec = self.spec.compiler - arch_spec = self.spec.architecture + compiler_spec = self.pkg.spec.compiler + arch_spec = self.pkg.spec.architecture if not spack.compilers.compilers_for_spec(compiler_spec, arch_spec=arch_spec): # The compiler is in the queue, identify it as dependency @@ -1751,9 +1802,27 @@ class BuildTask(object): self.uninstalled_deps = set(pkg_id for pkg_id in self.dependencies if pkg_id not in installed) - # Ensure the task gets a unique sequence number to preserve the - # order in which it was added. - self.sequence = next(_counter) + # Ensure key sequence-related properties are updated accordingly. + self.attempts = 0 + self._update() + + def __eq__(self, other): + return self.key == other.key + + def __ge__(self, other): + return self.key >= other.key + + def __gt__(self, other): + return self.key > other.key + + def __le__(self, other): + return self.key <= other.key + + def __lt__(self, other): + return self.key < other.key + + def __ne__(self, other): + return self.key != other.key def __repr__(self): """Returns a formal representation of the build task.""" @@ -1768,6 +1837,28 @@ class BuildTask(object): return ('priority={0}, status={1}, start={2}, {3}' .format(self.priority, self.status, self.start, dependencies)) + def _update(self): + """Update properties associated with a new instance of a task.""" + # Number of times the task has/will be queued + self.attempts = self.attempts + 1 + + # Ensure the task gets a unique sequence number to preserve the + # order in which it is added. + self.sequence = next(_counter) + + def add_dependent(self, pkg_id): + """ + Ensure the dependent package id is in the task's list so it will be + properly updated when this package is installed. + + Args: + pkg_id (str): package identifier of the dependent package + """ + if pkg_id != self.pkg_id and pkg_id not in self.dependents: + tty.debug('Adding {0} as a dependent of {1}' + .format(pkg_id, self.pkg_id)) + self.dependents.add(pkg_id) + def flag_installed(self, installed): """ Ensure the dependency is not considered to still be uninstalled. @@ -1783,20 +1874,162 @@ class BuildTask(object): .format(self.pkg_id, pkg_id, self.uninstalled_deps)) @property + def explicit(self): + """The package was explicitly requested by the user.""" + return self.pkg == self.request.pkg + + @property def key(self): """The key is the tuple (# uninstalled dependencies, sequence).""" return (self.priority, self.sequence) + def next_attempt(self, installed): + """Create a new, updated task for the next installation attempt.""" + task = copy.copy(self) + task._update() + task.start = self.start or time.time() + task.flag_installed(installed) + return task + @property def priority(self): """The priority is based on the remaining uninstalled dependencies.""" return len(self.uninstalled_deps) + +class BuildRequest(object): + """Class for representing an installation request.""" + + def __init__(self, pkg, install_args): + """ + Instantiate a build request for a package. + + Args: + pkg (Package): the package to be built and installed + install_args (dict): the install arguments associated with ``pkg`` + """ + # Ensure dealing with a package that has a concrete spec + if not isinstance(pkg, spack.package.PackageBase): + raise ValueError("{0} must be a package".format(str(pkg))) + + self.pkg = pkg + if not self.pkg.spec.concrete: + raise ValueError("{0} must have a concrete spec" + .format(self.pkg.name)) + + # Cache the package phase options with the explicit package, + # popping the options to ensure installation of associated + # dependencies is NOT affected by these options. + self.pkg.stop_before_phase = install_args.pop('stop_before', None) + self.pkg.last_phase = install_args.pop('stop_at', None) + + # Cache the package id for convenience + self.pkg_id = package_id(pkg) + + # Save off the original install arguments plus standard defaults + # since they apply to the requested package *and* dependencies. + self.install_args = install_args if install_args else {} + self._add_default_args() + + # Cache overwrite information + self.overwrite = set(self.install_args.get('overwrite', [])) + self.overwrite_time = time.time() + + # Save off dependency package ids for quick checks since traversals + # are not able to return full dependents for all packages across + # environment specs. + deptypes = self.get_deptypes(self.pkg) + self.dependencies = set(package_id(d.package) for d in + self.pkg.spec.dependencies(deptype=deptypes) + if package_id(d.package) != self.pkg_id) + + def __repr__(self): + """Returns a formal representation of the build request.""" + rep = '{0}('.format(self.__class__.__name__) + for attr, value in self.__dict__.items(): + rep += '{0}={1}, '.format(attr, value.__repr__()) + return '{0})'.format(rep.strip(', ')) + + def __str__(self): + """Returns a printable version of the build request.""" + return 'package={0}, install_args={1}' \ + .format(self.pkg.name, self.install_args) + + def _add_default_args(self): + """Ensure standard install options are set to at least the default.""" + for arg, default in [('cache_only', False), + ('dirty', False), + ('fail_fast', False), + ('fake', False), + ('full_hash_match', False), + ('install_deps', True), + ('install_package', True), + ('install_source', False), + ('keep_prefix', False), + ('keep_stage', False), + ('restage', False), + ('skip_patch', False), + ('tests', False), + ('unsigned', False), + ('use_cache', True), + ('verbose', False), ]: + _ = self.install_args.setdefault(arg, default) + + def get_deptypes(self, pkg): + """Determine the required dependency types for the associated package. + + Args: + pkg (PackageBase): explicit or implicit package being installed + + Returns: + (tuple) required dependency type(s) for the package + """ + deptypes = ['link', 'run'] + if not self.install_args.get('cache_only'): + deptypes.append('build') + if self.run_tests(pkg): + deptypes.append('test') + return tuple(sorted(deptypes)) + + def has_dependency(self, dep_id): + """Returns ``True`` if the package id represents a known dependency + of the requested package, ``False`` otherwise.""" + return dep_id in self.dependencies + + def run_tests(self, pkg): + """Determine if the tests should be run for the provided packages + + Args: + pkg (PackageBase): explicit or implicit package being installed + + Returns: + (bool) ``True`` if they should be run; ``False`` otherwise + """ + tests = self.install_args.get('tests', False) + return tests is True or (tests and pkg.name in tests) + @property def spec(self): """The specification associated with the package.""" return self.pkg.spec + def traverse_dependencies(self): + """ + Yield any dependencies of the appropriate type(s) + + Yields: + (Spec) The next child spec in the DAG + """ + get_spec = lambda s: s.spec + + deptypes = self.get_deptypes(self.pkg) + tty.debug('Processing dependencies for {0}: {1}' + .format(self.pkg_id, deptypes)) + for dspec in self.spec.traverse_edges( + deptype=deptypes, order='post', root=False, + direction='children'): + yield get_spec(dspec) + class InstallError(spack.error.SpackError): """Raised when something goes wrong during install or uninstall.""" @@ -1805,6 +2038,15 @@ class InstallError(spack.error.SpackError): super(InstallError, self).__init__(message, long_msg) +class BadInstallPhase(InstallError): + """Raised for an install phase option is not allowed for a package.""" + + def __init__(self, pkg_name, phase): + super(BadInstallPhase, self).__init__( + '\'{0}\' is not a valid phase for package {1}' + .format(phase, pkg_name)) + + class ExternalPackageError(InstallError): """Raised by install() when a package is only for external use.""" diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a692d04052..725e1a69d3 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -53,8 +53,7 @@ from six import StringIO from six import string_types from six import with_metaclass from spack.filesystem_view import YamlFilesystemView -from spack.installer import \ - install_args_docstring, PackageInstaller, InstallError +from spack.installer import PackageInstaller, InstallError from spack.stage import stage_prefix, Stage, ResourceStage, StageComposite from spack.util.package_hash import package_hash from spack.version import Version @@ -1591,17 +1590,45 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): Package implementations should override install() to describe their build process. - Args:""" + Args: + cache_only (bool): Fail if binary package unavailable. + dirty (bool): Don't clean the build environment before installing. + explicit (bool): True if package was explicitly installed, False + if package was implicitly installed (as a dependency). + fail_fast (bool): Fail if any dependency fails to install; + otherwise, the default is to install as many dependencies as + possible (i.e., best effort installation). + fake (bool): Don't really build; install fake stub files instead. + force (bool): Install again, even if already installed. + install_deps (bool): Install dependencies before installing this + package + install_source (bool): By default, source is not installed, but + for debugging it might be useful to keep it around. + keep_prefix (bool): Keep install prefix on failure. By default, + destroys it. + keep_stage (bool): By default, stage is destroyed only if there + are no exceptions during build. Set to True to keep the stage + even with exceptions. + restage (bool): Force spack to restage the package source. + skip_patch (bool): Skip patch stage of build if True. + stop_before (InstallPhase): stop execution before this + installation phase (or None) + stop_at (InstallPhase): last installation phase to be executed + (or None) + tests (bool or list or set): False to run no tests, True to test + all packages, or a list of package names to run tests for some + use_cache (bool): Install from binary package, if available. + verbose (bool): Display verbose build output (by default, + suppresses it) + """ # Non-transitive dev specs need to keep the dev stage and be built from # source every time. Transitive ones just need to be built from source. dev_path_var = self.spec.variants.get('dev_path', None) if dev_path_var: kwargs['keep_stage'] = True - builder = PackageInstaller(self) - builder.install(**kwargs) - - do_install.__doc__ += install_args_docstring + builder = PackageInstaller([(self, kwargs)]) + builder.install() def unit_test_check(self): """Hook for unit tests to assert things about package internals. diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py index e500f9fc05..4e5bc9c993 100644 --- a/lib/spack/spack/report.py +++ b/lib/spack/spack/report.py @@ -44,8 +44,8 @@ def fetch_package_log(pkg): class InfoCollector(object): - """Decorates PackageInstaller._install_task, which is called by - PackageBase.do_install for each spec, to collect information + """Decorates PackageInstaller._install_task, which is called via + PackageBase.do_install for individual specs, to collect information on the installation of certain specs. When exiting the context this change will be rolled-back. diff --git a/lib/spack/spack/test/buildrequest.py b/lib/spack/spack/test/buildrequest.py new file mode 100644 index 0000000000..5918838edc --- /dev/null +++ b/lib/spack/spack/test/buildrequest.py @@ -0,0 +1,55 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.installer as inst +import spack.repo +import spack.spec + + +def test_build_request_errors(install_mockery): + with pytest.raises(ValueError, match='must be a package'): + inst.BuildRequest('abc', {}) + + pkg = spack.repo.get('trivial-install-test-package') + with pytest.raises(ValueError, match='must have a concrete spec'): + inst.BuildRequest(pkg, {}) + + +def test_build_request_basics(install_mockery): + spec = spack.spec.Spec('dependent-install') + spec.concretize() + assert spec.concrete + + # Ensure key properties match expectations + request = inst.BuildRequest(spec.package, {}) + assert not request.pkg.stop_before_phase + assert not request.pkg.last_phase + assert request.spec == spec.package.spec + + # Ensure key default install arguments are set + assert 'install_package' in request.install_args + assert 'install_deps' in request.install_args + + +def test_build_request_strings(install_mockery): + """Tests of BuildRequest repr and str for coverage purposes.""" + # Using a package with one dependency + spec = spack.spec.Spec('dependent-install') + spec.concretize() + assert spec.concrete + + # Ensure key properties match expectations + request = inst.BuildRequest(spec.package, {}) + + # Cover __repr__ + irep = request.__repr__() + assert irep.startswith(request.__class__.__name__) + + # Cover __str__ + istr = str(request) + assert "package=dependent-install" in istr + assert "install_args=" in istr diff --git a/lib/spack/spack/test/buildtask.py b/lib/spack/spack/test/buildtask.py index dcf3d29a6f..f82ff6861a 100644 --- a/lib/spack/spack/test/buildtask.py +++ b/lib/spack/spack/test/buildtask.py @@ -12,17 +12,22 @@ import spack.spec def test_build_task_errors(install_mockery): with pytest.raises(ValueError, match='must be a package'): - inst.BuildTask('abc', False, 0, 0, 0, []) + inst.BuildTask('abc', None, False, 0, 0, 0, []) pkg = spack.repo.get('trivial-install-test-package') with pytest.raises(ValueError, match='must have a concrete spec'): - inst.BuildTask(pkg, False, 0, 0, 0, []) + inst.BuildTask(pkg, None, False, 0, 0, 0, []) spec = spack.spec.Spec('trivial-install-test-package') spec.concretize() assert spec.concrete + with pytest.raises(ValueError, match='must have a build request'): + inst.BuildTask(spec.package, None, False, 0, 0, 0, []) + + request = inst.BuildRequest(spec.package, {}) with pytest.raises(inst.InstallError, match='Cannot create a build task'): - inst.BuildTask(spec.package, False, 0, 0, inst.STATUS_REMOVED, []) + inst.BuildTask(spec.package, request, False, 0, 0, inst.STATUS_REMOVED, + []) def test_build_task_basics(install_mockery): @@ -31,7 +36,10 @@ def test_build_task_basics(install_mockery): assert spec.concrete # Ensure key properties match expectations - task = inst.BuildTask(spec.package, False, 0, 0, inst.STATUS_ADDED, []) + request = inst.BuildRequest(spec.package, {}) + task = inst.BuildTask(spec.package, request, False, 0, 0, + inst.STATUS_ADDED, []) + assert task.explicit # package was "explicitly" requested assert task.priority == len(task.uninstalled_deps) assert task.key == (task.priority, task.sequence) @@ -51,7 +59,9 @@ def test_build_task_strings(install_mockery): assert spec.concrete # Ensure key properties match expectations - task = inst.BuildTask(spec.package, False, 0, 0, inst.STATUS_ADDED, []) + request = inst.BuildRequest(spec.package, {}) + task = inst.BuildTask(spec.package, request, False, 0, 0, + inst.STATUS_ADDED, []) # Cover __repr__ irep = task.__repr__() diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py index f11025b7b1..30d99c8562 100644 --- a/lib/spack/spack/test/cmd/dev_build.py +++ b/lib/spack/spack/test/cmd/dev_build.py @@ -8,7 +8,7 @@ import pytest import spack.spec import llnl.util.filesystem as fs import spack.environment as ev -from spack.main import SpackCommand, SpackCommandError +from spack.main import SpackCommand dev_build = SpackCommand('dev-build') install = SpackCommand('install') @@ -99,13 +99,15 @@ def test_dev_build_before_until(tmpdir, mock_packages, install_mockery): dev_build('-u', 'edit', '-b', 'edit', 'dev-build-test-install@0.0.0') - with pytest.raises(SpackCommandError): - dev_build('-u', 'phase_that_does_not_exist', - 'dev-build-test-install@0.0.0') + bad_phase = 'phase_that_does_not_exist' + not_allowed = 'is not a valid phase' + out = dev_build('-u', bad_phase, 'dev-build-test-install@0.0.0') + assert bad_phase in out + assert not_allowed in out - with pytest.raises(SpackCommandError): - dev_build('-b', 'phase_that_does_not_exist', - 'dev-build-test-install@0.0.0') + out = dev_build('-b', bad_phase, 'dev-build-test-install@0.0.0') + assert bad_phase in out + assert not_allowed in out def print_spack_cc(*args): diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 1a6501440d..5fbd5f2c85 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -185,17 +185,53 @@ def test_env_modifications_error_on_activate( assert "Warning: couldn't get environment settings" in err -def test_env_install_same_spec_twice(install_mockery, mock_fetch, capfd): +def test_env_install_same_spec_twice(install_mockery, mock_fetch): env('create', 'test') e = ev.read('test') - with capfd.disabled(): - with e: - # The first installation outputs the package prefix - install('cmake-client') - # The second installation attempt will also update the view - out = install('cmake-client') - assert 'Updating view at' in out + with e: + # The first installation outputs the package prefix, updates the view + out = install('cmake-client') + assert 'Updating view at' in out + + # The second installation reports all packages already installed + out = install('cmake-client') + assert 'already installed' in out + + +def test_env_install_two_specs_same_dep( + install_mockery, mock_fetch, tmpdir, capsys): + """Test installation of two packages that share a dependency with no + connection and the second specifying the dependency as a 'build' + dependency. + """ + path = tmpdir.join('spack.yaml') + + with tmpdir.as_cwd(): + with open(str(path), 'w') as f: + f.write("""\ +env: + specs: + - a + - depb +""") + + env('create', 'test', 'spack.yaml') + + with ev.read('test'): + with capsys.disabled(): + out = install() + + # Ensure both packages reach install phase processing and are installed + out = str(out) + assert 'depb: Executing phase:' in out + assert 'a: Executing phase:' in out + + depb = spack.repo.path.get_pkg_class('depb') + assert depb.installed, 'Expected depb to be installed' + + a = spack.repo.path.get_pkg_class('a') + assert a.installed, 'Expected a to be installed' def test_remove_after_concretize(): @@ -2094,7 +2130,8 @@ def test_cant_install_single_spec_when_concretizing_together(): e.concretization = 'together' with pytest.raises(ev.SpackEnvironmentError, match=r'cannot install'): - e.install('zlib') + e.concretize_and_add('zlib') + e.install_all() def test_duplicate_packages_raise_when_concretizing_together(): diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 2d7571ef4b..0ba8e6a7c6 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -38,7 +38,7 @@ buildcache = SpackCommand('buildcache') def noop_install(monkeypatch): def noop(*args, **kwargs): pass - monkeypatch.setattr(spack.package.PackageBase, 'do_install', noop) + monkeypatch.setattr(spack.installer.PackageInstaller, 'install', noop) def test_install_package_and_dependency( @@ -321,15 +321,19 @@ def test_install_from_file(spec, concretize, error_code, tmpdir): with specfile.open('w') as f: spec.to_yaml(f) + err_msg = 'does not contain a concrete spec' if error_code else '' + # Relative path to specfile (regression for #6906) with fs.working_dir(specfile.dirname): # A non-concrete spec will fail to be installed - install('-f', specfile.basename, fail_on_error=False) + out = install('-f', specfile.basename, fail_on_error=False) assert install.returncode == error_code + assert err_msg in out # Absolute path to specfile (regression for #6983) - install('-f', str(specfile), fail_on_error=False) + out = install('-f', str(specfile), fail_on_error=False) assert install.returncode == error_code + assert err_msg in out @pytest.mark.disable_clean_stage_check @@ -615,12 +619,14 @@ def test_build_warning_output(tmpdir, mock_fetch, install_mockery, capfd): assert 'foo.c:89: warning: some weird warning!' in msg -def test_cache_only_fails(tmpdir, mock_fetch, install_mockery): +def test_cache_only_fails(tmpdir, mock_fetch, install_mockery, capfd): # libelf from cache fails to install, which automatically removes the - # the libdwarf build task and flags the package as failed to install. - err_msg = 'Installation of libdwarf failed' - with pytest.raises(spack.installer.InstallError, match=err_msg): - install('--cache-only', 'libdwarf') + # the libdwarf build task + with capfd.disabled(): + out = install('--cache-only', 'libdwarf') + + assert 'Failed to install libelf' in out + assert 'Skipping build of libdwarf' in out # Check that failure prefix locks are still cached failure_lock_prefixes = ','.join(spack.store.db._prefix_failures.keys()) @@ -828,6 +834,7 @@ def test_cache_install_full_hash_match( mirror_url = 'file://{0}'.format(mirror_dir.strpath) s = Spec('libdwarf').concretized() + package_id = spack.installer.package_id(s.package) # Install a package install(s.name) @@ -843,9 +850,9 @@ def test_cache_install_full_hash_match( # Make sure we get the binary version by default install_output = install('--no-check-signature', s.name, output=str) - expect_msg = 'Extracting {0} from binary cache'.format(s.name) + expect_extract_msg = 'Extracting {0} from binary cache'.format(package_id) - assert expect_msg in install_output + assert expect_extract_msg in install_output uninstall('-y', s.name) @@ -855,9 +862,7 @@ def test_cache_install_full_hash_match( # Check that even if the full hash changes, we install from binary when # we don't explicitly require the full hash to match install_output = install('--no-check-signature', s.name, output=str) - expect_msg = 'Extracting {0} from binary cache'.format(s.name) - - assert expect_msg in install_output + assert expect_extract_msg in install_output uninstall('-y', s.name) @@ -865,7 +870,7 @@ def test_cache_install_full_hash_match( # installs from source. install_output = install('--require-full-hash-match', s.name, output=str) expect_msg = 'No binary for {0} found: installing from source'.format( - s.name) + package_id) assert expect_msg in install_output diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 61475e4dc6..c2a8d0b100 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -507,7 +507,7 @@ def test_unconcretized_install(install_mockery, mock_fetch, mock_packages): """Test attempts to perform install phases with unconcretized spec.""" spec = Spec('trivial-install-test-package') - with pytest.raises(ValueError, match="only install concrete packages"): + with pytest.raises(ValueError, match='must have a concrete spec'): spec.package.do_install() with pytest.raises(ValueError, match="only patch concrete packages"): diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 99f0af709c..cec3721d0d 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -59,34 +59,52 @@ def _true(*args, **kwargs): return True -def create_build_task(pkg): +def create_build_task(pkg, install_args={}): """ Create a built task for the given (concretized) package Args: pkg (PackageBase): concretized package associated with the task + install_args (dict): dictionary of kwargs (or install args) Return: (BuildTask) A basic package build task """ - return inst.BuildTask(pkg, False, 0, 0, inst.STATUS_ADDED, []) + request = inst.BuildRequest(pkg, install_args) + return inst.BuildTask(pkg, request, False, 0, 0, inst.STATUS_ADDED, []) -def create_installer(spec_name): +def create_installer(installer_args): """ - Create an installer for the named spec + Create an installer using the concretized spec for each arg Args: - spec_name (str): Name of the explicit install spec + installer_args (list of tuples): the list of (spec name, kwargs) tuples Return: - spec (Spec): concretized spec installer (PackageInstaller): the associated package installer """ - spec = spack.spec.Spec(spec_name) - spec.concretize() - assert spec.concrete - return spec, inst.PackageInstaller(spec.package) + const_arg = [(spec.package, kwargs) for spec, kwargs in installer_args] + return inst.PackageInstaller(const_arg) + + +def installer_args(spec_names, kwargs={}): + """Return a the installer argument with each spec paired with kwargs + + Args: + spec_names (list of str): list of spec names + kwargs (dict or None): install arguments to apply to all of the specs + + Returns: + list of (spec, kwargs): the installer constructor argument + """ + arg = [] + for name in spec_names: + spec = spack.spec.Spec(name) + spec.concretize() + assert spec.concrete + arg.append((spec, kwargs)) + return arg @pytest.mark.parametrize('sec,result', [ @@ -99,6 +117,21 @@ def test_hms(sec, result): assert inst._hms(sec) == result +def test_get_dependent_ids(install_mockery, mock_packages): + # Concretize the parent package, which handle dependency too + spec = spack.spec.Spec('a') + spec.concretize() + assert spec.concrete + + pkg_id = inst.package_id(spec.package) + + # Grab the sole dependency of 'a', which is 'b' + dep = spec.dependencies()[0] + + # Ensure the parent package is a dependent of the dependency package + assert pkg_id in inst.get_dependent_ids(dep) + + def test_install_msg(monkeypatch): """Test results of call to install_msg based on debug level.""" name = 'some-package' @@ -190,7 +223,9 @@ def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd): spec = spack.spec.Spec('a').concretized() assert inst._process_binary_cache_tarball(spec.package, spec, False, False) - assert 'Extracting a from binary cache' in capfd.readouterr()[0] + out = capfd.readouterr()[0] + assert 'Extracting a' in out + assert 'from binary cache' in out def test_try_install_from_binary_cache(install_mockery, mock_packages, @@ -216,18 +251,9 @@ def test_try_install_from_binary_cache(install_mockery, mock_packages, assert 'add a spack mirror to allow download' in str(captured) -def test_installer_init_errors(install_mockery): - """Test to ensure cover installer constructor errors.""" - with pytest.raises(ValueError, match='must be a package'): - inst.PackageInstaller('abc') - - pkg = spack.repo.get('trivial-install-test-package') - with pytest.raises(ValueError, match='Can only install concrete'): - inst.PackageInstaller(pkg) - - def test_installer_repr(install_mockery): - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) irep = installer.__repr__() assert irep.startswith(installer.__class__.__name__) @@ -236,7 +262,8 @@ def test_installer_repr(install_mockery): def test_installer_str(install_mockery): - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) istr = str(installer) assert "#tasks=0" in istr @@ -244,20 +271,33 @@ def test_installer_str(install_mockery): assert "failed (0)" in istr -def test_installer_last_phase_error(install_mockery, capsys): - spec = spack.spec.Spec('trivial-install-test-package') - spec.concretize() - assert spec.concrete - with pytest.raises(SystemExit): - installer = inst.PackageInstaller(spec.package) - installer.install(stop_at='badphase') +def test_check_before_phase_error(install_mockery): + pkg = spack.repo.get('trivial-install-test-package') + pkg.stop_before_phase = 'beforephase' + with pytest.raises(inst.BadInstallPhase) as exc_info: + inst._check_last_phase(pkg) - captured = capsys.readouterr() - assert 'is not an allowed phase' in str(captured) + err = str(exc_info.value) + assert 'is not a valid phase' in err + assert pkg.stop_before_phase in err + + +def test_check_last_phase_error(install_mockery): + pkg = spack.repo.get('trivial-install-test-package') + pkg.stop_before_phase = None + pkg.last_phase = 'badphase' + with pytest.raises(inst.BadInstallPhase) as exc_info: + inst._check_last_phase(pkg) + + err = str(exc_info.value) + assert 'is not a valid phase' in err + assert pkg.last_phase in err def test_installer_ensure_ready_errors(install_mockery): - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec fmt = r'cannot be installed locally.*{0}' # Force an external package error @@ -290,7 +330,9 @@ def test_ensure_locked_err(install_mockery, monkeypatch, tmpdir, capsys): def _raise(lock, timeout): raise RuntimeError(mock_err_msg) - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec monkeypatch.setattr(ulk.Lock, 'acquire_read', _raise) with tmpdir.as_cwd(): @@ -304,14 +346,17 @@ def test_ensure_locked_err(install_mockery, monkeypatch, tmpdir, capsys): def test_ensure_locked_have(install_mockery, tmpdir, capsys): """Test _ensure_locked when already have lock.""" - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec + pkg_id = inst.package_id(spec.package) with tmpdir.as_cwd(): # Test "downgrade" of a read lock (to a read lock) lock = lk.Lock('./test', default_timeout=1e-9, desc='test') lock_type = 'read' tpl = (lock_type, lock) - installer.locks[installer.pkg_id] = tpl + installer.locks[pkg_id] = tpl assert installer._ensure_locked(lock_type, spec.package) == tpl # Test "upgrade" of a read lock without read count to a write @@ -341,7 +386,9 @@ def test_ensure_locked_have(install_mockery, tmpdir, capsys): def test_ensure_locked_new_lock( install_mockery, tmpdir, lock_type, reads, writes): pkg_id = 'a' - spec, installer = create_installer(pkg_id) + const_arg = installer_args([pkg_id], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec with tmpdir.as_cwd(): ltype, lock = installer._ensure_locked(lock_type, spec.package) assert ltype == lock_type @@ -359,7 +406,9 @@ def test_ensure_locked_new_warn(install_mockery, monkeypatch, tmpdir, capsys): return lock pkg_id = 'a' - spec, installer = create_installer(pkg_id) + const_arg = installer_args([pkg_id], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec monkeypatch.setattr(spack.database.Database, 'prefix_lock', _pl) @@ -529,52 +578,65 @@ def test_clear_failures_errs(install_mockery, monkeypatch, capsys): def test_check_deps_status_install_failure(install_mockery, monkeypatch): - spec, installer = create_installer('a') + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] # Make sure the package is identified as failed monkeypatch.setattr(spack.database.Database, 'prefix_failed', _true) with pytest.raises(inst.InstallError, match='install failure'): - installer._check_deps_status() + installer._check_deps_status(request) def test_check_deps_status_write_locked(install_mockery, monkeypatch): - spec, installer = create_installer('a') + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] # Ensure the lock is not acquired monkeypatch.setattr(inst.PackageInstaller, '_ensure_locked', _not_locked) with pytest.raises(inst.InstallError, match='write locked by another'): - installer._check_deps_status() + installer._check_deps_status(request) def test_check_deps_status_external(install_mockery, monkeypatch): - spec, installer = create_installer('a') + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] # Mock the known dependent, b, as external so assumed to be installed monkeypatch.setattr(spack.spec.Spec, 'external', True) - installer._check_deps_status() - assert 'b' in installer.installed + installer._check_deps_status(request) + assert list(installer.installed)[0].startswith('b') def test_check_deps_status_upstream(install_mockery, monkeypatch): - spec, installer = create_installer('a') + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] # Mock the known dependent, b, as installed upstream monkeypatch.setattr(spack.package.PackageBase, 'installed_upstream', True) - installer._check_deps_status() - assert 'b' in installer.installed + installer._check_deps_status(request) + assert list(installer.installed)[0].startswith('b') def test_add_bootstrap_compilers(install_mockery, monkeypatch): + from collections import defaultdict + def _pkgs(pkg): spec = spack.spec.Spec('mpi').concretized() return [(spec.package, True)] - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] + all_deps = defaultdict(set) monkeypatch.setattr(inst, '_packages_needed_to_bootstrap_compiler', _pkgs) - installer._add_bootstrap_compilers(spec.package) + installer._add_bootstrap_compilers(request.pkg, request, all_deps) ids = list(installer.build_tasks) assert len(ids) == 1 @@ -584,33 +646,40 @@ def test_add_bootstrap_compilers(install_mockery, monkeypatch): def test_prepare_for_install_on_installed(install_mockery, monkeypatch): """Test of _prepare_for_install's early return for installed task path.""" - spec, installer = create_installer('dependent-install') - task = create_build_task(spec.package) + const_arg = installer_args(['dependent-install'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] + + install_args = {'keep_prefix': True, 'keep_stage': True, 'restage': False} + task = create_build_task(request.pkg, install_args) installer.installed.add(task.pkg_id) monkeypatch.setattr(inst.PackageInstaller, '_ensure_install_ready', _noop) - installer._prepare_for_install(task, True, True, False) + installer._prepare_for_install(task) -def test_installer_init_queue(install_mockery): - """Test of installer queue functions.""" +def test_installer_init_requests(install_mockery): + """Test of installer initial requests.""" + spec_name = 'dependent-install' with spack.config.override('config:install_missing_compilers', True): - spec, installer = create_installer('dependent-install') - installer._init_queue(True, True) + const_arg = installer_args([spec_name], {}) + installer = create_installer(const_arg) - ids = list(installer.build_tasks) - assert len(ids) == 2 - assert 'dependency-install' in ids - assert 'dependent-install' in ids + # There is only one explicit request in this case + assert len(installer.build_requests) == 1 + request = installer.build_requests[0] + assert request.pkg.name == spec_name def test_install_task_use_cache(install_mockery, monkeypatch): - spec, installer = create_installer('trivial-install-test-package') - task = create_build_task(spec.package) + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) + request = installer.build_requests[0] + task = create_build_task(request.pkg) monkeypatch.setattr(inst, '_install_from_cache', _true) installer._install_task(task) - assert spec.package.name in installer.installed + assert request.pkg_id in installer.installed def test_install_task_add_compiler(install_mockery, monkeypatch, capfd): @@ -619,8 +688,9 @@ def test_install_task_add_compiler(install_mockery, monkeypatch, capfd): def _add(_compilers): tty.msg(config_msg) - spec, installer = create_installer('a') - task = create_build_task(spec.package) + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + task = create_build_task(installer.build_requests[0].pkg) task.compiler = True # Preclude any meaningful side-effects @@ -638,7 +708,8 @@ def test_install_task_add_compiler(install_mockery, monkeypatch, capfd): def test_release_lock_write_n_exception(install_mockery, tmpdir, capsys): """Test _release_lock for supposed write lock with exception.""" - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) pkg_id = 'test' with tmpdir.as_cwd(): @@ -652,10 +723,30 @@ def test_release_lock_write_n_exception(install_mockery, tmpdir, capsys): assert msg in out +@pytest.mark.parametrize('installed', [True, False]) +def test_push_task_skip_processed(install_mockery, installed): + """Test to ensure skip re-queueing a processed package.""" + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + assert len(list(installer.build_tasks)) == 0 + + # Mark the package as installed OR failed + task = create_build_task(installer.build_requests[0].pkg) + if installed: + installer.installed.add(task.pkg_id) + else: + installer.failed[task.pkg_id] = None + + installer._push_task(task) + + assert len(list(installer.build_tasks)) == 0 + + def test_requeue_task(install_mockery, capfd): """Test to ensure cover _requeue_task.""" - spec, installer = create_installer('a') - task = create_build_task(spec.package) + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + task = create_build_task(installer.build_requests[0].pkg) installer._requeue_task(task) @@ -663,9 +754,12 @@ def test_requeue_task(install_mockery, capfd): assert len(ids) == 1 qtask = installer.build_tasks[ids[0]] assert qtask.status == inst.STATUS_INSTALLING + assert qtask.sequence > task.sequence + assert qtask.attempts == task.attempts + 1 out = capfd.readouterr()[0] - assert 'Installing a in progress by another process' in out + assert 'Installing a' in out + assert ' in progress by another process' in out def test_cleanup_all_tasks(install_mockery, monkeypatch): @@ -676,7 +770,9 @@ def test_cleanup_all_tasks(install_mockery, monkeypatch): def _rmtask(installer, pkg_id): raise RuntimeError('Raise an exception to test except path') - spec, installer = create_installer('a') + const_arg = installer_args(['a'], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec # Cover task removal happy path installer.build_tasks['a'] = _mktask(spec.package) @@ -704,7 +800,9 @@ def test_setup_install_dir_grp(install_mockery, monkeypatch, capfd): monkeypatch.setattr(prefs, 'get_package_group', _get_group) monkeypatch.setattr(fs, 'chgrp', _chgrp) - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec fs.touchp(spec.prefix) metadatadir = spack.store.layout.metadata_path(spec) @@ -725,7 +823,8 @@ def test_cleanup_failed_err(install_mockery, tmpdir, monkeypatch, capsys): def _raise_except(lock): raise RuntimeError(msg) - spec, installer = create_installer('trivial-install-test-package') + const_arg = installer_args(['trivial-install-test-package'], {}) + installer = create_installer(const_arg) monkeypatch.setattr(lk.Lock, 'release_write', _raise_except) pkg_id = 'test' @@ -741,7 +840,9 @@ def test_cleanup_failed_err(install_mockery, tmpdir, monkeypatch, capsys): def test_update_failed_no_dependent_task(install_mockery): """Test _update_failed with missing dependent build tasks.""" - spec, installer = create_installer('dependent-install') + const_arg = installer_args(['dependent-install'], {}) + installer = create_installer(const_arg) + spec = installer.build_requests[0].pkg.spec for dep in spec.traverse(root=False): task = create_build_task(dep.package) @@ -751,7 +852,8 @@ def test_update_failed_no_dependent_task(install_mockery): def test_install_uninstalled_deps(install_mockery, monkeypatch, capsys): """Test install with uninstalled dependencies.""" - spec, installer = create_installer('dependent-install') + const_arg = installer_args(['dependent-install'], {}) + installer = create_installer(const_arg) # Skip the actual installation and any status updates monkeypatch.setattr(inst.PackageInstaller, '_install_task', _noop) @@ -759,7 +861,7 @@ def test_install_uninstalled_deps(install_mockery, monkeypatch, capsys): monkeypatch.setattr(inst.PackageInstaller, '_update_failed', _noop) msg = 'Cannot proceed with dependent-install' - with pytest.raises(spack.installer.InstallError, match=msg): + with pytest.raises(inst.InstallError, match=msg): installer.install() out = str(capsys.readouterr()) @@ -768,30 +870,47 @@ def test_install_uninstalled_deps(install_mockery, monkeypatch, capsys): def test_install_failed(install_mockery, monkeypatch, capsys): """Test install with failed install.""" - spec, installer = create_installer('b') + const_arg = installer_args(['b'], {}) + installer = create_installer(const_arg) # Make sure the package is identified as failed monkeypatch.setattr(spack.database.Database, 'prefix_failed', _true) - # Skip the actual installation though it should never get there - monkeypatch.setattr(inst.PackageInstaller, '_install_task', _noop) + installer.install() - msg = 'Installation of b failed' - with pytest.raises(spack.installer.InstallError, match=msg): - installer.install() + out = str(capsys.readouterr()) + assert installer.build_requests[0].pkg_id in out + assert 'failed to install' in out + + +def test_install_failed_not_fast(install_mockery, monkeypatch, capsys): + """Test install with failed install.""" + const_arg = installer_args(['a'], {'fail_fast': False}) + installer = create_installer(const_arg) + + # Make sure the package is identified as failed + monkeypatch.setattr(spack.database.Database, 'prefix_failed', _true) + + installer.install() out = str(capsys.readouterr()) - assert 'Warning: b failed to install' in out + assert 'failed to install' in out + assert 'Skipping build of a' in out def test_install_fail_on_interrupt(install_mockery, monkeypatch): """Test ctrl-c interrupted install.""" - err_msg = 'mock keyboard interrupt' + spec_name = 'a' + err_msg = 'mock keyboard interrupt for {0}'.format(spec_name) def _interrupt(installer, task, **kwargs): - raise KeyboardInterrupt(err_msg) + if task.pkg.name == spec_name: + raise KeyboardInterrupt(err_msg) + else: + installer.installed.add(task.pkg.name) - spec, installer = create_installer('a') + const_arg = installer_args([spec_name], {}) + installer = create_installer(const_arg) # Raise a KeyboardInterrupt error to trigger early termination monkeypatch.setattr(inst.PackageInstaller, '_install_task', _interrupt) @@ -799,41 +918,112 @@ def test_install_fail_on_interrupt(install_mockery, monkeypatch): with pytest.raises(KeyboardInterrupt, match=err_msg): installer.install() + assert 'b' in installer.installed # ensure dependency of a is 'installed' + assert spec_name not in installer.installed + + +def test_install_fail_single(install_mockery, monkeypatch): + """Test expected results for failure of single package.""" + spec_name = 'a' + err_msg = 'mock internal package build error for {0}'.format(spec_name) + + class MyBuildException(Exception): + pass + + def _install(installer, task, **kwargs): + if task.pkg.name == spec_name: + raise MyBuildException(err_msg) + else: + installer.installed.add(task.pkg.name) + + const_arg = installer_args([spec_name], {}) + installer = create_installer(const_arg) + + # Raise a KeyboardInterrupt error to trigger early termination + monkeypatch.setattr(inst.PackageInstaller, '_install_task', _install) + + with pytest.raises(MyBuildException, match=err_msg): + installer.install() + + assert 'b' in installer.installed # ensure dependency of a is 'installed' + assert spec_name not in installer.installed + + +def test_install_fail_multi(install_mockery, monkeypatch): + """Test expected results for failure of multiple packages.""" + spec_name = 'c' + err_msg = 'mock internal package build error' + + class MyBuildException(Exception): + pass + + def _install(installer, task, **kwargs): + if task.pkg.name == spec_name: + raise MyBuildException(err_msg) + else: + installer.installed.add(task.pkg.name) + + const_arg = installer_args([spec_name, 'a'], {}) + installer = create_installer(const_arg) + + # Raise a KeyboardInterrupt error to trigger early termination + monkeypatch.setattr(inst.PackageInstaller, '_install_task', _install) + + with pytest.raises(inst.InstallError, match='Installation request failed'): + installer.install() + + assert 'a' in installer.installed # ensure the the second spec installed + assert spec_name not in installer.installed + def test_install_fail_fast_on_detect(install_mockery, monkeypatch, capsys): """Test fail_fast install when an install failure is detected.""" - spec, installer = create_installer('a') + const_arg = installer_args(['b'], {'fail_fast': False}) + const_arg.extend(installer_args(['c'], {'fail_fast': True})) + installer = create_installer(const_arg) + pkg_ids = [inst.package_id(spec.package) for spec, _ in const_arg] - # Make sure the package is identified as failed + # Make sure all packages are identified as failed # # This will prevent b from installing, which will cause the build of a # to be skipped. monkeypatch.setattr(spack.database.Database, 'prefix_failed', _true) - with pytest.raises(spack.installer.InstallError): - installer.install(fail_fast=True) + with pytest.raises(inst.InstallError, match='after first install failure'): + installer.install() + + assert pkg_ids[0] in installer.failed, 'Expected b to be marked as failed' + assert pkg_ids[1] not in installer.failed, \ + 'Expected no attempt to install c' - out = str(capsys.readouterr()) - assert 'Skipping build of a' in out + out = capsys.readouterr()[1] + assert '{0} failed to install'.format(pkg_ids[0]) in out -def test_install_fail_fast_on_except(install_mockery, monkeypatch, capsys): - """Test fail_fast install when an install failure results from an error.""" - err_msg = 'mock patch failure' +def _test_install_fail_fast_on_except_patch(installer, **kwargs): + """Helper for test_install_fail_fast_on_except.""" + # This is a module-scope function and not a local function because it + # needs to be pickleable. + raise RuntimeError('mock patch failure') - def _patch(installer, task, **kwargs): - raise RuntimeError(err_msg) - spec, installer = create_installer('a') +def test_install_fail_fast_on_except(install_mockery, monkeypatch, capsys): + """Test fail_fast install when an install failure results from an error.""" + const_arg = installer_args(['a'], {'fail_fast': True}) + installer = create_installer(const_arg) # Raise a non-KeyboardInterrupt exception to trigger fast failure. # # This will prevent b from installing, which will cause the build of a # to be skipped. - monkeypatch.setattr(spack.package.PackageBase, 'do_patch', _patch) + monkeypatch.setattr( + spack.package.PackageBase, + 'do_patch', + _test_install_fail_fast_on_except_patch + ) - with pytest.raises(spack.installer.InstallError, matches=err_msg): - installer.install(fail_fast=True) + with pytest.raises(inst.InstallError, match='mock patch failure'): + installer.install() out = str(capsys.readouterr()) assert 'Skipping build of a' in out @@ -844,7 +1034,8 @@ def test_install_lock_failures(install_mockery, monkeypatch, capfd): def _requeued(installer, task): tty.msg('requeued {0}' .format(task.pkg.spec.name)) - spec, installer = create_installer('b') + const_arg = installer_args(['b'], {}) + installer = create_installer(const_arg) # Ensure never acquire a lock monkeypatch.setattr(inst.PackageInstaller, '_ensure_locked', _not_locked) @@ -852,9 +1043,6 @@ def test_install_lock_failures(install_mockery, monkeypatch, capfd): # Ensure don't continually requeue the task monkeypatch.setattr(inst.PackageInstaller, '_requeue_task', _requeued) - # Skip the actual installation though should never reach it - monkeypatch.setattr(inst.PackageInstaller, '_install_task', _noop) - installer.install() out = capfd.readouterr()[0] expected = ['write locked', 'read locked', 'requeued'] @@ -864,22 +1052,21 @@ def test_install_lock_failures(install_mockery, monkeypatch, capfd): def test_install_lock_installed_requeue(install_mockery, monkeypatch, capfd): """Cover basic install handling for installed package.""" - def _install(installer, task, **kwargs): - tty.msg('{0} installing'.format(task.pkg.spec.name)) + const_arg = installer_args(['b'], {}) + b, _ = const_arg[0] + installer = create_installer(const_arg) + b_pkg_id = inst.package_id(b.package) - def _prep(installer, task, keep_prefix, keep_stage, restage): - installer.installed.add('b') - tty.msg('{0} is installed' .format(task.pkg.spec.name)) + def _prep(installer, task): + installer.installed.add(b_pkg_id) + tty.msg('{0} is installed' .format(b_pkg_id)) # also do not allow the package to be locked again monkeypatch.setattr(inst.PackageInstaller, '_ensure_locked', _not_locked) def _requeued(installer, task): - tty.msg('requeued {0}' .format(task.pkg.spec.name)) - - # Skip the actual installation though should never reach it - monkeypatch.setattr(inst.PackageInstaller, '_install_task', _install) + tty.msg('requeued {0}' .format(inst.package_id(task.pkg))) # Flag the package as installed monkeypatch.setattr(inst.PackageInstaller, '_prepare_for_install', _prep) @@ -887,10 +1074,8 @@ def test_install_lock_installed_requeue(install_mockery, monkeypatch, capfd): # Ensure don't continually requeue the task monkeypatch.setattr(inst.PackageInstaller, '_requeue_task', _requeued) - spec, installer = create_installer('b') - installer.install() - assert 'b' not in installer.installed + assert b_pkg_id not in installer.installed out = capfd.readouterr()[0] expected = ['is installed', 'read locked', 'requeued'] @@ -902,14 +1087,11 @@ def test_install_read_locked_requeue(install_mockery, monkeypatch, capfd): """Cover basic read lock handling for uninstalled package with requeue.""" orig_fn = inst.PackageInstaller._ensure_locked - def _install(installer, task, **kwargs): - tty.msg('{0} installing'.format(task.pkg.spec.name)) - def _read(installer, lock_type, pkg): tty.msg('{0}->read locked {1}' .format(lock_type, pkg.spec.name)) return orig_fn(installer, 'read', pkg) - def _prep(installer, task, keep_prefix, keep_stage, restage): + def _prep(installer, task): tty.msg('preparing {0}' .format(task.pkg.spec.name)) assert task.pkg.spec.name not in installer.installed @@ -919,16 +1101,14 @@ def test_install_read_locked_requeue(install_mockery, monkeypatch, capfd): # Force a read lock monkeypatch.setattr(inst.PackageInstaller, '_ensure_locked', _read) - # Skip the actual installation though should never reach it - monkeypatch.setattr(inst.PackageInstaller, '_install_task', _install) - # Flag the package as installed monkeypatch.setattr(inst.PackageInstaller, '_prepare_for_install', _prep) # Ensure don't continually requeue the task monkeypatch.setattr(inst.PackageInstaller, '_requeue_task', _requeued) - spec, installer = create_installer('b') + const_arg = installer_args(['b'], {}) + installer = create_installer(const_arg) installer.install() assert 'b' not in installer.installed @@ -939,28 +1119,55 @@ def test_install_read_locked_requeue(install_mockery, monkeypatch, capfd): assert exp in ln -def test_install_dir_exists(install_mockery, monkeypatch, capfd): +def test_install_dir_exists(install_mockery, monkeypatch): """Cover capture of install directory exists error.""" - err = 'Mock directory exists error' + def _install(installer, task): + raise dl.InstallDirectoryAlreadyExistsError(task.pkg.prefix) + + # Ensure raise the desired exception + monkeypatch.setattr(inst.PackageInstaller, '_install_task', _install) + + const_arg = installer_args(['b'], {}) + installer = create_installer(const_arg) + + err = 'already exists' + with pytest.raises(dl.InstallDirectoryAlreadyExistsError, match=err): + installer.install() + + b, _ = const_arg[0] + assert inst.package_id(b.package) in installer.installed - def _install(installer, task, **kwargs): - raise dl.InstallDirectoryAlreadyExistsError(err) + +def test_install_dir_exists_multi(install_mockery, monkeypatch, capfd): + """Cover capture of install directory exists error for multiple specs.""" + def _install(installer, task): + raise dl.InstallDirectoryAlreadyExistsError(task.pkg.prefix) # Skip the actual installation though should never reach it monkeypatch.setattr(inst.PackageInstaller, '_install_task', _install) - spec, installer = create_installer('b') + # Use two packages to ensure multiple specs + const_arg = installer_args(['b', 'c'], {}) + installer = create_installer(const_arg) - with pytest.raises(dl.InstallDirectoryAlreadyExistsError, match=err): + with pytest.raises(inst.InstallError, match='Installation request failed'): installer.install() - assert 'b' in installer.installed + err = capfd.readouterr()[1] + assert 'already exists' in err + for spec, install_args in const_arg: + pkg_id = inst.package_id(spec.package) + assert pkg_id in installer.installed def test_install_skip_patch(install_mockery, mock_fetch): """Test the path skip_patch install path.""" - spec, installer = create_installer('b') + spec_name = 'b' + const_arg = installer_args([spec_name], + {'fake': False, 'skip_patch': True}) + installer = create_installer(const_arg) - installer.install(fake=False, skip_patch=True) + installer.install() - assert 'b' in installer.installed + spec, install_args = const_arg[0] + assert inst.package_id(spec.package) in installer.installed diff --git a/var/spack/repos/builtin.mock/packages/depb/package.py b/var/spack/repos/builtin.mock/packages/depb/package.py new file mode 100644 index 0000000000..4af71848d3 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/depb/package.py @@ -0,0 +1,22 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Depb(AutotoolsPackage): + """Simple package with one build dependency""" + + homepage = "http://www.example.com" + url = "http://www.example.com/a-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + depends_on('b') + + def install(self, spec, prefix): + # sanity_check_prefix requires something in the install directory + # Test requires overriding the one provided by `AutotoolsPackage` + mkdirp(prefix.bin) |