summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/basic_usage.rst37
-rw-r--r--lib/spack/docs/mirrors.rst13
-rw-r--r--lib/spack/docs/packaging_guide.rst65
-rw-r--r--lib/spack/docs/site_configuration.rst151
-rw-r--r--lib/spack/spack/__init__.py14
-rw-r--r--lib/spack/spack/abi.py128
-rw-r--r--lib/spack/spack/concretize.py242
-rw-r--r--lib/spack/spack/config.py67
-rw-r--r--lib/spack/spack/directory_layout.py4
-rw-r--r--lib/spack/spack/package.py5
-rw-r--r--lib/spack/spack/preferred_packages.py175
-rw-r--r--lib/spack/spack/spec.py127
-rw-r--r--lib/spack/spack/test/concretize.py92
-rw-r--r--lib/spack/spack/test/directory_layout.py6
-rw-r--r--lib/spack/spack/test/mock_packages_test.py20
15 files changed, 966 insertions, 180 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index f94ac3d2ba..accf09cc2a 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to
where the compiler is installed. For example::
$ spack compiler add /usr/local/tools/ic-13.0.079
- ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+ ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
intel@13.0.079
Or you can run ``spack compiler add`` with no arguments to force
@@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your
$ module load gcc-4.9.0
$ spack compiler add
- ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+ ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
gcc@4.9.0
This loads the environment module for gcc-4.9.0 to get it into the
@@ -398,27 +398,34 @@ Manual compiler configuration
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If auto-detection fails, you can manually configure a compiler by
-editing your ``~/.spackconfig`` file. You can do this by running
-``spack config edit``, which will open the file in your ``$EDITOR``.
+editing your ``~/.spack/compilers.yaml`` file. You can do this by running
+``spack config edit compilers``, which will open the file in your ``$EDITOR``.
Each compiler configuration in the file looks like this::
...
- [compiler "intel@15.0.0"]
- cc = /usr/local/bin/icc-15.0.024-beta
- cxx = /usr/local/bin/icpc-15.0.024-beta
- f77 = /usr/local/bin/ifort-15.0.024-beta
- fc = /usr/local/bin/ifort-15.0.024-beta
- ...
+ chaos_5_x86_64_ib:
+ ...
+ intel@15.0.0:
+ cc: /usr/local/bin/icc-15.0.024-beta
+ cxx: /usr/local/bin/icpc-15.0.024-beta
+ f77: /usr/local/bin/ifort-15.0.024-beta
+ fc: /usr/local/bin/ifort-15.0.024-beta
+ ...
+
+The chaos_5_x86_64_ib string is an architecture string, and multiple
+compilers can be listed underneath an architecture. The architecture
+string may be replaced with the string 'all' to signify compilers that
+work on all architectures.
For compilers, like ``clang``, that do not support Fortran, put
``None`` for ``f77`` and ``fc``::
- [compiler "clang@3.3svn"]
- cc = /usr/bin/clang
- cxx = /usr/bin/clang++
- f77 = None
- fc = None
+ clang@3.3svn:
+ cc: /usr/bin/clang
+ cxx: /usr/bin/clang++
+ f77: None
+ fc: None
Once you save the file, the configured compilers will show up in the
list displayed by ``spack compilers``.
diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst
index d732a3dd54..b20fedb55f 100644
--- a/lib/spack/docs/mirrors.rst
+++ b/lib/spack/docs/mirrors.rst
@@ -186,7 +186,7 @@ Each mirror has a name so that you can refer to it again later.
``spack mirror list``
----------------------------
-If you want to see all the mirrors Spack knows about you can run ``spack mirror list``::
+To see all the mirrors Spack knows about, run ``spack mirror list``::
$ spack mirror list
local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24
@@ -196,7 +196,7 @@ If you want to see all the mirrors Spack knows about you can run ``spack mirror
``spack mirror remove``
----------------------------
-And, if you want to remove a mirror, just remove it by name::
+To remove a mirror by name::
$ spack mirror remove local_filesystem
$ spack mirror list
@@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name::
Mirror precedence
----------------------------
-Adding a mirror really just adds a section in ``~/.spackconfig``::
+Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``::
- [mirror "local_filesystem"]
- url = file:///Users/gamblin2/spack-mirror-2014-06-24
- [mirror "remote_server"]
- url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
+ mirrors:
+ local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24
+ remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
If you want to change the order in which mirrors are searched for
packages, you can edit this file and reorder the sections. Spack will
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index b7e0b6a4f3..ef9fd89b62 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -661,7 +661,7 @@ Default
revision instead.
Revisions
- Add ``hg`` and ``revision``parameters:
+ Add ``hg`` and ``revision`` parameters:
.. code-block:: python
@@ -1553,6 +1553,69 @@ This is useful when you want to know exactly what Spack will do when
you ask for a particular spec.
+``Concretization Policies``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A user may have certain perferrences for how packages should
+be concretized on their system. For example, one user may prefer packages
+built with OpenMPI and the Intel compiler. Another user may prefer
+packages be built with MVAPICH and GCC.
+
+Spack can be configurated to prefer certain compilers, package
+versions, depends_on, and variants during concretization.
+The preferred configuration can be controlled via the
+``~/.spack/packages.yaml`` file for user configuations, or the
+``etc/spack/packages.yaml`` site configuration.
+
+
+Here's an example packages.yaml file that sets preferred packages:
+
+.. code-block:: sh
+
+ packages:
+ dyninst:
+ compiler: [gcc@4.9]
+ variants: +debug
+ gperftools:
+ version: [2.2, 2.4, 2.3]
+ all:
+ compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi]
+ providers:
+ mpi: [mvapich, mpich, openmpi]
+
+
+At a high level, this example is specifying how packages should be
+concretized. The dyninst package should prefer using gcc 4.9 and
+be built with debug options. The gperftools package should prefer version
+2.2 over 2.4. Every package on the system should prefer mvapich for
+its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by perfering gcc 4.9).
+These options are used to fill in implicit defaults. Any of them can be overwritten
+on the command line if explicitly requested.
+
+Each packages.yaml file begin with the string ``packages:`` and
+package names are specified on the next level. The special string ``all``
+applies settings to each package. Underneath each package name is
+one or more components: ``compiler``, ``variants``, ``version``,
+or ``providers``. Each component has an ordered list of spec
+``constraints``, with earlier entries in the list being prefered over
+later entries.
+
+Sometimes a package installation may have constraints that forbid
+the first concretization rule, in which case Spack will use the first
+legal concretization rule. Going back to the example, if a user
+requests gperftools 2.3 or later, then Spack will install version 2.4
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always
+take preference over unlisted concretizations. In the above example,
+xlc isn't listed in the compiler list. Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``provider`` section differs slightly from other
+concretization rules. A provider lists a value that packages may
+``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
+dependency.
+
.. _install-method:
Implementing the ``install`` method
diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index b03df29573..76fe8fdd7d 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -54,88 +54,75 @@ more elements to the list to indicate where your own site's temporary
directory is.
-.. _concretization-policies:
-
-Concretization policies
-----------------------------
-
-When a user asks for a package like ``mpileaks`` to be installed,
-Spack has to make decisions like what version should be installed,
-what compiler to use, and how its dependencies should be configured.
-This process is called *concretization*, and it's covered in detail in
-:ref:`its own section <abstract-and-concrete>`.
-
-The default concretization policies are in the
-:py:mod:`spack.concretize` module, specifically in the
-:py:class:`spack.concretize.DefaultConcretizer` class. These are the
-important methods used in the concretization process:
-
-* :py:meth:`concretize_version(self, spec) <spack.concretize.DefaultConcretizer.concretize_version>`
-* :py:meth:`concretize_architecture(self, spec) <spack.concretize.DefaultConcretizer.concretize_architecture>`
-* :py:meth:`concretize_compiler(self, spec) <spack.concretize.DefaultConcretizer.concretize_compiler>`
-* :py:meth:`choose_provider(self, spec, providers) <spack.concretize.DefaultConcretizer.choose_provider>`
-
-The first three take a :py:class:`Spec <spack.spec.Spec>` object and
-modify it by adding constraints for the version. For example, if the
-input spec had a version range like `1.0:5.0.3`, then the
-``concretize_version`` method should set the spec's version to a
-*single* version in that range. Likewise, ``concretize_architecture``
-selects an architecture when the input spec does not have one, and
-``concretize_compiler`` needs to set both a concrete compiler and a
-concrete compiler version.
-
-``choose_provider()`` affects how concrete implementations are chosen
-based on a virtual dependency spec. The input spec is some virtual
-dependency and the ``providers`` index is a :py:class:`ProviderIndex
-<spack.packages.ProviderIndex>` object. The ``ProviderIndex`` maps
-the virtual spec to specs for possible implementations, and
-``choose_provider()`` should simply choose one of these. The
-``concretize_*`` methods will be called on the chosen implementation
-later, so there is no need to fully concretize the spec when returning
-it.
-
-The ``DefaultConcretizer`` is intended to provide sensible defaults
-for each policy, but there are certain choices that it can't know
-about. For example, one site might prefer ``OpenMPI`` over ``MPICH``,
-or another might prefer an old version of some packages. These types
-of special cases can be integrated with custom concretizers.
-
-Writing a custom concretizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To write your own concretizer, you need only subclass
-``DefaultConcretizer`` and override the methods you want to change.
-For example, you might write a class like this to change *only* the
-``concretize_version()`` behavior:
-
-.. code-block:: python
-
- from spack.concretize import DefaultConcretizer
-
- class MyConcretizer(DefaultConcretizer):
- def concretize_version(self, spec):
- # implement custom logic here.
-
-Once you have written your custom concretizer, you can make Spack use
-it by editing ``globals.py``. Find this part of the file:
-
-.. code-block:: python
-
- #
- # This controls how things are concretized in spack.
- # Replace it with a subclass if you want different
- # policies.
- #
- concretizer = DefaultConcretizer()
-
-Set concretizer to *your own* class instead of the default:
-
-.. code-block:: python
-
- concretizer = MyConcretizer()
-
-The next time you run Spack, your changes should take effect.
-
+External Packages
+~~~~~~~~~~~~~~~~~~~~~
+Spack can be configured to use externally-installed
+packages rather than building its own packages. This may be desirable
+if machines ship with system packages, such as a customized MPI
+that should be used instead of Spack building its own MPI.
+
+External packages are configured through the ``packages.yaml`` file found
+in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/``
+directory. Here's an example of an external configuration:
+
+.. code-block:: yaml
+
+ packages:
+ openmpi:
+ paths:
+ openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+ openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+ openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
+
+This example lists three installations of OpenMPI, one built with gcc,
+one built with gcc and debug information, and another built with Intel.
+If Spack is asked to build a package that uses one of these MPIs as a
+dependency, it will use the the pre-installed OpenMPI in
+the given directory. This example also specifies that Spack should never
+build its own OpenMPI via the ``nobuild: True`` option.
+
+Each ``packages.yaml`` begins with a ``packages:`` token, followed
+by a list of package names. To specify externals, add a ``paths``
+token under the package name, which lists externals in a
+``spec : /path`` format. Each spec should be as
+well-defined as reasonably possible. If a
+package lacks a spec component, such as missing a compiler or
+package version, then Spack will guess the missing component based
+on its most-favored packages, and it may guess incorrectly.
+
+Each package version and compilers listed in an external should
+have entries in Spack's packages and compiler configuration, even
+though the package and compiler may not every be built.
+
+The packages configuration can tell Spack to use an external location
+for certain package versions, but it does not restrict Spack to using
+external packages. In the above example, if an OpenMPI 1.8.4 became
+available Spack may choose to start building and linking with that version
+rather than continue using the pre-installed OpenMPI versions.
+
+To prevent this, the ``packages.yaml`` configuration also allows packages
+to be flagged as non-buildable. The previous example could be modified to
+be:
+
+.. code-block:: yaml
+
+ packages:
+ openmpi:
+ paths:
+ openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+ openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+ openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
+ nobuild: True
+
+The addition of the ``nobuild`` flag tells Spack that it should never build
+its own version of OpenMPI, and it will instead always rely on a pre-built
+OpenMPI. Similar to ``paths``, ``nobuild`` is specified as a property under
+a package name.
+
+The ``nobuild`` does not need to be paired with external packages.
+It could also be used alone to forbid packages that may be
+buggy or otherwise undesirable.
+
Profiling
~~~~~~~~~~~~~~~~~~~~~
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index ab78ecef30..3051d3f742 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -82,6 +82,20 @@ from spack.directory_layout import YamlDirectoryLayout
install_layout = YamlDirectoryLayout(install_path)
#
+# This controls how packages are sorted when trying to choose
+# the most preferred package. More preferred packages are sorted
+# first.
+#
+from spack.preferred_packages import PreferredPackages
+pkgsort = PreferredPackages()
+
+#
+# This tests ABI compatibility between packages
+#
+from spack.abi import ABI
+abi = ABI()
+
+#
# This controls how things are concretized in spack.
# Replace it with a subclass if you want different
# policies.
diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py
new file mode 100644
index 0000000000..7e565bcbf9
--- /dev/null
+++ b/lib/spack/spack/abi.py
@@ -0,0 +1,128 @@
+##############################################################################
+# Copyright (c) 2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import os
+import spack
+import spack.spec
+from spack.spec import CompilerSpec
+from spack.util.executable import Executable, ProcessError
+from llnl.util.lang import memoized
+
+class ABI(object):
+ """This class provides methods to test ABI compatibility between specs.
+ The current implementation is rather rough and could be improved."""
+
+ def architecture_compatible(self, parent, child):
+ """Returns true iff the parent and child specs have ABI compatible architectures."""
+ return not parent.architecture or not child.architecture or parent.architecture == child.architecture
+
+
+ @memoized
+ def _gcc_get_libstdcxx_version(self, version):
+ """Returns gcc ABI compatibility info by getting the library version of
+ a compiler's libstdc++.so or libgcc_s.so"""
+ spec = CompilerSpec("gcc", version)
+ compilers = spack.compilers.compilers_for_spec(spec)
+ if not compilers:
+ return None
+ compiler = compilers[0]
+ rungcc = None
+ libname = None
+ output = None
+ if compiler.cxx:
+ rungcc = Executable(compiler.cxx)
+ libname = "libstdc++.so"
+ elif compiler.cc:
+ rungcc = Executable(compiler.cc)
+ libname = "libgcc_s.so"
+ else:
+ return None
+ try:
+ output = rungcc("--print-file-name=%s" % libname, return_output=True)
+ except ProcessError, e:
+ return None
+ if not output:
+ return None
+ libpath = os.readlink(output.strip())
+ if not libpath:
+ return None
+ return os.path.basename(libpath)
+
+
+ @memoized
+ def _gcc_compiler_compare(self, pversion, cversion):
+ """Returns true iff the gcc version pversion and cversion
+ are ABI compatible."""
+ plib = self._gcc_get_libstdcxx_version(pversion)
+ clib = self._gcc_get_libstdcxx_version(cversion)
+ if not plib or not clib:
+ return False
+ return plib == clib
+
+
+ def _intel_compiler_compare(self, pversion, cversion):
+ """Returns true iff the intel version pversion and cversion
+ are ABI compatible"""
+
+ # Test major and minor versions. Ignore build version.
+ if (len(pversion.version) < 2 or len(cversion.version) < 2):
+ return False
+ return pversion.version[:2] == cversion.version[:2]
+
+
+ def compiler_compatible(self, parent, child, **kwargs):
+ """Returns true iff the compilers for parent and child specs are ABI compatible"""
+ if not parent.compiler or not child.compiler:
+ return True
+
+ if parent.compiler.name != child.compiler.name:
+ # Different compiler families are assumed ABI incompatible
+ return False
+
+ if kwargs.get('loose', False):
+ return True
+
+ # TODO: Can we move the specialized ABI matching stuff
+ # TODO: into compiler classes?
+ for pversion in parent.compiler.versions:
+ for cversion in child.compiler.versions:
+ # For a few compilers use specialized comparisons. Otherwise
+ # match on version match.
+ if pversion.satisfies(cversion):
+ return True
+ elif (parent.compiler.name == "gcc" and
+ self._gcc_compiler_compare(pversion, cversion)):
+ return True
+ elif (parent.compiler.name == "intel" and
+ self._intel_compiler_compare(pversion, cversion)):
+ return True
+ return False
+
+
+ def compatible(self, parent, child, **kwargs):
+ """Returns true iff a parent and child spec are ABI compatible"""
+ loosematch = kwargs.get('loose', False)
+ return self.architecture_compatible(parent, child) and \
+ self.compiler_compatible(parent, child, loose=loosematch)
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 85cdb202d5..bad67c34e3 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -33,12 +33,16 @@ or user preferences.
TODO: make this customizable and allow users to configure
concretization policies.
"""
+import spack
import spack.spec
import spack.compilers
import spack.architecture
import spack.error
from spack.version import *
-
+from functools import partial
+from spec import DependencyMap
+from itertools import chain
+from spack.config import *
class DefaultConcretizer(object):
"""This class doesn't have any state, it just provides some methods for
@@ -46,10 +50,113 @@ class DefaultConcretizer(object):
default concretization strategies, or you can override all of them.
"""
+ def _valid_virtuals_and_externals(self, spec):
+ """Returns a list of spec/external-path pairs for both virtuals and externals
+ that can concretize this spec."""
+ # Get a list of candidate packages that could satisfy this spec
+ packages = []
+ if spec.virtual:
+ providers = spack.repo.providers_for(spec)
+ if not providers:
+ raise UnsatisfiableProviderSpecError(providers[0], spec)
+ spec_w_preferred_providers = find_spec(
+ spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name))
+ if not spec_w_preferred_providers:
+ spec_w_preferred_providers = spec
+ provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
+ packages = sorted(providers, cmp=provider_cmp)
+ else:
+ packages = [spec]
+
+ # For each candidate package, if it has externals add those to the candidates
+ # if it's a nobuild, then only add the externals.
+ result = []
+ all_compilers = spack.compilers.all_compilers()
+ for pkg in packages:
+ externals = spec_externals(pkg)
+ buildable = not is_spec_nobuild(pkg)
+ if buildable:
+ result.append((pkg, None))
+ for ext in externals:
+ if ext[0].satisfies(spec):
+ result.append(ext)
+ if not result:
+ raise NoBuildError(spec)
+
+ def cmp_externals(a, b):
+ result = cmp_specs(a[0], b[0])
+ if result != 0:
+ return result
+ if not a[1] and b[1]:
+ return 1
+ if not b[1] and a[1]:
+ return -1
+ return cmp_specs(a[1], b[1])
+
+ result = sorted(result, cmp=cmp_externals)
+ return result
+
+
+ def concretize_virtual_and_external(self, spec):
+ """From a list of candidate virtual and external packages, concretize to one that
+ is ABI compatible with the rest of the DAG."""
+ candidates = self._valid_virtuals_and_externals(spec)
+ if not candidates:
+ return False
+
+ # Find the nearest spec in the dag that has a compiler. We'll use that
+ # spec to test compiler compatibility.
+ other_spec = find_spec(spec, lambda(x): x.compiler)
+ if not other_spec:
+ other_spec = spec.root
+
+ # Choose an ABI-compatible candidate, or the first match otherwise.
+ candidate = None
+ if other_spec:
+ candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None)
+ if not candidate:
+ # Try a looser ABI matching
+ candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None)
+ if not candidate:
+ # No ABI matches. Pick the top choice based on the orignal preferences.
+ candidate = candidates[0]
+ candidate_spec = candidate[0]
+ external = candidate[1]
+ changed = False
+
+ # If we're external then trim the dependencies
+ if external:
+ if (spec.dependencies):
+ changed = True
+ spec.dependencies = DependencyMap()
+ candidate_spec.dependencies = DependencyMap()
+
+ def fequal(candidate_field, spec_field):
+ return (not candidate_field) or (candidate_field == spec_field)
+ if (fequal(candidate_spec.name, spec.name) and
+ fequal(candidate_spec.versions, spec.versions) and
+ fequal(candidate_spec.compiler, spec.compiler) and
+ fequal(candidate_spec.architecture, spec.architecture) and
+ fequal(candidate_spec.dependencies, spec.dependencies) and
+ fequal(candidate_spec.variants, spec.variants) and
+ fequal(external, spec.external)):
+ return changed
+
+ # Refine this spec to the candidate.
+ if spec.virtual:
+ spec._replace_with(candidate_spec)
+ changed = True
+ if spec._dup(candidate_spec, deps=False, cleardeps=False):
+ changed = True
+ spec.external = external
+
+ return changed
+
+
def concretize_version(self, spec):
"""If the spec is already concrete, return. Otherwise take
- the most recent available version, and default to the package's
- version if there are no avaialble versions.
+ the preferred version from spackconfig, and default to the package's
+ version if there are no available versions.
TODO: In many cases we probably want to look for installed
versions of each package and use an installed version
@@ -67,20 +174,14 @@ class DefaultConcretizer(object):
# If there are known available versions, return the most recent
# version that satisfies the spec
pkg = spec.package
-
- # Key function to sort versions first by whether they were
- # marked `preferred=True`, then by most recent.
- def preferred_key(v):
- prefer = pkg.versions[v].get('preferred', False)
- return (prefer, v)
-
+ cmp_versions = partial(spack.pkgsort.version_compare, spec.name)
valid_versions = sorted(
[v for v in pkg.versions
if any(v.satisfies(sv) for sv in spec.versions)],
- key=preferred_key)
+ cmp=cmp_versions)
if valid_versions:
- spec.versions = ver([valid_versions[-1]])
+ spec.versions = ver([valid_versions[0]])
else:
# We don't know of any SAFE versions that match the given
# spec. Grab the spec's versions and grab the highest
@@ -145,10 +246,10 @@ class DefaultConcretizer(object):
"""If the spec already has a compiler, we're done. If not, then take
the compiler used for the nearest ancestor with a compiler
spec and use that. If the ancestor's compiler is not
- concrete, then give it a valid version. If there is no
- ancestor with a compiler, use the system default compiler.
+ concrete, then used the preferred compiler as specified in
+ spackconfig.
- Intuition: Use the system default if no package that depends on
+ Intuition: Use the spackconfig default if no package that depends on
this one has a strict compiler requirement. Otherwise, try to
build with the compiler that will be used by libraries that
link to this one, to maximize compatibility.
@@ -160,40 +261,91 @@ class DefaultConcretizer(object):
spec.compiler in all_compilers):
return False
- try:
- nearest = next(p for p in spec.traverse(direction='parents')
- if p.compiler is not None).compiler
+ #Find the another spec that has a compiler, or the root if none do
+ other_spec = find_spec(spec, lambda(x) : x.compiler)
+ if not other_spec:
+ other_spec = spec.root
+ other_compiler = other_spec.compiler
+ assert(other_spec)
+
+ # Check if the compiler is already fully specified
+ if other_compiler in all_compilers:
+ spec.compiler = other_compiler.copy()
+ return True
+
+ # Filter the compilers into a sorted list based on the compiler_order from spackconfig
+ compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler)
+ cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name)
+ matches = sorted(compiler_list, cmp=cmp_compilers)
+ if not matches:
+ raise UnavailableCompilerVersionError(other_compiler)
+
+ # copy concrete version into other_compiler
+ spec.compiler = matches[0].copy()
+ assert(spec.compiler.concrete)
+ return True # things changed.
- if not nearest in all_compilers:
- # Take the newest compiler that saisfies the spec
- matches = sorted(spack.compilers.find(nearest))
- if not matches:
- raise UnavailableCompilerVersionError(nearest)
- # copy concrete version into nearest spec
- nearest.versions = matches[-1].versions.copy()
- assert(nearest.concrete)
+def find_spec(spec, condition):
+ """Searches the dag from spec in an intelligent order and looks
+ for a spec that matches a condition"""
+ # First search parents, then search children
+ dagiter = chain(spec.traverse(direction='parents', root=False),
+ spec.traverse(direction='children', root=False))
+ visited = set()
+ for relative in dagiter:
+ if condition(relative):
+ return relative
+ visited.add(id(relative))
- spec.compiler = nearest.copy()
+ # Then search all other relatives in the DAG *except* spec
+ for relative in spec.root.traverse():
+ if relative is spec: continue
+ if id(relative) in visited: continue
+ if condition(relative):
+ return relative
- except StopIteration:
- spec.compiler = spack.compilers.default_compiler().copy()
+ # Finally search spec itself.
+ if condition(spec):
+ return spec
- return True # things changed.
+ return None # Nohting matched the condition.
- def choose_provider(self, spec, providers):
- """This is invoked for virtual specs. Given a spec with a virtual name,
- say "mpi", and a list of specs of possible providers of that spec,
- select a provider and return it.
- """
- assert(spec.virtual)
- assert(providers)
+def cmp_specs(lhs, rhs):
+ # Package name sort order is not configurable, always goes alphabetical
+ if lhs.name != rhs.name:
+ return cmp(lhs.name, rhs.name)
+
+ # Package version is second in compare order
+ pkgname = lhs.name
+ if lhs.versions != rhs.versions:
+ return spack.pkgsort.version_compare(
+ pkgname, lhs.versions, rhs.versions)
+
+ # Compiler is third
+ if lhs.compiler != rhs.compiler:
+ return spack.pkgsort.compiler_compare(
+ pkgname, lhs.compiler, rhs.compiler)
+
+ # Variants
+ if lhs.variants != rhs.variants:
+ return spack.pkgsort.variant_compare(
+ pkgname, lhs.variants, rhs.variants)
+
+ # Architecture
+ if lhs.architecture != rhs.architecture:
+ return spack.pkgsort.architecture_compare(
+ pkgname, lhs.architecture, rhs.architecture)
+
+ # Dependency is not configurable
+ lhash, rhash = hash(lhs), hash(rhs)
+ if lhash != rhash:
+ return -1 if lhash < rhash else 1
+
+ # Equal specs
+ return 0
- index = spack.spec.index_specs(providers)
- first_key = sorted(index.keys())[0]
- latest_version = sorted(index[first_key])[-1]
- return latest_version
class UnavailableCompilerVersionError(spack.error.SpackError):
@@ -211,3 +363,11 @@ class NoValidVersionError(spack.error.SpackError):
def __init__(self, spec):
super(NoValidVersionError, self).__init__(
"There are no valid versions for %s that match '%s'" % (spec.name, spec.versions))
+
+
+class NoBuildError(spack.error.SpackError):
+ """Raised when a package is configured with the nobuild option, but
+ no satisfactory external versions can be found"""
+ def __init__(self, spec):
+ super(NoBuildError, self).__init__(
+ "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name)
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 576a5afa2e..95a988f7ff 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -129,6 +129,7 @@ from ordereddict_backport import OrderedDict
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp
+import copy
import spack
from spack.error import SpackError
@@ -194,6 +195,40 @@ section_schemas = {
'default': [],
'items': {
'type': 'string'},},},},
+ 'packages': {
+ '$schema': 'http://json-schema.org/schema#',
+ 'title': 'Spack package configuration file schema',
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'patternProperties': {
+ r'packages:?': {
+ 'type': 'object',
+ 'default': {},
+ 'additionalProperties': False,
+ 'patternProperties': {
+ r'\w[\w-]*': { # package name
+ 'type': 'object',
+ 'default': {},
+ 'additionalProperties': False,
+ 'properties': {
+ 'version': {
+ 'type' : 'array',
+ 'default' : [],
+ 'items' : { 'anyOf' : [ { 'type' : 'string' },
+ { 'type' : 'number'}]}}, #version strings
+ 'compiler': {
+ 'type' : 'array',
+ 'default' : [],
+ 'items' : { 'type' : 'string' } }, #compiler specs
+ 'nobuild': {
+ 'type': 'boolean',
+ 'default': False,
+ },
+ 'paths': {
+ 'type' : 'object',
+ 'default' : {},
+ }
+ },},},},},}
}
"""OrderedDict of config scopes keyed by name.
@@ -494,6 +529,36 @@ def print_section(section):
raise ConfigError("Error reading configuration: %s" % section)
+def spec_externals(spec):
+ """Return a list of spec, directory pairs for each external location for spec"""
+ allpkgs = get_config('packages')
+ name = spec.name
+ spec_locations = []
+
+ pkg_paths = allpkgs.get(name, {}).get('paths', None)
+ if not pkg_paths:
+ return []
+
+ for pkg,path in pkg_paths.iteritems():
+ if not spec.satisfies(pkg):
+ continue
+ if not path:
+ continue
+ spec_locations.append( (spack.spec.Spec(pkg), path) )
+ return spec_locations
+
+
+def is_spec_nobuild(spec):
+ """Return true if the spec pkgspec is configured as nobuild"""
+ allpkgs = get_config('packages')
+ name = spec.name
+ if not spec.name in allpkgs:
+ return False
+ if not 'nobuild' in allpkgs[spec.name]:
+ return False
+ return allpkgs[spec.name]['nobuild']
+
+
class ConfigError(SpackError): pass
class ConfigFileError(ConfigError): pass
@@ -509,7 +574,7 @@ class ConfigFormatError(ConfigError):
# Try to get line number from erroneous instance and its parent
instance_mark = getattr(validation_error.instance, '_start_mark', None)
parent_mark = getattr(validation_error.parent, '_start_mark', None)
- path = getattr(validation_error, 'path', None)
+ path = [str(s) for s in getattr(validation_error, 'path', None)]
# Try really hard to get the parent (which sometimes is not
# set) This digs it out of the validated structure if it's not
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 242eb1afa0..39ee4e203d 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -198,6 +198,10 @@ class YamlDirectoryLayout(DirectoryLayout):
def relative_path_for_spec(self, spec):
_check_concrete(spec)
+
+ if spec.external:
+ return spec.external
+
dir_name = "%s-%s-%s" % (
spec.name,
spec.version,
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index ca9e9c4bd1..696adaf896 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -846,6 +846,11 @@ class Package(object):
if not self.spec.concrete:
raise ValueError("Can only install concrete packages.")
+ # No installation needed if package is external
+ if self.spec.external:
+ tty.msg("%s is externally installed in %s" % (self.name, self.spec.external))
+ return
+
# Ensure package is not already installed
if spack.install_layout.check_installed(self.spec):
tty.msg("%s is already installed in %s" % (self.name, self.prefix))
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
new file mode 100644
index 0000000000..9d219a1a6e
--- /dev/null
+++ b/lib/spack/spack/preferred_packages.py
@@ -0,0 +1,175 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+from spack.version import *
+
+class PreferredPackages(object):
+ _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, # Arbitrary, but consistent
+
+ def __init__(self):
+ self.preferred = spack.config.get_config('packages')
+ self._spec_for_pkgname_cache = {}
+
+ # Given a package name, sort component (e.g, version, compiler, ...), and
+ # a second_key (used by providers), return the list
+ def _order_for_package(self, pkgname, component, second_key, test_all=True):
+ pkglist = [pkgname]
+ if test_all:
+ pkglist.append('all')
+ for pkg in pkglist:
+ order = self.preferred.get(pkg, {}).get(component, {})
+ if type(order) is dict:
+ order = order.get(second_key, {})
+ if not order:
+ continue
+ return [str(s).strip() for s in order]
+ return []
+
+
+ # A generic sorting function. Given a package name and sort
+ # component, return less-than-0, 0, or greater-than-0 if
+ # a is respectively less-than, equal to, or greater than b.
+ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+ if a is None:
+ return -1
+ if b is None:
+ return 1
+ orderlist = self._order_for_package(pkgname, component, second_key)
+ a_in_list = str(a) in orderlist
+ b_in_list = str(b) in orderlist
+ if a_in_list and not b_in_list:
+ return -1
+ elif b_in_list and not a_in_list:
+ return 1
+
+ cmp_a = None
+ cmp_b = None
+ reverse = None
+ if not a_in_list and not b_in_list:
+ cmp_a = a
+ cmp_b = b
+ reverse = -1 if reverse_natural_compare else 1
+ else:
+ cmp_a = orderlist.index(str(a))
+ cmp_b = orderlist.index(str(b))
+ reverse = 1
+
+ if cmp_a < cmp_b:
+ return -1 * reverse
+ elif cmp_a > cmp_b:
+ return 1 * reverse
+ else:
+ return 0
+
+
+ # A sorting function for specs. Similar to component_compare, but
+ # a and b are considered to match entries in the sorting list if they
+ # satisfy the list component.
+ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+ if not a or not a.concrete:
+ return -1
+ if not b or not b.concrete:
+ return 1
+ specs = self._spec_for_pkgname(pkgname, component, second_key)
+ a_index = None
+ b_index = None
+ reverse = -1 if reverse_natural_compare else 1
+ for i, cspec in enumerate(specs):
+ if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)):
+ a_index = i
+ if b_index:
+ break
+ if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)):
+ b_index = i
+ if a_index:
+ break
+
+ if a_index != None and b_index == None: return -1
+ elif a_index == None and b_index != None: return 1
+ elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
+ elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
+ else: return cmp(a, b) * reverse
+
+
+
+ # Given a sort order specified by the pkgname/component/second_key, return
+ # a list of CompilerSpecs, VersionLists, or Specs for that sorting list.
+ def _spec_for_pkgname(self, pkgname, component, second_key):
+ key = (pkgname, component, second_key)
+ if not key in self._spec_for_pkgname_cache:
+ pkglist = self._order_for_package(pkgname, component, second_key)
+ if not pkglist:
+ if component in self._default_order:
+ pkglist = self._default_order[component]
+ if component == 'compiler':
+ self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist]
+ elif component == 'version':
+ self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist]
+ else:
+ self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist]
+ return self._spec_for_pkgname_cache[key]
+
+
+ def provider_compare(self, pkgname, provider_str, a, b):
+ """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or
+ greater-than b. A and b are possible implementations of provider_str.
+ One provider is less-than another if it is preferred over the other.
+ For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if
+ mvapich should be preferred over openmpi for scorep."""
+ return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
+
+
+ def spec_has_preferred_provider(self, pkgname, provider_str):
+ """Return True iff the named package has a list of preferred provider"""
+ return bool(self._order_for_package(pkgname, 'providers', provider_str, False))
+
+
+ def version_compare(self, pkgname, a, b):
+ """Return less-than-0, 0, or greater than 0 if version a of pkgname is
+ respecively less-than, equal-to, or greater-than version b of pkgname.
+ One version is less-than another if it is preferred over the other."""
+ return self._spec_compare(pkgname, 'version', a, b, True, None)
+
+
+ def variant_compare(self, pkgname, a, b):
+ """Return less-than-0, 0, or greater than 0 if variant a of pkgname is
+ respecively less-than, equal-to, or greater-than variant b of pkgname.
+ One variant is less-than another if it is preferred over the other."""
+ return self._component_compare(pkgname, 'variant', a, b, False, None)
+
+
+ def architecture_compare(self, pkgname, a, b):
+ """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is
+ respecively less-than, equal-to, or greater-than architecture b of pkgname.
+ One architecture is less-than another if it is preferred over the other."""
+ return self._component_compare(pkgname, 'architecture', a, b, False, None)
+
+
+ def compiler_compare(self, pkgname, a, b):
+ """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is
+ respecively less-than, equal-to, or greater-than compiler b of pkgname.
+ One compiler is less-than another if it is preferred over the other."""
+ return self._spec_compare(pkgname, 'compiler', a, b, False, None)
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 10e246bf2e..c045e80365 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -420,6 +420,7 @@ class Spec(object):
# package.py files for.
self._normal = kwargs.get('normal', False)
self._concrete = kwargs.get('concrete', False)
+ self.external = None
# This allows users to construct a spec DAG with literals.
# Note that given two specs a and b, Spec(a) copies a, but
@@ -770,12 +771,11 @@ class Spec(object):
# Concretize virtual dependencies last. Because they're added
# to presets below, their constraints will all be merged, but we'll
# still need to select a concrete package later.
- if not self.virtual:
- changed |= any(
- (spack.concretizer.concretize_architecture(self),
- spack.concretizer.concretize_compiler(self),
- spack.concretizer.concretize_version(self),
- spack.concretizer.concretize_variants(self)))
+ changed |= any(
+ (spack.concretizer.concretize_architecture(self),
+ spack.concretizer.concretize_compiler(self),
+ spack.concretizer.concretize_version(self),
+ spack.concretizer.concretize_variants(self)))
presets[self.name] = self
visited.add(self.name)
@@ -808,21 +808,18 @@ class Spec(object):
a problem.
"""
changed = False
- while True:
- virtuals =[v for v in self.traverse() if v.virtual]
- if not virtuals:
- return changed
-
- for spec in virtuals:
- providers = spack.repo.providers_for(spec)
- concrete = spack.concretizer.choose_provider(spec, providers)
- concrete = concrete.copy()
- spec._replace_with(concrete)
- changed = True
+ done = False
+ while not done:
+ done = True
+ for spec in list(self.traverse()):
+ if spack.concretizer.concretize_virtual_and_external(spec):
+ done = False
+ changed = True
- # If there are duplicate providers or duplicate provider deps, this
- # consolidates them and merge constraints.
- changed |= self.normalize(force=True)
+ # If there are duplicate providers or duplicate provider deps, this
+ # consolidates them and merge constraints.
+ changed |= self.normalize(force=True)
+ return changed
def concretize(self):
@@ -837,6 +834,7 @@ class Spec(object):
with requirements of its pacakges. See flatten() and normalize() for
more details on this.
"""
+
if self._concrete:
return
@@ -1069,7 +1067,7 @@ class Spec(object):
# if we descend into a virtual spec, there's nothing more
# to normalize. Concretize will finish resolving it later.
- if self.virtual:
+ if self.virtual or self.external:
return False
# Combine constraints from package deps with constraints from
@@ -1404,15 +1402,26 @@ class Spec(object):
Whether deps should be copied too. Set to false to copy a
spec but not its dependencies.
"""
+
+ # We don't count dependencies as changes here
+ changed = True
+ if hasattr(self, 'name'):
+ changed = (self.name != other.name and self.versions != other.versions and \
+ self.architecture != other.architecture and self.compiler != other.compiler and \
+ self.variants != other.variants and self._normal != other._normal and \
+ self.concrete != other.concrete and self.external != other.external)
+
# Local node attributes get copied first.
self.name = other.name
self.versions = other.versions.copy()
self.architecture = other.architecture
self.compiler = other.compiler.copy() if other.compiler else None
- self.dependents = DependencyMap()
- self.dependencies = DependencyMap()
+ if kwargs.get('cleardeps', True):
+ self.dependents = DependencyMap()
+ self.dependencies = DependencyMap()
self.variants = other.variants.copy()
self.variants.spec = self
+ self.external = other.external
self.namespace = other.namespace
# If we copy dependencies, preserve DAG structure in the new spec
@@ -1431,6 +1440,8 @@ class Spec(object):
# Since we preserved structure, we can copy _normal safely.
self._normal = other._normal
self._concrete = other._concrete
+ self.external = other.external
+ return changed
def copy(self, **kwargs):
@@ -1571,14 +1582,28 @@ class Spec(object):
$_ Package name
$. Full package name (with namespace)
- $@ Version
- $% Compiler
- $%@ Compiler & compiler version
+ $@ Version with '@' prefix
+ $% Compiler with '%' prefix
+ $%@ Compiler with '%' prefix & compiler version with '@' prefix
$+ Options
- $= Architecture
- $# 7-char prefix of DAG hash
+ $= Architecture with '=' prefix
+ $# 7-char prefix of DAG hash with '-' prefix
$$ $
+ You can also use full-string versions, which leave off the prefixes:
+
+ ${PACKAGE} Package name
+ ${VERSION} Version
+ ${COMPILER} Full compiler string
+ ${COMPILERNAME} Compiler name
+ ${COMPILERVER} Compiler version
+ ${OPTIONS} Options
+ ${ARCHITECTURE} Architecture
+ ${SHA1} Dependencies 8-char sha1 prefix
+
+ ${SPACK_ROOT} The spack root directory
+ ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt
+
Optionally you can provide a width, e.g. $20_ for a 20-wide name.
Like printf, you can provide '-' for left justification, e.g.
$-20_ for a left-justified name.
@@ -1594,7 +1619,8 @@ class Spec(object):
color = kwargs.get('color', False)
length = len(format_string)
out = StringIO()
- escape = compiler = False
+ named = escape = compiler = False
+ named_str = fmt = ''
def write(s, c):
if color:
@@ -1636,9 +1662,12 @@ class Spec(object):
elif c == '#':
out.write('-' + fmt % (self.dag_hash(7)))
elif c == '$':
- if fmt != '':
+ if fmt != '%s':
raise ValueError("Can't use format width with $$.")
out.write('$')
+ elif c == '{':
+ named = True
+ named_str = ''
escape = False
elif compiler:
@@ -1652,6 +1681,43 @@ class Spec(object):
out.write(c)
compiler = False
+ elif named:
+ if not c == '}':
+ if i == length - 1:
+ raise ValueError("Error: unterminated ${ in format: '%s'"
+ % format_string)
+ named_str += c
+ continue;
+ if named_str == 'PACKAGE':
+ write(fmt % self.name, '@')
+ if named_str == 'VERSION':
+ if self.versions and self.versions != _any_version:
+ write(fmt % str(self.versions), '@')
+ elif named_str == 'COMPILER':
+ if self.compiler:
+ write(fmt % self.compiler, '%')
+ elif named_str == 'COMPILERNAME':
+ if self.compiler:
+ write(fmt % self.compiler.name, '%')
+ elif named_str == 'COMPILERVER':
+ if self.compiler:
+ write(fmt % self.compiler.versions, '%')
+ elif named_str == 'OPTIONS':
+ if self.variants:
+ write(fmt % str(self.variants), '+')
+ elif named_str == 'ARCHITECTURE':
+ if self.architecture:
+ write(fmt % str(self.architecture), '=')
+ elif named_str == 'SHA1':
+ if self.dependencies:
+ out.write(fmt % str(self.dag_hash(7)))
+ elif named_str == 'SPACK_ROOT':
+ out.write(fmt % spack.prefix)
+ elif named_str == 'SPACK_INSTALL':
+ out.write(fmt % spack.install_path)
+
+ named = False
+
elif c == '$':
escape = True
if i == length - 1:
@@ -1782,6 +1848,7 @@ class SpecParser(spack.parse.Parser):
spec.variants = VariantMap(spec)
spec.architecture = None
spec.compiler = None
+ spec.external = None
spec.dependents = DependencyMap()
spec.dependencies = DependencyMap()
spec.namespace = spec_namespace
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 794344fb6a..07828d8ea6 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -24,6 +24,7 @@
##############################################################################
import spack
from spack.spec import Spec, CompilerSpec
+from spack.concretize import find_spec
from spack.test.mock_packages_test import *
class ConcretizeTest(MockPackagesTest):
@@ -190,3 +191,94 @@ class ConcretizeTest(MockPackagesTest):
# TODO: not exactly the syntax I would like.
self.assertTrue(spec['libdwarf'].compiler.satisfies('clang'))
self.assertTrue(spec['libelf'].compiler.satisfies('clang'))
+
+
+ def test_external_package(self):
+ spec = Spec('externaltool')
+ spec.concretize()
+
+ self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+ self.assertFalse('externalprereq' in spec)
+ self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+
+
+ def test_nobuild_package(self):
+ got_error = False
+ spec = Spec('externaltool%clang')
+ try:
+ spec.concretize()
+ except spack.concretize.NoBuildError:
+ got_error = True
+ self.assertTrue(got_error)
+
+
+ def test_external_and_virtual(self):
+ spec = Spec('externaltest')
+ spec.concretize()
+ self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+ self.assertEqual(spec['stuff'].external, '/path/to/external_virtual_gcc')
+ self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+ self.assertTrue(spec['stuff'].compiler.satisfies('gcc'))
+
+
+ def test_find_spec_parents(self):
+ """Tests the spec finding logic used by concretization. """
+ s = Spec('a +foo',
+ Spec('b +foo',
+ Spec('c'),
+ Spec('d +foo')),
+ Spec('e +foo'))
+
+ self.assertEqual('a', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+ def test_find_spec_children(self):
+ s = Spec('a',
+ Spec('b +foo',
+ Spec('c'),
+ Spec('d +foo')),
+ Spec('e +foo'))
+ self.assertEqual('d', find_spec(s['b'], lambda s: '+foo' in s).name)
+ s = Spec('a',
+ Spec('b +foo',
+ Spec('c +foo'),
+ Spec('d')),
+ Spec('e +foo'))
+ self.assertEqual('c', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+ def test_find_spec_sibling(self):
+ s = Spec('a',
+ Spec('b +foo',
+ Spec('c'),
+ Spec('d')),
+ Spec('e +foo'))
+ self.assertEqual('e', find_spec(s['b'], lambda s: '+foo' in s).name)
+ self.assertEqual('b', find_spec(s['e'], lambda s: '+foo' in s).name)
+
+ s = Spec('a',
+ Spec('b +foo',
+ Spec('c'),
+ Spec('d')),
+ Spec('e',
+ Spec('f +foo')))
+ self.assertEqual('f', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+ def test_find_spec_self(self):
+ s = Spec('a',
+ Spec('b +foo',
+ Spec('c'),
+ Spec('d')),
+ Spec('e'))
+ self.assertEqual('b', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+ def test_find_spec_none(self):
+ s = Spec('a',
+ Spec('b',
+ Spec('c'),
+ Spec('d')),
+ Spec('e'))
+ self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s))
+
diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py
index d814572d4a..8ad8f1a360 100644
--- a/lib/spack/spack/test/directory_layout.py
+++ b/lib/spack/spack/test/directory_layout.py
@@ -66,6 +66,9 @@ class DirectoryLayoutTest(MockPackagesTest):
packages = list(spack.repo.all_packages())[:max_packages]
for pkg in packages:
+ if pkg.name.startswith('external'):
+ #External package tests cannot be installed
+ continue
spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a
@@ -171,6 +174,9 @@ class DirectoryLayoutTest(MockPackagesTest):
# Create install prefixes for all packages in the list
installed_specs = {}
for pkg in packages:
+ if pkg.name.startswith('external'):
+ #External package tests cannot be installed
+ continue
spec = pkg.spec.concretized()
installed_specs[spec.name] = spec
self.layout.create_install_directory(spec)
diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py
index 0b8867b61e..079cbcc136 100644
--- a/lib/spack/spack/test/mock_packages_test.py
+++ b/lib/spack/spack/test/mock_packages_test.py
@@ -49,6 +49,19 @@ compilers:
fc: /path/to/gfortran
"""
+mock_packages_config = """\
+packages:
+ externaltool:
+ nobuild: True
+ paths:
+ externaltool@1.0%gcc@4.5.0: /path/to/external_tool
+ externalvirtual:
+ nobuild: True
+ paths:
+ externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang
+ externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc
+"""
+
class MockPackagesTest(unittest.TestCase):
def initmock(self):
# Use the mock packages database for these tests. This allows
@@ -66,9 +79,10 @@ class MockPackagesTest(unittest.TestCase):
self.mock_user_config = os.path.join(self.temp_config, 'user')
mkdirp(self.mock_site_config)
mkdirp(self.mock_user_config)
- comp_yaml = os.path.join(self.mock_site_config, 'compilers.yaml')
- with open(comp_yaml, 'w') as f:
- f.write(mock_compiler_config)
+ for confs in [('compilers.yaml', mock_compiler_config), ('packages.yaml', mock_packages_config)]:
+ conf_yaml = os.path.join(self.mock_site_config, confs[0])
+ with open(conf_yaml, 'w') as f:
+ f.write(confs[1])
# TODO: Mocking this up is kind of brittle b/c ConfigScope
# TODO: constructor modifies config_scopes. Make it cleaner.
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-02 13:19:17 +0000