summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/basic_usage.rst276
-rw-r--r--lib/spack/docs/configuration.rst (renamed from lib/spack/docs/site_configuration.rst)88
-rw-r--r--lib/spack/docs/features.rst17
-rw-r--r--lib/spack/docs/index.rst2
-rw-r--r--lib/spack/docs/packaging_guide.rst108
-rwxr-xr-xlib/spack/env/cc101
-rw-r--r--lib/spack/llnl/util/filesystem.py76
-rw-r--r--lib/spack/llnl/util/tty/__init__.py4
-rw-r--r--lib/spack/llnl/util/tty/colify.py9
-rw-r--r--lib/spack/spack/__init__.py4
-rw-r--r--lib/spack/spack/abi.py5
-rw-r--r--lib/spack/spack/architecture.py506
-rw-r--r--lib/spack/spack/build_environment.py205
-rw-r--r--lib/spack/spack/cmd/arch.py6
-rw-r--r--lib/spack/spack/cmd/compiler.py18
-rw-r--r--lib/spack/spack/cmd/create.py20
-rw-r--r--lib/spack/spack/cmd/find.py191
-rw-r--r--lib/spack/spack/cmd/install.py3
-rw-r--r--lib/spack/spack/cmd/uninstall.py13
-rw-r--r--lib/spack/spack/cmd/view.py295
-rw-r--r--lib/spack/spack/compiler.py95
-rw-r--r--lib/spack/spack/compilers/__init__.py258
-rw-r--r--lib/spack/spack/compilers/clang.py2
-rw-r--r--lib/spack/spack/compilers/craype.py58
-rw-r--r--lib/spack/spack/compilers/gcc.py7
-rw-r--r--lib/spack/spack/compilers/intel.py3
-rw-r--r--lib/spack/spack/compilers/pgi.py7
-rw-r--r--lib/spack/spack/compilers/xl.py4
-rw-r--r--lib/spack/spack/concretize.py162
-rw-r--r--lib/spack/spack/config.py62
-rw-r--r--lib/spack/spack/database.py137
-rw-r--r--lib/spack/spack/directives.py51
-rw-r--r--lib/spack/spack/directory_layout.py5
-rw-r--r--lib/spack/spack/environment.py173
-rw-r--r--lib/spack/spack/fetch_strategy.py261
-rw-r--r--lib/spack/spack/hooks/licensing.py5
-rw-r--r--lib/spack/spack/hooks/sbang.py8
-rw-r--r--lib/spack/spack/modules.py6
-rw-r--r--lib/spack/spack/multimethod.py4
-rw-r--r--lib/spack/spack/operating_systems/__init__.py0
-rw-r--r--lib/spack/spack/operating_systems/cnl.py62
-rw-r--r--lib/spack/spack/operating_systems/linux_distro.py22
-rw-r--r--lib/spack/spack/operating_systems/mac_os.py29
-rw-r--r--lib/spack/spack/package.py28
-rw-r--r--lib/spack/spack/platforms/__init__.py0
-rw-r--r--lib/spack/spack/platforms/bgq.py18
-rw-r--r--lib/spack/spack/platforms/cray_xc.py46
-rw-r--r--lib/spack/spack/platforms/darwin.py26
-rw-r--r--lib/spack/spack/platforms/linux.py24
-rw-r--r--lib/spack/spack/platforms/test.py28
-rw-r--r--lib/spack/spack/spec.py1071
-rw-r--r--lib/spack/spack/test/__init__.py7
-rw-r--r--lib/spack/spack/test/architecture.py112
-rw-r--r--lib/spack/spack/test/cc.py72
-rw-r--r--lib/spack/spack/test/cmd/find.py60
-rw-r--r--lib/spack/spack/test/cmd/test_compiler_cmd.py81
-rw-r--r--lib/spack/spack/test/concretize.py32
-rw-r--r--lib/spack/spack/test/config.py124
-rw-r--r--lib/spack/spack/test/data/sourceme_first.sh3
-rw-r--r--lib/spack/spack/test/data/sourceme_second.sh3
-rw-r--r--lib/spack/spack/test/environment.py65
-rw-r--r--lib/spack/spack/test/mock_packages_test.py119
-rw-r--r--lib/spack/spack/test/modules.py19
-rw-r--r--lib/spack/spack/test/multimethod.py29
-rw-r--r--lib/spack/spack/test/operating_system.py55
-rw-r--r--lib/spack/spack/test/optional_deps.py7
-rw-r--r--lib/spack/spack/test/spec_dag.py9
-rw-r--r--lib/spack/spack/test/spec_semantics.py122
-rw-r--r--lib/spack/spack/test/spec_syntax.py10
-rw-r--r--lib/spack/spack/test/versions.py54
-rw-r--r--lib/spack/spack/util/executable.py93
-rw-r--r--lib/spack/spack/variant.py2
-rw-r--r--lib/spack/spack/version.py158
-rw-r--r--lib/spack/spack/virtual.py5
-rw-r--r--lib/spack/spack/yaml_version_check.py55
75 files changed, 4494 insertions, 1411 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index 15db2f7a16..50c48b802b 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -102,8 +102,8 @@ that the packages is installed:
==> adept-utils is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da.
==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
######################################################################## 100.0%
- ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz
- ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23.
+ ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz
+ ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23.
==> No patches needed for mpileaks.
==> Building mpileaks.
@@ -132,10 +132,10 @@ sites, as installing a version that one user needs will not disrupt
existing installations for other users.
In addition to different versions, Spack can customize the compiler,
-compile-time options (variants), and platform (for cross compiles) of
-an installation. Spack is unique in that it can also configure the
-*dependencies* a package is built with. For example, two
-configurations of the same version of a package, one built with boost
+compile-time options (variants), compiler flags, and platform (for
+cross compiles) of an installation. Spack is unique in that it can
+also configure the *dependencies* a package is built with. For example,
+two configurations of the same version of a package, one built with boost
1.39.0, and the other version built with version 1.43.0, can coexist.
This can all be done on the command line using the *spec* syntax.
@@ -246,6 +246,12 @@ Packages are divided into groups according to their architecture and
compiler. Within each group, Spack tries to keep the view simple, and
only shows the version of installed packages.
+``spack find`` can filter the package list based on the package name, spec, or
+a number of properties of their installation status. For example, missing
+dependencies of a spec can be shown with ``-m``, packages which were
+explicitly installed with ``spack install <package>`` can be singled out with
+``-e`` and those which have been pulled in only as dependencies with ``-E``.
+
In some cases, there may be different configurations of the *same*
version of a package installed. For example, there are two
installations of of ``libdwarf@20130729`` above. We can look at them
@@ -328,9 +334,15 @@ of libelf would look like this:
-- chaos_5_x86_64_ib / gcc@4.4.7 --------------------------------
libdwarf@20130729-d9b90962
+We can also search for packages that have a certain attribute. For example,
+``spack find -l libdwarf +debug`` will show only installations of libdwarf
+with the 'debug' compile-time option enabled, while ``spack find -l +debug``
+will find every installed package with a 'debug' compile-time option enabled.
+
The full spec syntax is discussed in detail in :ref:`sec-specs`.
+
Compiler configuration
-----------------------------------
@@ -457,6 +469,26 @@ For compilers, like ``clang``, that do not support Fortran, put
Once you save the file, the configured compilers will show up in the
list displayed by ``spack compilers``.
+You can also add compiler flags to manually configured compilers. The
+valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``,
+``ldflags``, and ``ldlibs``. For example,::
+
+ ...
+ chaos_5_x86_64_ib:
+ ...
+ intel@15.0.0:
+ cc: /usr/local/bin/icc-15.0.024-beta
+ cxx: /usr/local/bin/icpc-15.0.024-beta
+ f77: /usr/local/bin/ifort-15.0.024-beta
+ fc: /usr/local/bin/ifort-15.0.024-beta
+ cppflags: -O3 -fPIC
+ ...
+
+These flags will be treated by spack as if they were enterred from
+the command line each time this compiler is used. The compiler wrappers
+then inject those flags into the compiler command. Compiler flags
+enterred from the command line will be discussed in more detail in the
+following section.
.. _sec-specs:
@@ -474,7 +506,7 @@ the full syntax of specs.
Here is an example of a much longer spec than we've seen thus far::
- mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt =bgqos_0 ^callpath @1.1 %gcc@4.7.2
+ mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt arch=bgq_os ^callpath @1.1 %gcc@4.7.2
If provided to ``spack install``, this will install the ``mpileaks``
library at some version between ``1.2`` and ``1.4`` (inclusive),
@@ -492,8 +524,12 @@ More formally, a spec consists of the following pieces:
* ``%`` Optional compiler specifier, with an optional compiler version
(``gcc`` or ``gcc@4.7.3``)
* ``+`` or ``-`` or ``~`` Optional variant specifiers (``+debug``,
- ``-qt``, or ``~qt``)
-* ``=`` Optional architecture specifier (``bgqos_0``)
+ ``-qt``, or ``~qt``) for boolean variants
+* ``name=<value>`` Optional variant specifiers that are not restricted to
+boolean variants
+* ``name=<value>`` Optional compiler flag specifiers. Valid flag names are
+``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``.
+* ``arch=<value>`` Optional architecture specifier (``arch=bgq_os``)
* ``^`` Dependency specs (``^callpath@1.1``)
There are two things to notice here. The first is that specs are
@@ -573,7 +609,7 @@ compilers, variants, and architectures just like any other spec.
Specifiers are associated with the nearest package name to their left.
For example, above, ``@1.1`` and ``%gcc@4.7.2`` associates with the
``callpath`` package, while ``@1.2:1.4``, ``%gcc@4.7.5``, ``+debug``,
-``-qt``, and ``=bgqos_0`` all associate with the ``mpileaks`` package.
+``-qt``, and ``arch=bgq_os`` all associate with the ``mpileaks`` package.
In the diagram above, ``mpileaks`` depends on ``mpich`` with an
unspecified version, but packages can depend on other packages with
@@ -629,22 +665,25 @@ based on site policies.
Variants
~~~~~~~~~~~~~~~~~~~~~~~
-.. Note::
-
- Variants are not yet supported, but will be in the next Spack
- release (0.9), due in Q2 2015.
-
-Variants are named options associated with a particular package, and
-they can be turned on or off. For example, above, supplying
-``+debug`` causes ``mpileaks`` to be built with debug flags. The
-names of particular variants available for a package depend on what
-was provided by the package author. ``spack info <package>`` will
+Variants are named options associated with a particular package. They are
+optional, as each package must provide default values for each variant it
+makes available. Variants can be specified using
+a flexible parameter syntax ``name=<value>``. For example,
+``spack install libelf debug=True`` will install libelf build with debug
+flags. The names of particular variants available for a package depend on
+what was provided by the package author. ``spack into <package>`` will
provide information on what build variants are available.
-Depending on the package a variant may be on or off by default. For
-``mpileaks`` here, ``debug`` is off by default, and we turned it on
-with ``+debug``. If a package is on by default you can turn it off by
-either adding ``-name`` or ``~name`` to the spec.
+For compatibility with earlier versions, variants which happen to be
+boolean in nature can be specified by a syntax that represents turning
+options on and off. For example, in the previous spec we could have
+supplied ``libelf +debug`` with the same effect of enabling the debug
+compile time option for the libelf package.
+
+Depending on the package a variant may have any default value. For
+``libelf`` here, ``debug`` is ``False`` by default, and we turned it on
+with ``debug=True`` or ``+debug``. If a package is ``True`` by default
+you can turn it off by either adding ``-name`` or ``~name`` to the spec.
There are two syntaxes here because, depending on context, ``~`` and
``-`` may mean different things. In most shells, the following will
@@ -656,7 +695,7 @@ result in the shell performing home directory substitution:
mpileaks~debug # use this instead
If there is a user called ``debug``, the ``~`` will be incorrectly
-expanded. In this situation, you would want to write ``mpileaks
+expanded. In this situation, you would want to write ``libelf
-debug``. However, ``-`` can be ambiguous when included after a
package name without spaces:
@@ -671,12 +710,35 @@ package, not a request for ``mpileaks`` built without ``debug``
options. In this scenario, you should write ``mpileaks~debug`` to
avoid ambiguity.
-When spack normalizes specs, it prints them out with no spaces and
-uses only ``~`` for disabled variants. We allow ``-`` and spaces on
-the command line is provided for convenience and legibility.
+When spack normalizes specs, it prints them out with no spaces boolean
+variants using the backwards compatibility syntax and uses only ``~``
+for disabled boolean variants. We allow ``-`` and spaces on the command
+line is provided for convenience and legibility.
+
+Compiler Flags
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Compiler flags are specified using the same syntax as non-boolean variants,
+but fulfill a different purpose. While the function of a variant is set by
+the package, compiler flags are used by the compiler wrappers to inject
+flags into the compile line of the build. Additionally, compiler flags are
+inherited by dependencies. ``spack install libdwarf cppflags=\"-g\"`` will
+install both libdwarf and libelf with the ``-g`` flag injected into their
+compile line.
+
+Notice that the value of the compiler flags must be escape quoted on the
+command line. From within python files, the same spec would be specified
+``libdwarf cppflags="-g"``. This is necessary because of how the shell
+handles the quote symbols.
-Architecture specifier
+The six compiler flags are injected in the order of implicit make commands
+in gnu autotools. If all flags are set, the order is
+``$cppflags $cflags|$cxxflags $ldflags command $ldlibs`` for C and C++ and
+``$fflags $cppflags $ldflags command $ldlibs`` for fortran.
+
+
+Architecture specifiers
~~~~~~~~~~~~~~~~~~~~~~~
.. Note::
@@ -684,12 +746,9 @@ Architecture specifier
Architecture specifiers are part of specs but are not yet
functional. They will be in Spack version 1.0, due in Q3 2015.
-The architecture specifier starts with a ``=`` and also comes after
-some package name within a spec. It allows a user to specify a
-particular architecture for the package to be built. This is mostly
-used for architectures that need cross-compilation, and in most cases,
-users will not need to specify the architecture when they install a
-package.
+The architecture specifier looks identical to a variant specifier for a
+non-boolean variant. The architecture can be specified only using the
+reserved name ``arch`` (``arch=bgq_os``).
.. _sec-virtual-dependencies:
@@ -767,6 +826,23 @@ any MPI implementation will do. If another package depends on
error. Likewise, if you try to plug in some package that doesn't
provide MPI, Spack will raise an error.
+Specifying Specs by Hash
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Complicated specs can become cumbersome to enter on the command line,
+especially when many of the qualifications are necessary to
+distinguish between similar installs, for example when using the
+``uninstall`` command. To avoid this, when referencing an existing spec,
+Spack allows you to reference specs by their hash. We previously
+discussed the spec hash that Spack computes. In place of a spec in any
+command, substitute ``/<hash>`` where ``<hash>`` is any amount from
+the beginning of a spec hash. If the given spec hash is sufficient
+to be unique, Spack will replace the reference with the spec to which
+it refers. Otherwise, it will prompt for a more qualified hash.
+
+Note that this will not work to reinstall a depencency uninstalled by
+``spack uninstall -f``.
+
.. _spack-providers:
``spack providers``
@@ -996,8 +1072,8 @@ than one installed package matches it), then Spack will warn you:
$ spack load libelf
==> Error: Multiple matches for spec libelf. Choose one:
- libelf@0.8.13%gcc@4.4.7=chaos_5_x86_64_ib
- libelf@0.8.13%intel@15.0.0=chaos_5_x86_64_ib
+ libelf@0.8.13%gcc@4.4.7 arch=chaos_5_x86_64_ib
+ libelf@0.8.13%intel@15.0.0 arch=chaos_5_x86_64_ib
You can either type the ``spack load`` command again with a fully
qualified argument, or you can add just enough extra constraints to
@@ -1245,6 +1321,120 @@ regenerate all module and dotkit files from scratch:
.. _extensions:
+Filesystem Views
+-------------------------------
+
+.. Maybe this is not the right location for this documentation.
+
+The Spack installation area allows for many package installation trees
+to coexist and gives the user choices as to what versions and variants
+of packages to use. To use them, the user must rely on a way to
+aggregate a subset of those packages. The section on Environment
+Modules gives one good way to do that which relies on setting various
+environment variables. An alternative way to aggregate is through
+**filesystem views**.
+
+A filesystem view is a single directory tree which is the union of the
+directory hierarchies of the individual package installation trees
+that have been included. The files of the view's installed packages
+are brought into the view by symbolic or hard links back to their
+location in the original Spack installation area. As the view is
+formed, any clashes due to a file having the exact same path in its
+package installation tree are handled in a first-come-first-served
+basis and a warning is printed. Packages and their dependencies can
+be both added and removed. During removal, empty directories will be
+purged. These operations can be limited to pertain to just the
+packages listed by the user or to exclude specific dependencies and
+they allow for software installed outside of Spack to coexist inside
+the filesystem view tree.
+
+By its nature, a filesystem view represents a particular choice of one
+set of packages among all the versions and variants that are available
+in the Spack installation area. It is thus equivalent to the
+directory hiearchy that might exist under ``/usr/local``. While this
+limits a view to including only one version/variant of any package, it
+provides the benefits of having a simpler and traditional layout which
+may be used without any particular knowledge that its packages were
+built by Spack.
+
+Views can be used for a variety of purposes including:
+
+- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin.
+- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``.
+- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball.
+- Producing ephemeral testing or developing environments.
+
+Using Filesystem Views
+~~~~~~~~~~~~~~~~~~~~~~
+
+A filesystem view is created and packages are linked in by the ``spack
+view`` command's ``symlink`` and ``hardlink`` sub-commands. The
+``spack view remove`` command can be used to unlink some or all of the
+filesystem view.
+
+The following example creates a filesystem view based
+on an installed ``cmake`` package and then removes from the view the
+files in the ``cmake`` package while retaining its dependencies.
+
+.. code-block:: sh
+
+
+ $ spack view -v symlink myview cmake@3.5.2
+ ==> Linking package: "ncurses"
+ ==> Linking package: "zlib"
+ ==> Linking package: "openssl"
+ ==> Linking package: "cmake"
+
+ $ ls myview/
+ bin doc etc include lib share
+
+ $ ls myview/bin/
+ captoinfo clear cpack ctest infotocap openssl tabs toe tset
+ ccmake cmake c_rehash infocmp ncurses6-config reset tic tput
+
+ $ spack view -v -d false rm myview cmake@3.5.2
+ ==> Removing package: "cmake"
+
+ $ ls myview/bin/
+ captoinfo c_rehash infotocap openssl tabs toe tset
+ clear infocmp ncurses6-config reset tic tput
+
+
+Limitations of Filesystem Views
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section describes some limitations that should be considered in
+using filesystems views.
+
+Filesystem views are merely organizational. The binary executable
+programs, shared libraries and other build products found in a view
+are mere links into the "real" Spack installation area. If a view is
+built with symbolic links it requires the Spack-installed package to
+be kept in place. Building a view with hardlinks removes this
+requirement but any internal paths (eg, rpath or ``#!`` interpreter
+specifications) will still require the Spack-installed package files
+to be in place.
+
+.. FIXME: reference the relocation work of Hegner and Gartung.
+
+As described above, when a view is built only a single instance of a
+file may exist in the unified filesystem tree. If more than one
+package provides a file at the same path (relative to its own root)
+then it is the first package added to the view that "wins". A warning
+is printed and it is up to the user to determine if the conflict
+matters.
+
+It is up to the user to assure a consistent view is produced. In
+particular if the user excludes packages, limits the following of
+dependencies or removes packages the view may become inconsistent. In
+particular, if two packages require the same sub-tree of dependencies,
+removing one package (recursively) will remove its dependencies and
+leave the other package broken.
+
+
+
+
+
Extensions & Python support
------------------------------------
@@ -1276,7 +1466,7 @@ You can find extensions for your Python installation like this:
.. code-block:: sh
$ spack extensions python
- ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96
+ ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96
==> 36 extensions:
geos py-ipython py-pexpect py-pyside py-sip
py-basemap py-libxml2 py-pil py-pytz py-six
@@ -1366,9 +1556,9 @@ installation:
.. code-block:: sh
$ spack activate py-numpy
- ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7.
- ==> Activated extension py-nose@1.3.4%gcc@4.4.7=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7.
- ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7.
+ ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7.
+ ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7.
+ ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7.
Several things have happened here. The user requested that
``py-numpy`` be activated in the ``python`` installation it was built
@@ -1383,7 +1573,7 @@ packages listed as activated:
.. code-block:: sh
$ spack extensions python
- ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96
+ ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96
==> 36 extensions:
geos py-ipython py-pexpect py-pyside py-sip
py-basemap py-libxml2 py-pil py-pytz py-six
@@ -1431,7 +1621,7 @@ dependencies, you can use ``spack activate -f``:
.. code-block:: sh
$ spack activate -f py-numpy
- ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7.
+ ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7.
.. _spack-deactivate:
diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/configuration.rst
index 3abfa21a9d..c613071c65 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/configuration.rst
@@ -1,6 +1,6 @@
-.. _site-configuration:
+.. _configuration:
-Site configuration
+Configuration
===================================
.. _temp-space:
@@ -55,7 +55,7 @@ directory is.
External Packages
-~~~~~~~~~~~~~~~~~~~~~
+----------------------------
Spack can be configured to use externally-installed
packages rather than building its own packages. This may be desirable
if machines ship with system packages, such as a customized MPI
@@ -70,15 +70,15 @@ directory. Here's an example of an external configuration:
packages:
openmpi:
paths:
- openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
- openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
- openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
+ openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+ openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+ openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
This example lists three installations of OpenMPI, one built with gcc,
one built with gcc and debug information, and another built with Intel.
If Spack is asked to build a package that uses one of these MPIs as a
dependency, it will use the the pre-installed OpenMPI in
-the given directory.
+the given directory.
Each ``packages.yaml`` begins with a ``packages:`` token, followed
by a list of package names. To specify externals, add a ``paths``
@@ -108,9 +108,9 @@ be:
packages:
openmpi:
paths:
- openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
- openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
- openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
+ openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+ openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+ openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
buildable: False
The addition of the ``buildable`` flag tells Spack that it should never build
@@ -118,13 +118,73 @@ its own version of OpenMPI, and it will instead always rely on a pre-built
OpenMPI. Similar to ``paths``, ``buildable`` is specified as a property under
a package name.
-The ``buildable`` does not need to be paired with external packages.
-It could also be used alone to forbid packages that may be
+The ``buildable`` does not need to be paired with external packages.
+It could also be used alone to forbid packages that may be
buggy or otherwise undesirable.
+Concretization Preferences
+--------------------------------
+
+Spack can be configured to prefer certain compilers, package
+versions, depends_on, and variants during concretization.
+The preferred configuration can be controlled via the
+``~/.spack/packages.yaml`` file for user configuations, or the
+``etc/spack/packages.yaml`` site configuration.
+
+
+Here's an example packages.yaml file that sets preferred packages:
+
+.. code-block:: sh
+
+ packages:
+ dyninst:
+ compiler: [gcc@4.9]
+ variants: +debug
+ gperftools:
+ version: [2.2, 2.4, 2.3]
+ all:
+ compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi]
+ providers:
+ mpi: [mvapich, mpich, openmpi]
+
+
+At a high level, this example is specifying how packages should be
+concretized. The dyninst package should prefer using gcc 4.9 and
+be built with debug options. The gperftools package should prefer version
+2.2 over 2.4. Every package on the system should prefer mvapich for
+its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9).
+These options are used to fill in implicit defaults. Any of them can be overwritten
+on the command line if explicitly requested.
+
+Each packages.yaml file begins with the string ``packages:`` and
+package names are specified on the next level. The special string ``all``
+applies settings to each package. Underneath each package name is
+one or more components: ``compiler``, ``variants``, ``version``,
+or ``providers``. Each component has an ordered list of spec
+``constraints``, with earlier entries in the list being preferred over
+later entries.
+
+Sometimes a package installation may have constraints that forbid
+the first concretization rule, in which case Spack will use the first
+legal concretization rule. Going back to the example, if a user
+requests gperftools 2.3 or later, then Spack will install version 2.4
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always
+take preference over unlisted concretizations. In the above example,
+xlc isn't listed in the compiler list. Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``provider`` section differs slightly from other
+concretization rules. A provider lists a value that packages may
+``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
+dependency.
+
+
+
Profiling
-~~~~~~~~~~~~~~~~~~~~~
+------------------
Spack has some limited built-in support for profiling, and can report
statistics using standard Python timing tools. To use this feature,
@@ -133,7 +193,7 @@ supply ``-p`` to Spack on the command line, before any subcommands.
.. _spack-p:
``spack -p``
-^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~
``spack -p`` output looks like this:
diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst
index 0998ba8da4..27a3b4b435 100644
--- a/lib/spack/docs/features.rst
+++ b/lib/spack/docs/features.rst
@@ -31,14 +31,21 @@ platform, all on the command line.
# Specify a compiler (and its version), with %
$ spack install mpileaks@1.1.2 %gcc@4.7.3
- # Add special compile-time options with +
+ # Add special compile-time options by name
+ $ spack install mpileaks@1.1.2 %gcc@4.7.3 debug=True
+
+ # Add special boolean compile-time options with +
$ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug
- # Cross-compile for a different architecture with =
- $ spack install mpileaks@1.1.2 =bgqos_0
+ # Add compiler flags using the conventional names
+ $ spack install mpileaks@1.1.2 %gcc@4.7.3 cppflags=\"-O3 -floop-block\"
+
+ # Cross-compile for a different architecture with arch=
+ $ spack install mpileaks@1.1.2 arch=bgqos_0
-Users can specify as many or few options as they care about. Spack
-will fill in the unspecified values with sensible defaults.
+Users can specify as many or few options as they care about. Spack
+will fill in the unspecified values with sensible defaults. The two listed
+syntaxes for variants are identical when the value is boolean.
Customize dependencies
diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst
index d6ce52b747..98ed9ff0fe 100644
--- a/lib/spack/docs/index.rst
+++ b/lib/spack/docs/index.rst
@@ -47,7 +47,7 @@ Table of Contents
basic_usage
packaging_guide
mirrors
- site_configuration
+ configuration
developer_guide
command_index
package_list
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index f7f39d0d7b..54b886310a 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -1221,11 +1221,13 @@ just as easily provide a version range:
depends_on("libelf@0.8.2:0.8.4:")
-Or a requirement for a particular variant:
+Or a requirement for a particular variant or compiler flags:
.. code-block:: python
depends_on("libelf@0.8+debug")
+ depends_on('libelf debug=True')
+ depends_on('libelf cppflags="-fPIC")
Both users *and* package authors can use the same spec syntax to refer
to different package configurations. Users use the spec syntax on the
@@ -1623,21 +1625,21 @@ the user runs ``spack install`` and the time the ``install()`` method
is called. The concretized version of the spec above might look like
this::
- mpileaks@2.3%gcc@4.7.3=linux-ppc64
- ^callpath@1.0%gcc@4.7.3+debug=linux-ppc64
- ^dyninst@8.1.2%gcc@4.7.3=linux-ppc64
- ^libdwarf@20130729%gcc@4.7.3=linux-ppc64
- ^libelf@0.8.11%gcc@4.7.3=linux-ppc64
- ^mpich@3.0.4%gcc@4.7.3=linux-ppc64
+ mpileaks@2.3%gcc@4.7.3 arch=linux-ppc64
+ ^callpath@1.0%gcc@4.7.3+debug arch=linux-ppc64
+ ^dyninst@8.1.2%gcc@4.7.3 arch=linux-ppc64
+ ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64
+ ^libelf@0.8.11%gcc@4.7.3 arch=linux-ppc64
+ ^mpich@3.0.4%gcc@4.7.3 arch=linux-ppc64
.. graphviz::
digraph {
- "mpileaks@2.3\n%gcc@4.7.3\n=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n=linux-ppc64"
- "mpileaks@2.3\n%gcc@4.7.3\n=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n=linux-ppc64"
- "callpath@1.0\n%gcc@4.7.3+debug\n=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64"
- "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n=linux-ppc64"
- "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n=linux-ppc64"
+ "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64"
+ "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64"
+ "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64"
+ "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64"
+ "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64"
}
Here, all versions, compilers, and platforms are filled in, and there
@@ -1648,8 +1650,8 @@ point will Spack call the ``install()`` method for your package.
Concretization in Spack is based on certain selection policies that
tell Spack how to select, e.g., a version, when one is not specified
explicitly. Concretization policies are discussed in more detail in
-:ref:`site-configuration`. Sites using Spack can customize them to
-match the preferences of their own users.
+:ref:`configuration`. Sites using Spack can customize them to match
+the preferences of their own users.
.. _spack-spec:
@@ -1666,9 +1668,9 @@ running ``spack spec``. For example:
^libdwarf
^libelf
- dyninst@8.0.1%gcc@4.7.3=linux-ppc64
- ^libdwarf@20130729%gcc@4.7.3=linux-ppc64
- ^libelf@0.8.13%gcc@4.7.3=linux-ppc64
+ dyninst@8.0.1%gcc@4.7.3 arch=linux-ppc64
+ ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64
+ ^libelf@0.8.13%gcc@4.7.3 arch=linux-ppc64
This is useful when you want to know exactly what Spack will do when
you ask for a particular spec.
@@ -1682,60 +1684,8 @@ be concretized on their system. For example, one user may prefer packages
built with OpenMPI and the Intel compiler. Another user may prefer
packages be built with MVAPICH and GCC.
-Spack can be configured to prefer certain compilers, package
-versions, depends_on, and variants during concretization.
-The preferred configuration can be controlled via the
-``~/.spack/packages.yaml`` file for user configuations, or the
-``etc/spack/packages.yaml`` site configuration.
-
-
-Here's an example packages.yaml file that sets preferred packages:
-
-.. code-block:: sh
-
- packages:
- dyninst:
- compiler: [gcc@4.9]
- variants: +debug
- gperftools:
- version: [2.2, 2.4, 2.3]
- all:
- compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi]
- providers:
- mpi: [mvapich, mpich, openmpi]
-
-
-At a high level, this example is specifying how packages should be
-concretized. The dyninst package should prefer using gcc 4.9 and
-be built with debug options. The gperftools package should prefer version
-2.2 over 2.4. Every package on the system should prefer mvapich for
-its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9).
-These options are used to fill in implicit defaults. Any of them can be overwritten
-on the command line if explicitly requested.
-
-Each packages.yaml file begins with the string ``packages:`` and
-package names are specified on the next level. The special string ``all``
-applies settings to each package. Underneath each package name is
-one or more components: ``compiler``, ``variants``, ``version``,
-or ``providers``. Each component has an ordered list of spec
-``constraints``, with earlier entries in the list being preferred over
-later entries.
-
-Sometimes a package installation may have constraints that forbid
-the first concretization rule, in which case Spack will use the first
-legal concretization rule. Going back to the example, if a user
-requests gperftools 2.3 or later, then Spack will install version 2.4
-as the 2.4 version of gperftools is preferred over 2.3.
-
-An explicit concretization rule in the preferred section will always
-take preference over unlisted concretizations. In the above example,
-xlc isn't listed in the compiler list. Every listed compiler from
-gcc to pgi will thus be preferred over the xlc compiler.
-
-The syntax for the ``provider`` section differs slightly from other
-concretization rules. A provider lists a value that packages may
-``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
-dependency.
+See the `documentation in the config section <concretization-preferences_>`_
+for more details.
.. _install-method:
@@ -1960,6 +1910,12 @@ the command line.
``$rpath_flag`` can be overriden on a compiler specific basis in
``lib/spack/spack/compilers/$compiler.py``.
+The compiler wrappers also pass the compiler flags specified by the user from
+the command line (``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``,
+and/or ``ldlibs``). They do not override the canonical autotools flags with the
+same names (but in ALL-CAPS) that may be passed into the build by particularly
+challenging package scripts.
+
Compiler flags
~~~~~~~~~~~~~~
In rare circumstances such as compiling and running small unit tests, a package
@@ -2219,12 +2175,12 @@ example:
def install(self, prefix):
# Do default install
- @when('=chaos_5_x86_64_ib')
+ @when('arch=chaos_5_x86_64_ib')
def install(self, prefix):
# This will be executed instead of the default install if
# the package's sys_type() is chaos_5_x86_64_ib.
- @when('=bgqos_0")
+ @when('arch=bgqos_0")
def install(self, prefix):
# This will be executed if the package's sys_type is bgqos_0
@@ -2814,11 +2770,11 @@ build it:
$ spack stage libelf
==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz
######################################################################## 100.0%
- ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13.tar.gz
- ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64.
+ ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13.tar.gz
+ ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64.
$ spack cd libelf
$ pwd
- /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13
+ /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13
``spack cd`` here changed he current working directory to the
directory containing the expanded ``libelf`` source code. There are a
diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index b9b79f83a3..bf98b4c354 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -55,7 +55,10 @@ parameters=(
# The compiler input variables are checked for sanity later:
# SPACK_CC, SPACK_CXX, SPACK_F77, SPACK_FC
-# Debug flag is optional; set to "TRUE" for debug logging:
+# The default compiler flags are passed from these variables:
+# SPACK_CFLAGS, SPACK_CXXFLAGS, SPACK_FCFLAGS, SPACK_FFLAGS,
+# SPACK_LDFLAGS, SPACK_LDLIBS
+# Debug env var is optional; set to true for debug logging:
# SPACK_DEBUG
# Test command is used to unit test the compiler script.
# SPACK_TEST_COMMAND
@@ -99,21 +102,25 @@ case "$command" in
command="$SPACK_CC"
language="C"
comp="CC"
+ lang_flags=C
;;
c++|CC|g++|clang++|icpc|pgc++|xlc++)
command="$SPACK_CXX"
language="C++"
comp="CXX"
+ lang_flags=CXX
;;
f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor)
command="$SPACK_FC"
language="Fortran 90"
comp="FC"
+ lang_flags=F
;;
f77|gfortran|ifort|pgfortran|xlf|nagfor)
command="$SPACK_F77"
language="Fortran 77"
comp="F77"
+ lang_flags=F
;;
ld)
mode=ld
@@ -131,7 +138,7 @@ if [[ -z $mode ]]; then
if [[ $arg == -v || $arg == -V || $arg == --version || $arg == -dumpversion ]]; then
mode=vcheck
break
- fi
+ fi
done
fi
@@ -167,6 +174,28 @@ if [[ -z $command ]]; then
die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs."
fi
+#
+# Filter '.' and Spack environment directories out of PATH so that
+# this script doesn't just call itself
+#
+IFS=':' read -ra env_path <<< "$PATH"
+IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH"
+spack_env_dirs+=("" ".")
+PATH=""
+for dir in "${env_path[@]}"; do
+ addpath=true
+ for env_dir in "${spack_env_dirs[@]}"; do
+ if [[ $dir == $env_dir ]]; then
+ addpath=false
+ break
+ fi
+ done
+ if $addpath; then
+ PATH="${PATH:+$PATH:}$dir"
+ fi
+done
+export PATH
+
if [[ $mode == vcheck ]]; then
exec ${command} "$@"
fi
@@ -188,6 +217,42 @@ fi
input_command="$@"
args=("$@")
+# Prepend cppflags, cflags, cxxflags, fcflags, fflags, and ldflags
+
+# Add ldflags
+case "$mode" in
+ ld|ccld)
+ args=(${SPACK_LDFLAGS[@]} "${args[@]}") ;;
+esac
+
+# Add compiler flags.
+case "$mode" in
+ cc|ccld)
+ # Add c, cxx, fc, and f flags
+ case $lang_flags in
+ C)
+ args=(${SPACK_CFLAGS[@]} "${args[@]}") ;;
+ CXX)
+ args=(${SPACK_CXXFLAGS[@]} "${args[@]}") ;;
+ esac
+ ;;
+esac
+
+# Add cppflags
+case "$mode" in
+ cpp|as|cc|ccld)
+ args=(${SPACK_CPPFLAGS[@]} "${args[@]}") ;;
+esac
+
+case "$mode" in cc|ccld)
+ # Add fortran flags
+ case $lang_flags in
+ F)
+ args=(${SPACK_FFLAGS[@]} "${args[@]}") ;;
+ esac
+ ;;
+esac
+
# Read spack dependencies from the path environment variable
IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES"
for dep in "${deps[@]}"; do
@@ -230,6 +295,12 @@ elif [[ $mode == ld ]]; then
$add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "${args[@]}")
fi
+# Add SPACK_LDLIBS to args
+case "$mode" in
+ ld|ccld)
+ args=("${args[@]}" ${SPACK_LDLIBS[@]}) ;;
+esac
+
#
# Unset pesky environment variables that could affect build sanity.
#
@@ -237,28 +308,6 @@ unset LD_LIBRARY_PATH
unset LD_RUN_PATH
unset DYLD_LIBRARY_PATH
-#
-# Filter '.' and Spack environment directories out of PATH so that
-# this script doesn't just call itself
-#
-IFS=':' read -ra env_path <<< "$PATH"
-IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH"
-spack_env_dirs+=("" ".")
-PATH=""
-for dir in "${env_path[@]}"; do
- addpath=true
- for env_dir in "${spack_env_dirs[@]}"; do
- if [[ $dir == $env_dir ]]; then
- addpath=false
- break
- fi
- done
- if $addpath; then
- PATH="${PATH:+$PATH:}$dir"
- fi
-done
-export PATH
-
full_command=("$command" "${args[@]}")
# In test command mode, write out full command for Spack tests.
@@ -275,8 +324,8 @@ fi
if [[ $SPACK_DEBUG == TRUE ]]; then
input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log"
output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log"
- echo "[$mode] $command $input_command" >> $input_log
- echo "[$mode] ${full_command[@]}" >> $output_log
+ echo "[$mode] $command $input_command" >> "$input_log"
+ echo "[$mode] ${full_command[@]}" >> "$output_log"
fi
exec "${full_command[@]}"
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 6661a80f27..d72e8bae92 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -22,28 +22,28 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-__all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree',
- 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
- 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
- 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
- 'set_executable', 'copy_mode', 'unset_executable_mode',
- 'remove_dead_links', 'remove_linked_tree', 'find_library_path',
- 'fix_darwin_install_name']
-
import os
import glob
-import sys
import re
import shutil
import stat
import errno
import getpass
from contextlib import contextmanager, closing
-from tempfile import NamedTemporaryFile
import subprocess
import llnl.util.tty as tty
-from spack.util.compression import ALLOWED_ARCHIVE_TYPES
+
+__all__ = ['set_install_permissions', 'install', 'install_tree',
+ 'traverse_tree',
+ 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
+ 'force_remove', 'join_path', 'ancestor', 'can_access',
+ 'filter_file',
+ 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
+ 'set_executable', 'copy_mode', 'unset_executable_mode',
+ 'remove_dead_links', 'remove_linked_tree', 'find_library_path',
+ 'fix_darwin_install_name', 'to_link_flags']
+
def filter_file(regex, repl, *filenames, **kwargs):
"""Like sed, but uses python regular expressions.
@@ -69,6 +69,7 @@ def filter_file(regex, repl, *filenames, **kwargs):
# Allow strings to use \1, \2, etc. for replacement, like sed
if not callable(repl):
unescaped = repl.replace(r'\\', '\\')
+
def replace_groups_with_groupid(m):
def groupid_to_group(x):
return m.group(int(x.group(1)))
@@ -157,9 +158,12 @@ def set_install_permissions(path):
def copy_mode(src, dest):
src_mode = os.stat(src).st_mode
dest_mode = os.stat(dest).st_mode
- if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
- if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
- if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
+ if src_mode & stat.S_IXUSR:
+ dest_mode |= stat.S_IXUSR
+ if src_mode & stat.S_IXGRP:
+ dest_mode |= stat.S_IXGRP
+ if src_mode & stat.S_IXOTH:
+ dest_mode |= stat.S_IXOTH
os.chmod(dest, dest_mode)
@@ -224,9 +228,10 @@ def force_remove(*paths):
for path in paths:
try:
os.remove(path)
- except OSError, e:
+ except OSError:
pass
+
@contextmanager
def working_dir(dirname, **kwargs):
if kwargs.get('create', False):
@@ -240,7 +245,7 @@ def working_dir(dirname, **kwargs):
def touch(path):
"""Creates an empty file at the specified path."""
- with open(path, 'a') as file:
+ with open(path, 'a'):
os.utime(path, None)
@@ -253,7 +258,7 @@ def touchp(path):
def force_symlink(src, dest):
try:
os.symlink(src, dest)
- except OSError as e:
+ except OSError:
os.remove(dest)
os.symlink(src, dest)
@@ -275,7 +280,7 @@ def ancestor(dir, n=1):
def can_access(file_name):
"""True if we have read/write access to the file."""
- return os.access(file_name, os.R_OK|os.W_OK)
+ return os.access(file_name, os.R_OK | os.W_OK)
def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
@@ -343,13 +348,14 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
# Treat as a directory
if os.path.isdir(source_child) and (
- follow_links or not os.path.islink(source_child)):
+ follow_links or not os.path.islink(source_child)):
# When follow_nonexisting isn't set, don't descend into dirs
# in source that do not exist in dest
if follow_nonexisting or os.path.exists(dest_child):
- tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs)
- for t in tuples: yield t
+ tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) # NOQA: ignore=E501
+ for t in tuples:
+ yield t
# Treat as a file.
elif not ignore(os.path.join(rel_path, f)):
@@ -379,6 +385,7 @@ def remove_dead_links(root):
if not os.path.exists(real_path):
os.unlink(path)
+
def remove_linked_tree(path):
"""
Removes a directory and its contents. If the directory is a
@@ -402,28 +409,41 @@ def fix_darwin_install_name(path):
Fix install name of dynamic libraries on Darwin to have full path.
There are two parts of this task:
(i) use install_name('-id',...) to change install name of a single lib;
- (ii) use install_name('-change',...) to change the cross linking between libs.
- The function assumes that all libraries are in one folder and currently won't
- follow subfolders.
+ (ii) use install_name('-change',...) to change the cross linking between
+ libs. The function assumes that all libraries are in one folder and
+ currently won't follow subfolders.
Args:
path: directory in which .dylib files are alocated
"""
- libs = glob.glob(join_path(path,"*.dylib"))
+ libs = glob.glob(join_path(path, "*.dylib"))
for lib in libs:
# fix install name first:
- subprocess.Popen(["install_name_tool", "-id",lib,lib], stdout=subprocess.PIPE).communicate()[0]
- long_deps = subprocess.Popen(["otool", "-L",lib], stdout=subprocess.PIPE).communicate()[0].split('\n')
+ subprocess.Popen(["install_name_tool", "-id", lib, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501
+ long_deps = subprocess.Popen(["otool", "-L", lib], stdout=subprocess.PIPE).communicate()[0].split('\n') # NOQA: ignore=E501
deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]]
# fix all dependencies:
for dep in deps:
for loc in libs:
if dep == os.path.basename(loc):
- subprocess.Popen(["install_name_tool", "-change",dep,loc,lib], stdout=subprocess.PIPE).communicate()[0]
+ subprocess.Popen(["install_name_tool", "-change", dep, loc, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501
break
+def to_link_flags(library):
+ """Transforms a path to a <library> into linking flags -L<dir> -l<name>.
+
+ Return:
+ A string of linking flags.
+ """
+ dir = os.path.dirname(library)
+ # Asume libXYZ.suffix
+ name = os.path.basename(library)[3:].split(".")[0]
+ res = '-L%s -l%s' % (dir, name)
+ return res
+
+
def find_library_path(libname, *paths):
"""Searches for a file called <libname> in each path.
diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py
index c638b113fd..ee81e11a20 100644
--- a/lib/spack/llnl/util/tty/__init__.py
+++ b/lib/spack/llnl/util/tty/__init__.py
@@ -64,12 +64,14 @@ def info(message, *args, **kwargs):
format = kwargs.get('format', '*b')
stream = kwargs.get('stream', sys.stdout)
wrap = kwargs.get('wrap', False)
+ break_long_words = kwargs.get('break_long_words', False)
cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream)
for arg in args:
if wrap:
lines = textwrap.wrap(
- str(arg), initial_indent=indent, subsequent_indent=indent)
+ str(arg), initial_indent=indent, subsequent_indent=indent,
+ break_long_words=break_long_words)
for line in lines:
stream.write(line + '\n')
else:
diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py
index 429ba45882..81a83691d7 100644
--- a/lib/spack/llnl/util/tty/colify.py
+++ b/lib/spack/llnl/util/tty/colify.py
@@ -198,8 +198,13 @@ def colify(elts, **options):
for col in xrange(cols):
elt = col * rows + row
width = config.widths[col] + cextra(elts[elt])
- fmt = '%%-%ds' % width
- output.write(fmt % elts[elt])
+ if col < cols - 1:
+ fmt = '%%-%ds' % width
+ output.write(fmt % elts[elt])
+ else:
+ # Don't pad the rightmost column (sapces can wrap on
+ # small teriminals if one line is overlong)
+ output.write(elts[elt])
output.write("\n")
row += 1
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index 164340bf0f..75ddca1abc 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -39,7 +39,9 @@ spack_file = join_path(spack_root, "bin", "spack")
lib_path = join_path(spack_root, "lib", "spack")
build_env_path = join_path(lib_path, "env")
module_path = join_path(lib_path, "spack")
+platform_path = join_path(module_path, 'platforms')
compilers_path = join_path(module_path, "compilers")
+operating_system_path = join_path(module_path, 'operating_systems')
test_path = join_path(module_path, "test")
hooks_path = join_path(module_path, "hooks")
var_path = join_path(spack_root, "var", "spack")
@@ -105,7 +107,7 @@ concretizer = DefaultConcretizer()
# Version information
from spack.version import Version
-spack_version = Version("0.8.15")
+spack_version = Version("0.9.1")
#
# Executables used by Spack
diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py
index 91d1d2003d..38cff62af4 100644
--- a/lib/spack/spack/abi.py
+++ b/lib/spack/spack/abi.py
@@ -35,8 +35,9 @@ class ABI(object):
The current implementation is rather rough and could be improved."""
def architecture_compatible(self, parent, child):
- """Returns true iff the parent and child specs have ABI compatible architectures."""
- return not parent.architecture or not child.architecture or parent.architecture == child.architecture
+ """Returns true iff the parent and child specs have ABI compatible targets."""
+ return not parent.architecture or not child.architecture \
+ or parent.architecture == child.architecture
@memoized
diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py
index b14cb2bea2..cbac7b41d6 100644
--- a/lib/spack/spack/architecture.py
+++ b/lib/spack/spack/architecture.py
@@ -22,68 +22,498 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+"""
+This module contains all the elements that are required to create an
+architecture object. These include, the target processor, the operating system,
+and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes.
+
+On a multiple architecture machine, the architecture spec field can be set to
+build a package against any target and operating system that is present on the
+platform. On Cray platforms or any other architecture that has different front
+and back end environments, the operating system will determine the method of
+compiler
+detection.
+
+There are two different types of compiler detection:
+ 1. Through the $PATH env variable (front-end detection)
+ 2. Through the tcl module system. (back-end detection)
+
+Depending on which operating system is specified, the compiler will be detected
+using one of those methods.
+
+For platforms such as linux and darwin, the operating system is autodetected
+and the target is set to be x86_64.
+
+The command line syntax for specifying an architecture is as follows:
+
+ target=<Target name> os=<OperatingSystem name>
+
+If the user wishes to use the defaults, either target or os can be left out of
+the command line and Spack will concretize using the default. These defaults
+are set in the 'platforms/' directory which contains the different subclasses
+for platforms. If the machine has multiple architectures, the user can
+also enter front-end, or fe or back-end or be. These settings will concretize
+to their respective front-end and back-end targets and operating systems.
+Additional platforms can be added by creating a subclass of Platform
+and adding it inside the platform directory.
+
+Platforms are an abstract class that are extended by subclasses. If the user
+wants to add a new type of platform (such as cray_xe), they can create a
+subclass and set all the class attributes such as priority, front_target,
+back_target, front_os, back_os. Platforms also contain a priority class
+attribute. A lower number signifies higher priority. These numbers are
+arbitrarily set and can be changed though often there isn't much need unless a
+new platform is added and the user wants that to be detected first.
+
+Targets are created inside the platform subclasses. Most architecture
+(like linux, and darwin) will have only one target (x86_64) but in the case of
+Cray machines, there is both a frontend and backend processor. The user can
+specify which targets are present on front-end and back-end architecture
+
+Depending on the platform, operating systems are either auto-detected or are
+set. The user can set the front-end and back-end operating setting by the class
+attributes front_os and back_os. The operating system as described earlier,
+will be responsible for compiler detection.
+"""
import os
-import re
-import platform
+import imp
+import inspect
-from llnl.util.lang import memoized
+from llnl.util.lang import memoized, list_modules, key_ordering
+from llnl.util.filesystem import join_path
+import llnl.util.tty as tty
import spack
+import spack.compilers
+from spack.util.naming import mod_to_class
+from spack.util.environment import get_path
+from spack.util.multiproc import parmap
import spack.error as serr
class InvalidSysTypeError(serr.SpackError):
def __init__(self, sys_type):
- super(InvalidSysTypeError,
- self).__init__("Invalid sys_type value for Spack: " + sys_type)
+ super(InvalidSysTypeError, self).__init__(
+ "Invalid sys_type value for Spack: " + sys_type)
class NoSysTypeError(serr.SpackError):
def __init__(self):
- super(NoSysTypeError,
- self).__init__("Could not determine sys_type for this machine.")
+ super(NoSysTypeError, self).__init__(
+ "Could not determine sys_type for this machine.")
+
+
+@key_ordering
+class Target(object):
+ """ Target is the processor of the host machine.
+ The host machine may have different front-end and back-end targets,
+ especially if it is a Cray machine. The target will have a name and
+ also the module_name (e.g craype-compiler). Targets will also
+ recognize which platform they came from using the set_platform method.
+ Targets will have compiler finding strategies
+ """
+
+ def __init__(self, name, module_name=None):
+ self.name = name # case of cray "ivybridge" but if it's x86_64
+ self.module_name = module_name # craype-ivybridge
+
+ # Sets only the platform name to avoid recursiveness
+
+ def _cmp_key(self):
+ return (self.name, self.module_name)
+
+ def __repr__(self):
+ return self.__str__()
+
+ def __str__(self):
+ return self.name
+
+
+@key_ordering
+class Platform(object):
+ """ Abstract class that each type of Platform will subclass.
+ Will return a instance of it once it
+ is returned
+ """
+
+ priority = None # Subclass sets number. Controls detection order
+ front_end = None
+ back_end = None
+ default = None # The default back end target. On cray ivybridge
+
+ front_os = None
+ back_os = None
+ default_os = None
+
+ def __init__(self, name):
+ self.targets = {}
+ self.operating_sys = {}
+ self.name = name
+
+ def add_target(self, name, target):
+ """Used by the platform specific subclass to list available targets.
+ Raises an error if the platform specifies a name
+ that is reserved by spack as an alias.
+ """
+ if name in ['frontend', 'fe', 'backend', 'be', 'default_target']:
+ raise ValueError(
+ "%s is a spack reserved alias "
+ "and cannot be the name of a target" % name)
+ self.targets[name] = target
+
+ def target(self, name):
+ """This is a getter method for the target dictionary
+ that handles defaulting based on the values provided by default,
+ front-end, and back-end. This can be overwritten
+ by a subclass for which we want to provide further aliasing options.
+ """
+ if name == 'default_target':
+ name = self.default
+ elif name == 'frontend' or name == 'fe':
+ name = self.front_end
+ elif name == 'backend' or name == 'be':
+ name = self.back_end
+
+ return self.targets.get(name, None)
+
+ def add_operating_system(self, name, os_class):
+ """ Add the operating_system class object into the
+ platform.operating_sys dictionary
+ """
+ if name in ['frontend', 'fe', 'backend', 'be', 'default_os']:
+ raise ValueError(
+ "%s is a spack reserved alias "
+ "and cannot be the name of an OS" % name)
+ self.operating_sys[name] = os_class
+
+ def operating_system(self, name):
+ if name == 'default_os':
+ name = self.default_os
+ if name == 'frontend' or name == "fe":
+ name = self.front_os
+ if name == 'backend' or name == 'be':
+ name = self.back_os
+
+ return self.operating_sys.get(name, None)
+
+
+ @classmethod
+ def detect(self):
+ """ Subclass is responsible for implementing this method.
+ Returns True if the Platform class detects that
+ it is the current platform
+ and False if it's not.
+ """
+ raise NotImplementedError()
+
+
+ def __repr__(self):
+ return self.__str__()
+
+
+ def __str__(self):
+ return self.name
+
+
+ def _cmp_key(self):
+ t_keys = ''.join(str(t._cmp_key()) for t in
+ sorted(self.targets.values()))
+ o_keys = ''.join(str(o._cmp_key()) for o in
+ sorted(self.operating_sys.values()))
+ return (self.name,
+ self.default,
+ self.front_end,
+ self.back_end,
+ self.default_os,
+ self.front_os,
+ self.back_os,
+ t_keys,
+ o_keys)
+
+
+@key_ordering
+class OperatingSystem(object):
+ """ Operating System will be like a class similar to platform extended
+ by subclasses for the specifics. Operating System will contain the
+ compiler finding logic. Instead of calling two separate methods to
+ find compilers we call find_compilers method for each operating system
+ """
+
+ def __init__(self, name, version):
+ self.name = name
+ self.version = version
+
+ def __str__(self):
+ return self.name + self.version
+
+ def __repr__(self):
+ return self.__str__()
+
+ def _cmp_key(self):
+ return (self.name, self.version)
+ def find_compilers(self, *paths):
+ """
+ Return a list of compilers found in the suppied paths.
+ This invokes the find() method for each Compiler class,
+ and appends the compilers detected to a list.
+ """
+ if not paths:
+ paths = get_path('PATH')
+ # Make sure path elements exist, and include /bin directories
+ # under prefixes.
+ filtered_path = []
+ for p in paths:
+ # Eliminate symlinks and just take the real directories.
+ p = os.path.realpath(p)
+ if not os.path.isdir(p):
+ continue
+ filtered_path.append(p)
-def get_sys_type_from_spack_globals():
- """Return the SYS_TYPE from spack globals, or None if it isn't set."""
- if not hasattr(spack, "sys_type"):
- return None
- elif hasattr(spack.sys_type, "__call__"):
- return spack.sys_type()
+ # Check for a bin directory, add it if it exists
+ bin = join_path(p, 'bin')
+ if os.path.isdir(bin):
+ filtered_path.append(os.path.realpath(bin))
+
+ # Once the paths are cleaned up, do a search for each type of
+ # compiler. We can spawn a bunch of parallel searches to reduce
+ # the overhead of spelunking all these directories.
+ types = spack.compilers.all_compiler_types()
+ compiler_lists = parmap(lambda cmp_cls:
+ self.find_compiler(cmp_cls, *filtered_path),
+ types)
+
+ # ensure all the version calls we made are cached in the parent
+ # process, as well. This speeds up Spack a lot.
+ clist = reduce(lambda x, y: x+y, compiler_lists)
+ return clist
+
+ def find_compiler(self, cmp_cls, *path):
+ """Try to find the given type of compiler in the user's
+ environment. For each set of compilers found, this returns
+ compiler objects with the cc, cxx, f77, fc paths and the
+ version filled in.
+
+ This will search for compilers with the names in cc_names,
+ cxx_names, etc. and it will group them if they have common
+ prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would
+ be grouped with g++-mp-4.7 and gfortran-mp-4.7.
+ """
+ dicts = parmap(
+ lambda t: cmp_cls._find_matches_in_path(*t),
+ [(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path),
+ (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path),
+ (cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path),
+ (cmp_cls.fc_names, cmp_cls.fc_version) + tuple(path)])
+
+ all_keys = set()
+ for d in dicts:
+ all_keys.update(d)
+
+ compilers = {}
+ for k in all_keys:
+ ver, pre, suf = k
+
+ # Skip compilers with unknown version.
+ if ver == 'unknown':
+ continue
+
+ paths = tuple(pn[k] if k in pn else None for pn in dicts)
+ spec = spack.spec.CompilerSpec(cmp_cls.name, ver)
+
+ if ver in compilers:
+ prev = compilers[ver]
+
+ # prefer the one with more compilers.
+ prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
+ newcount = len([p for p in paths if p is not None])
+ prevcount = len([p for p in prev_paths if p is not None])
+
+ # Don't add if it's not an improvement over prev compiler.
+ if newcount <= prevcount:
+ continue
+
+ compilers[ver] = cmp_cls(spec, self, paths)
+
+ return list(compilers.values())
+
+ def to_dict(self):
+ d = {}
+ d['name'] = self.name
+ d['version'] = self.version
+ return d
+
+@key_ordering
+class Arch(object):
+ "Architecture is now a class to help with setting attributes"
+
+ def __init__(self, platform=None, platform_os=None, target=None):
+ self.platform = platform
+ if platform and platform_os:
+ platform_os = self.platform.operating_system(platform_os)
+ self.platform_os = platform_os
+ if platform and target:
+ target = self.platform.target(target)
+ self.target = target
+
+ # Hooks for parser to use when platform is set after target or os
+ self.target_string = None
+ self.os_string = None
+
+ @property
+ def concrete(self):
+ return all((self.platform is not None,
+ isinstance(self.platform, Platform),
+ self.platform_os is not None,
+ isinstance(self.platform_os, OperatingSystem),
+ self.target is not None, isinstance(self.target, Target)))
+
+ def __str__(self):
+ if self.platform or self.platform_os or self.target:
+ if self.platform.name == 'darwin':
+ os_name = self.platform_os.name if self.platform_os else "None"
+ else:
+ os_name = str(self.platform_os)
+
+ return (str(self.platform) + "-" +
+ os_name + "-" + str(self.target))
+ else:
+ return ''
+
+
+ def __contains__(self, string):
+ return string in str(self)
+
+
+ def _cmp_key(self):
+ if isinstance(self.platform, Platform):
+ platform = self.platform.name
+ else:
+ platform = self.platform
+ if isinstance(self.platform_os, OperatingSystem):
+ platform_os = self.platform_os.name
+ else:
+ platform_os = self.platform_os
+ if isinstance(self.target, Target):
+ target = self.target.name
+ else:
+ target = self.target
+ return (platform, platform_os, target)
+
+ def to_dict(self):
+ d = {}
+ d['platform'] = str(self.platform) if self.platform else None
+ d['platform_os'] = str(self.platform_os) if self.platform_os else None
+ d['target'] = str(self.target) if self.target else None
+
+ return d
+
+
+def _target_from_dict(target_name, platform=None):
+ """ Creates new instance of target and assigns all the attributes of
+ that target from the dictionary
+ """
+ if not platform:
+ platform = sys_type()
+ return platform.target(target_name)
+
+
+def _operating_system_from_dict(os_name, platform=None):
+ """ uses platform's operating system method to grab the constructed
+ operating systems that are valid on the platform.
+ """
+ if not platform:
+ platform = sys_type()
+ if isinstance(os_name, dict):
+ name = os_name['name']
+ version = os_name['version']
+ return platform.operating_system(name+version)
else:
- return spack.sys_type
+ return platform.operating_system(os_name)
+
+def _platform_from_dict(platform_name):
+ """ Constructs a platform from a dictionary. """
+ platform_list = all_platforms()
+ for p in platform_list:
+ if platform_name.replace("_", "").lower() == p.__name__.lower():
+ return p()
-def get_sys_type_from_environment():
- """Return $SYS_TYPE or None if it's not defined."""
- return os.environ.get('SYS_TYPE')
+def arch_from_dict(d):
+ """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict
+ helper methods to recreate the arch tuple from the dictionary read from
+ a yaml file
+ """
+ arch = Arch()
-def get_sys_type_from_platform():
- """Return the architecture from Python's platform module."""
- sys_type = platform.system() + '-' + platform.machine()
- sys_type = re.sub(r'[^\w-]', '_', sys_type)
- return sys_type.lower()
+ if isinstance(d, basestring):
+ # We have an old spec using a string for the architecture
+ arch.platform = Platform('spack_compatibility')
+ arch.platform_os = OperatingSystem('unknown', '')
+ arch.target = Target(d)
+
+ arch.os_string = None
+ arch.target_string = None
+ else:
+ if d is None:
+ return None
+ platform_name = d['platform']
+ os_name = d['platform_os']
+ target_name = d['target']
+
+ if platform_name:
+ arch.platform = _platform_from_dict(platform_name)
+ else:
+ arch.platform = None
+ if target_name:
+ arch.target = _target_from_dict(target_name, arch.platform)
+ else:
+ arch.target = None
+ if os_name:
+ arch.platform_os = _operating_system_from_dict(os_name,
+ arch.platform)
+ else:
+ arch.platform_os = None
+
+ arch.os_string = None
+ arch.target_string = None
+
+ return arch
@memoized
-def sys_type():
- """Returns a SysType for the current machine."""
- methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment,
- get_sys_type_from_platform]
+def all_platforms():
+ classes = []
+ mod_path = spack.platform_path
+ parent_module = "spack.platforms"
+
+ for name in list_modules(mod_path):
+ mod_name = '%s.%s' % (parent_module, name)
+ class_name = mod_to_class(name)
+ mod = __import__(mod_name, fromlist=[class_name])
+ if not hasattr(mod, class_name):
+ tty.die('No class %s defined in %s' % (class_name, mod_name))
+ cls = getattr(mod, class_name)
+ if not inspect.isclass(cls):
+ tty.die('%s.%s is not a class' % (mod_name, class_name))
+
+ classes.append(cls)
- # search for a method that doesn't return None
- sys_type = None
- for method in methods:
- sys_type = method()
- if sys_type:
- break
+ return classes
- # Couldn't determine the sys_type for this machine.
- if sys_type is None:
- return "unknown_arch"
- if not isinstance(sys_type, basestring):
- raise InvalidSysTypeError(sys_type)
+@memoized
+def sys_type():
+ """ Gather a list of all available subclasses of platforms.
+ Sorts the list according to their priority looking. Priority is
+ an arbitrarily set number. Detects platform either using uname or
+ a file path (/opt/cray...)
+ """
+ # Try to create a Platform object using the config file FIRST
+ platform_list = all_platforms()
+ platform_list.sort(key=lambda a: a.priority)
- return sys_type
+ for platform in platform_list:
+ if platform.detect():
+ return platform()
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 5ce4cb1ce1..3fcfb151b8 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -51,15 +51,16 @@ There are two parts to the build environment:
Skimming this module is a nice way to get acquainted with the types of
calls you can make from within the install() function.
"""
-import multiprocessing
import os
-import platform
-import shutil
import sys
+import shutil
+import multiprocessing
+import platform
-import spack
import llnl.util.tty as tty
from llnl.util.filesystem import *
+
+import spack
from spack.environment import EnvironmentModifications, validate
from spack.util.environment import *
from spack.util.executable import Executable, which
@@ -74,20 +75,19 @@ SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE'
# set_build_environment_variables and used to pass parameters to
# Spack's compiler wrappers.
#
-SPACK_ENV_PATH = 'SPACK_ENV_PATH'
-SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES'
-SPACK_PREFIX = 'SPACK_PREFIX'
-SPACK_INSTALL = 'SPACK_INSTALL'
-SPACK_DEBUG = 'SPACK_DEBUG'
-SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
-SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
+SPACK_ENV_PATH = 'SPACK_ENV_PATH'
+SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES'
+SPACK_PREFIX = 'SPACK_PREFIX'
+SPACK_INSTALL = 'SPACK_INSTALL'
+SPACK_DEBUG = 'SPACK_DEBUG'
+SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
+SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
# Platform-specific library suffix.
dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so'
-
class MakeExecutable(Executable):
"""Special callable executable object for make so the user can
specify parallel or not on a per-invocation basis. Using
@@ -98,6 +98,7 @@ class MakeExecutable(Executable):
Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides
everything.
"""
+
def __init__(self, name, jobs):
super(MakeExecutable, self).__init__(name)
self.jobs = jobs
@@ -113,30 +114,95 @@ class MakeExecutable(Executable):
return super(MakeExecutable, self).__call__(*args, **kwargs)
+def load_module(mod):
+ """Takes a module name and removes modules until it is possible to
+ load that module. It then loads the provided module. Depends on the
+ modulecmd implementation of modules used in cray and lmod.
+ """
+ # Create an executable of the module command that will output python code
+ modulecmd = which('modulecmd')
+ modulecmd.add_default_arg('python')
+
+ # Read the module and remove any conflicting modules
+ # We do this without checking that they are already installed
+ # for ease of programming because unloading a module that is not
+ # loaded does nothing.
+ text = modulecmd('show', mod, output=str, error=str).split()
+ for i, word in enumerate(text):
+ if word == 'conflict':
+ exec(compile(modulecmd('unload', text[
+ i + 1], output=str, error=str), '<string>', 'exec'))
+ # Load the module now that there are no conflicts
+ load = modulecmd('load', mod, output=str, error=str)
+ exec(compile(load, '<string>', 'exec'))
+
+
+def get_path_from_module(mod):
+ """Inspects a TCL module for entries that indicate the absolute path
+ at which the library supported by said module can be found.
+ """
+ # Create a modulecmd executable
+ modulecmd = which('modulecmd')
+ modulecmd.add_default_arg('python')
+
+ # Read the module
+ text = modulecmd('show', mod, output=str, error=str).split('\n')
+ # If it lists its package directory, return that
+ for line in text:
+ if line.find(mod.upper() + '_DIR') >= 0:
+ words = line.split()
+ return words[2]
+
+ # If it lists a -rpath instruction, use that
+ for line in text:
+ rpath = line.find('-rpath/')
+ if rpath >= 0:
+ return line[rpath + 6:line.find('/lib')]
+
+ # If it lists a -L instruction, use that
+ for line in text:
+ L = line.find('-L/')
+ if L >= 0:
+ return line[L + 2:line.find('/lib')]
+
+ # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that
+ for line in text:
+ if line.find('LD_LIBRARY_PATH') >= 0:
+ words = line.split()
+ path = words[2]
+ return path[:path.find('/lib')]
+ # Unable to find module path
+ return None
+
+
def set_compiler_environment_variables(pkg, env):
- assert pkg.spec.concrete
+ assert(pkg.spec.concrete)
+ compiler = pkg.compiler
+ flags = pkg.spec.compiler_flags
+
# Set compiler variables used by CMake and autotools
- assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc'))
+ assert all(key in compiler.link_paths for key in (
+ 'cc', 'cxx', 'f77', 'fc'))
# Populate an object with the list of environment modifications
# and return it
- # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc.
+ # TODO : add additional kwargs for better diagnostics, like requestor,
+ # ttyout, ttyerr, etc.
link_dir = spack.build_env_path
- env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc']))
- env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx']))
- env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77']))
- env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc']))
# Set SPACK compiler variables so that our wrapper knows what to call
- compiler = pkg.compiler
if compiler.cc:
- env.set('SPACK_CC', compiler.cc)
+ env.set('SPACK_CC', compiler.cc)
+ env.set('CC', join_path(link_dir, compiler.link_paths['cc']))
if compiler.cxx:
env.set('SPACK_CXX', compiler.cxx)
+ env.set('CXX', join_path(link_dir, compiler.link_paths['cxx']))
if compiler.f77:
env.set('SPACK_F77', compiler.f77)
+ env.set('F77', join_path(link_dir, compiler.link_paths['f77']))
if compiler.fc:
env.set('SPACK_FC', compiler.fc)
+ env.set('FC', join_path(link_dir, compiler.link_paths['fc']))
# Set SPACK compiler rpath flags so that our wrapper knows what to use
env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg)
@@ -144,7 +210,17 @@ def set_compiler_environment_variables(pkg, env):
env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg)
env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg)
+ # Add every valid compiler flag to the environment, prefixed with "SPACK_"
+ for flag in spack.spec.FlagMap.valid_compiler_flags():
+ # Concreteness guarantees key safety here
+ if flags[flag] != []:
+ env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag]))
+
env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))
+
+ for mod in compiler.modules:
+ load_module(mod)
+
return env
@@ -163,7 +239,8 @@ def set_build_environment_variables(pkg, env):
# handled by putting one in the <build_env_path>/case-insensitive
# directory. Add that to the path too.
env_paths = []
- for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]:
+ compiler_specific = join_path(spack.build_env_path, pkg.compiler.name)
+ for item in [spack.build_env_path, compiler_specific]:
env_paths.append(item)
ci = join_path(item, 'case-insensitive')
if os.path.isdir(ci):
@@ -176,7 +253,8 @@ def set_build_environment_variables(pkg, env):
# Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES
dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)]
env.set_path(SPACK_DEPENDENCIES, dep_prefixes)
- env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH
+ # Add dependencies to CMAKE_PREFIX_PATH
+ env.set_path('CMAKE_PREFIX_PATH', dep_prefixes)
# Install prefix
env.set(SPACK_PREFIX, pkg.prefix)
@@ -192,7 +270,8 @@ def set_build_environment_variables(pkg, env):
env.unset('DYLD_LIBRARY_PATH')
# Add bin directories from dependencies to the PATH for the build.
- bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes]))
+ bin_dirs = reversed(
+ filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes]))
for item in bin_dirs:
env.prepend_path('PATH', item)
@@ -203,13 +282,15 @@ def set_build_environment_variables(pkg, env):
env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir)
# Add any pkgconfig directories to PKG_CONFIG_PATH
- pkg_config_dirs = []
- for p in dep_prefixes:
- for maybe in ('lib', 'lib64', 'share'):
- pcdir = join_path(p, maybe, 'pkgconfig')
+ for pre in dep_prefixes:
+ for directory in ('lib', 'lib64', 'share'):
+ pcdir = join_path(pre, directory, 'pkgconfig')
if os.path.isdir(pcdir):
- pkg_config_dirs.append(pcdir)
- env.set_path('PKG_CONFIG_PATH', pkg_config_dirs)
+ # pkg_config_dirs.append(pcdir)
+ env.prepend_path('PKG_CONFIG_PATH', pcdir)
+
+ if pkg.spec.architecture.target.module_name:
+ load_module(pkg.spec.architecture.target.module_name)
return env
@@ -229,7 +310,7 @@ def set_module_variables_for_package(pkg, module):
m.make_jobs = jobs
# TODO: make these build deps that can be installed if not found.
- m.make = MakeExecutable('make', jobs)
+ m.make = MakeExecutable('make', jobs)
m.gmake = MakeExecutable('gmake', jobs)
# easy shortcut to os.environ
@@ -253,33 +334,34 @@ def set_module_variables_for_package(pkg, module):
# Set up CMake rpath
m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE')
- m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg)))
+ m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' %
+ ":".join(get_rpaths(pkg)))
# Put spack compiler paths in module scope.
link_dir = spack.build_env_path
- m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc'])
+ m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc'])
m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx'])
m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77'])
- m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc'])
+ m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc'])
# Emulate some shell commands for convenience
- m.pwd = os.getcwd
- m.cd = os.chdir
- m.mkdir = os.mkdir
- m.makedirs = os.makedirs
- m.remove = os.remove
- m.removedirs = os.removedirs
- m.symlink = os.symlink
-
- m.mkdirp = mkdirp
- m.install = install
+ m.pwd = os.getcwd
+ m.cd = os.chdir
+ m.mkdir = os.mkdir
+ m.makedirs = os.makedirs
+ m.remove = os.remove
+ m.removedirs = os.removedirs
+ m.symlink = os.symlink
+
+ m.mkdirp = mkdirp
+ m.install = install
m.install_tree = install_tree
- m.rmtree = shutil.rmtree
- m.move = shutil.move
+ m.rmtree = shutil.rmtree
+ m.move = shutil.move
# Useful directories within the prefix are encapsulated in
# a Prefix object.
- m.prefix = pkg.prefix
+ m.prefix = pkg.prefix
# Platform-specific library suffix.
m.dso_suffix = dso_suffix
@@ -292,26 +374,40 @@ def get_rpaths(pkg):
if os.path.isdir(d.prefix.lib))
rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values()
if os.path.isdir(d.prefix.lib64))
+ # Second module is our compiler mod name. We use that to get rpaths from
+ # module show output.
+ if pkg.compiler.modules and len(pkg.compiler.modules) > 1:
+ rpaths.append(get_path_from_module(pkg.compiler.modules[1]))
return rpaths
def parent_class_modules(cls):
- """Get list of super class modules that are all descend from spack.Package"""
+ """
+ Get list of super class modules that are all descend from spack.Package
+ """
if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls):
return []
result = []
module = sys.modules.get(cls.__module__)
if module:
- result = [ module ]
+ result = [module]
for c in cls.__bases__:
result.extend(parent_class_modules(c))
return result
+def load_external_modules(pkg):
+ """ traverse the spec list and find any specs that have external modules.
+ """
+ for dep in list(pkg.spec.traverse()):
+ if dep.external_module:
+ load_module(dep.external_module)
+
+
def setup_package(pkg):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
- run_env = EnvironmentModifications()
+ run_env = EnvironmentModifications()
# Before proceeding, ensure that specs and packages are consistent
#
@@ -327,11 +423,12 @@ def setup_package(pkg):
# throwaway environment, but it is kind of dirty.
#
# TODO: Think about how to avoid this fix and do something cleaner.
- for s in pkg.spec.traverse(): s.package.spec = s
+ for s in pkg.spec.traverse():
+ s.package.spec = s
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env)
-
+ load_external_modules(pkg)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post', root=False):
@@ -415,7 +512,9 @@ def fork(pkg, function):
# message. Just make the parent exit with an error code.
pid, returncode = os.waitpid(pid, 0)
if returncode != 0:
- raise InstallError("Installation process had nonzero exit code.".format(str(returncode)))
+ message = "Installation process had nonzero exit code : {code}"
+ strcode = str(returncode)
+ raise InstallError(message.format(code=strcode))
class InstallError(spack.error.SpackError):
diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py
index dc96dd0faa..cf2f96fd21 100644
--- a/lib/spack/spack/cmd/arch.py
+++ b/lib/spack/spack/cmd/arch.py
@@ -28,8 +28,4 @@ import spack.architecture as architecture
description = "Print the architecture for this machine"
def arch(parser, args):
- configured_sys_type = architecture.get_sys_type_from_spack_globals()
- if not configured_sys_type:
- configured_sys_type = "autodetect"
- print "Configured sys_type: %s" % configured_sys_type
- print "Autodetected default sys_type: %s" % architecture.sys_type()
+ print architecture.sys_type()
diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index dc7731a290..c95045ef85 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -70,7 +70,7 @@ def setup_parser(subparser):
def compiler_find(args):
- """Search either $PATH or a list of paths for compilers and add them
+ """Search either $PATH or a list of paths OR MODULES for compilers and add them
to Spack's configuration."""
paths = args.add_paths
if not paths:
@@ -78,7 +78,6 @@ def compiler_find(args):
compilers = [c for c in spack.compilers.find_compilers(*args.add_paths)
if c.spec not in spack.compilers.all_compilers(scope=args.scope)]
-
if compilers:
spack.compilers.add_compilers_to_config(compilers, scope=args.scope)
n = len(compilers)
@@ -93,7 +92,6 @@ def compiler_find(args):
def compiler_remove(args):
cspec = CompilerSpec(args.compiler_spec)
compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope)
-
if not compilers:
tty.die("No compilers match spec %s" % cspec)
elif not args.all and len(compilers) > 1:
@@ -121,6 +119,8 @@ def compiler_info(args):
print "\tcxx = %s" % c.cxx
print "\tf77 = %s" % c.f77
print "\tfc = %s" % c.fc
+ print "\tmodules = %s" % c.modules
+ print "\toperating system = %s" % c.operating_system
def compiler_list(args):
@@ -135,10 +135,10 @@ def compiler_list(args):
def compiler(parser, args):
- action = { 'add' : compiler_find,
- 'find' : compiler_find,
- 'remove' : compiler_remove,
- 'rm' : compiler_remove,
- 'info' : compiler_info,
- 'list' : compiler_list }
+ action = {'add' : compiler_find,
+ 'find' : compiler_find,
+ 'remove' : compiler_remove,
+ 'rm' : compiler_remove,
+ 'info' : compiler_info,
+ 'list' : compiler_list }
action[args.compiler_command](args)
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index 7071af2686..41bfa741f6 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -1,3 +1,4 @@
+_copyright = """\
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
@@ -22,6 +23,7 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+"""
import string
import os
import hashlib
@@ -47,22 +49,22 @@ from spack.stage import Stage
description = "Create a new package file from an archive URL"
-package_template = string.Template("""\
-# FIXME:
-# This is a template package file for Spack. We've conveniently
-# put "FIXME" labels next to all the things you'll want to change.
+package_template = string.Template(
+ _copyright + """
#
-# Once you've edited all the FIXME's, delete this whole message,
-# save this file, and test out your package like this:
+# This is a template package file for Spack. We've put "FIXME"
+# next to all the things you'll want to change. Once you've handled
+# them, you can save this file and test your package like this:
#
# spack install ${name}
#
-# You can always get back here to change things with:
+# You can edit this file again by typing:
#
# spack edit ${name}
#
-# See the spack documentation for more information on building
-# packages.
+# See the Spack documentation for more information on packaging.
+# If you submit this package back to Spack as a pull request,
+# please first remove this boilerplate and all FIXME comments.
#
from spack import *
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index a99012a275..3ec671f93f 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -22,57 +22,86 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-import sys
-import collections
-import itertools
import argparse
-from StringIO import StringIO
+import sys
import llnl.util.tty as tty
+import spack
+import spack.spec
+from llnl.util.lang import *
from llnl.util.tty.colify import *
from llnl.util.tty.color import *
from llnl.util.lang import *
-import spack
-import spack.spec
+description = "Find installed spack packages"
-description ="Find installed spack packages"
def setup_parser(subparser):
format_group = subparser.add_mutually_exclusive_group()
+ format_group.add_argument('-s', '--short',
+ action='store_const',
+ dest='mode',
+ const='short',
+ help='Show only specs (default)')
+ format_group.add_argument('-p', '--paths',
+ action='store_const',
+ dest='mode',
+ const='paths',
+ help='Show paths to package install directories')
format_group.add_argument(
- '-s', '--short', action='store_const', dest='mode', const='short',
- help='Show only specs (default)')
- format_group.add_argument(
- '-p', '--paths', action='store_const', dest='mode', const='paths',
- help='Show paths to package install directories')
- format_group.add_argument(
- '-d', '--deps', action='store_const', dest='mode', const='deps',
+ '-d', '--deps',
+ action='store_const',
+ dest='mode',
+ const='deps',
help='Show full dependency DAG of installed packages')
+ subparser.add_argument('-l', '--long',
+ action='store_true',
+ dest='long',
+ help='Show dependency hashes as well as versions.')
+ subparser.add_argument('-L', '--very-long',
+ action='store_true',
+ dest='very_long',
+ help='Show dependency hashes as well as versions.')
+ subparser.add_argument('-f', '--show-flags',
+ action='store_true',
+ dest='show_flags',
+ help='Show spec compiler flags.')
+
subparser.add_argument(
- '-l', '--long', action='store_true',
- help='Show dependency hashes as well as versions.')
+ '-e', '--explicit',
+ action='store_true',
+ help='Show only specs that were installed explicitly')
subparser.add_argument(
- '-L', '--very-long', action='store_true',
- help='Show dependency hashes as well as versions.')
-
+ '-E', '--implicit',
+ action='store_true',
+ help='Show only specs that were installed as dependencies')
subparser.add_argument(
- '-u', '--unknown', action='store_true',
+ '-u', '--unknown',
+ action='store_true',
+ dest='unknown',
help='Show only specs Spack does not have a package for.')
subparser.add_argument(
- '-m', '--missing', action='store_true',
+ '-m', '--missing',
+ action='store_true',
+ dest='missing',
help='Show missing dependencies as well as installed specs.')
subparser.add_argument(
- '-M', '--only-missing', action='store_true',
- help='Show only missing dependencies.')
- subparser.add_argument(
- '-N', '--namespace', action='store_true',
- help='Show fully qualified package names.')
-
- subparser.add_argument(
- 'query_specs', nargs=argparse.REMAINDER,
- help='optional specs to filter results')
+ '-v', '--variants',
+ action='store_true',
+ dest='variants',
+ help='Show variants in output (can be long)')
+ subparser.add_argument('-M', '--only-missing',
+ action='store_true',
+ dest='only_missing',
+ help='Show only missing dependencies.')
+ subparser.add_argument('-N', '--namespace',
+ action='store_true',
+ help='Show fully qualified package names.')
+
+ subparser.add_argument('query_specs',
+ nargs=argparse.REMAINDER,
+ help='optional specs to filter results')
def gray_hash(spec, length):
@@ -83,29 +112,36 @@ def display_specs(specs, **kwargs):
mode = kwargs.get('mode', 'short')
hashes = kwargs.get('long', False)
namespace = kwargs.get('namespace', False)
+ flags = kwargs.get('show_flags', False)
+ variants = kwargs.get('variants', False)
hlen = 7
if kwargs.get('very_long', False):
hashes = True
hlen = None
+ nfmt = '.' if namespace else '_'
+ ffmt = '$%+' if flags else ''
+ vfmt = '$+' if variants else ''
+ format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt)
+
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
# Traverse the index and print out each package
for i, (architecture, compiler) in enumerate(sorted(index)):
- if i > 0: print
+ if i > 0:
+ print
- header = "%s{%s} / %s{%s}" % (
- spack.spec.architecture_color, architecture,
- spack.spec.compiler_color, compiler)
+ header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
+ architecture, spack.spec.compiler_color,
+ compiler)
tty.hline(colorize(header), char='-')
- specs = index[(architecture,compiler)]
+ specs = index[(architecture, compiler)]
specs.sort()
- nfmt = '.' if namespace else '_'
- abbreviated = [s.format('$%s$@$+' % nfmt, color=True) for s in specs]
+ abbreviated = [s.format(format_string, color=True) for s in specs]
if mode == 'paths':
# Print one spec per line along with prefix path
width = max(len(s) for s in abbreviated)
@@ -114,38 +150,71 @@ def display_specs(specs, **kwargs):
for abbrv, spec in zip(abbreviated, specs):
if hashes:
- print gray_hash(spec, hlen),
- print format % (abbrv, spec.prefix)
+ print(gray_hash(spec, hlen), )
+ print(format % (abbrv, spec.prefix))
elif mode == 'deps':
for spec in specs:
- print spec.tree(
- format='$%s$@$+' % nfmt,
+ print(spec.tree(
+ format=format_string,
color=True,
indent=4,
- prefix=(lambda s: gray_hash(s, hlen)) if hashes else None)
+ prefix=(lambda s: gray_hash(s, hlen)) if hashes else None))
elif mode == 'short':
- def fmt(s):
- string = ""
- if hashes:
- string += gray_hash(s, hlen) + ' '
- string += s.format('$-%s$@$+' % nfmt, color=True)
+ # Print columns of output if not printing flags
+ if not flags:
+
+ def fmt(s):
+ string = ""
+ if hashes:
+ string += gray_hash(s, hlen) + ' '
+ string += s.format('$-%s$@%s' % (nfmt, vfmt), color=True)
+
+ return string
- return string
- colify(fmt(s) for s in specs)
+ colify(fmt(s) for s in specs)
+ # Print one entry per line if including flags
+ else:
+ for spec in specs:
+ # Print the hash if necessary
+ hsh = gray_hash(spec, hlen) + ' ' if hashes else ''
+ print(hsh + spec.format(format_string, color=True) + '\n')
else:
raise ValueError(
- "Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode)
+ "Invalid mode for display_specs: %s. Must be one of (paths,"
+ "deps, short)." % mode) # NOQA: ignore=E501
+def query_arguments(args):
+ # Check arguments
+ if args.explicit and args.implicit:
+ tty.error('You can\'t pass -E and -e options simultaneously.')
+ raise SystemExit(1)
+
+ # Set up query arguments.
+ installed, known = True, any
+ if args.only_missing:
+ installed = False
+ elif args.missing:
+ installed = any
+ if args.unknown:
+ known = False
+ explicit = any
+ if args.explicit:
+ explicit = True
+ if args.implicit:
+ explicit = False
+ q_args = {'installed': installed, 'known': known, "explicit": explicit}
+ return q_args
+
def find(parser, args):
# Filter out specs that don't exist.
query_specs = spack.cmd.parse_specs(args.query_specs)
query_specs, nonexisting = partition_list(
- query_specs, lambda s: spack.repo.exists(s.name))
+ query_specs, lambda s: spack.repo.exists(s.name) or not s.name)
if nonexisting:
msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '')
@@ -155,21 +224,14 @@ def find(parser, args):
if not query_specs:
return
- # Set up query arguments.
- installed, known = True, any
- if args.only_missing:
- installed = False
- elif args.missing:
- installed = any
- if args.unknown:
- known = False
- q_args = { 'installed' : installed, 'known' : known }
+ q_args = query_arguments(args)
# Get all the specs the user asked for
if not query_specs:
specs = set(spack.installed_db.query(**q_args))
else:
- results = [set(spack.installed_db.query(qs, **q_args)) for qs in query_specs]
+ results = [set(spack.installed_db.query(qs, **q_args))
+ for qs in query_specs]
specs = set.union(*results)
if not args.mode:
@@ -177,7 +239,10 @@ def find(parser, args):
if sys.stdout.isatty():
tty.msg("%d installed packages." % len(specs))
- display_specs(specs, mode=args.mode,
+ display_specs(specs,
+ mode=args.mode,
long=args.long,
very_long=args.very_long,
- namespace=args.namespace)
+ show_flags=args.show_flags,
+ namespace=args.namespace,
+ variants=args.variants)
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index fef21f15ba..9d3175786b 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -78,4 +78,5 @@ def install(parser, args):
ignore_deps=args.ignore_deps,
make_jobs=args.jobs,
verbose=args.verbose,
- fake=args.fake)
+ fake=args.fake,
+ explicit=True)
diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py
index 3bffc2633b..a6f08d09ed 100644
--- a/lib/spack/spack/cmd/uninstall.py
+++ b/lib/spack/spack/cmd/uninstall.py
@@ -39,6 +39,13 @@ error_message = """You can either:
b) use spack uninstall -a to uninstall ALL matching specs.
"""
+# Arguments for display_specs when we find ambiguity
+display_args = {
+ 'long': True,
+ 'show_flags': True,
+ 'variants':True
+}
+
def ask_for_confirmation(message):
while True:
@@ -92,7 +99,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False):
if not allow_multiple_matches and len(matching) > 1:
tty.error("%s matches multiple packages:" % spec)
print()
- display_specs(matching, long=True)
+ display_specs(matching, **display_args)
print()
has_errors = True
@@ -172,7 +179,7 @@ def uninstall(parser, args):
tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True))
print('')
print("The following packages depend on it:")
- display_specs(lst, long=True)
+ display_specs(lst, **display_args)
print('')
has_error = True
elif args.dependents:
@@ -186,7 +193,7 @@ def uninstall(parser, args):
if not args.yes_to_all:
tty.msg("The following packages will be uninstalled : ")
print('')
- display_specs(uninstall_list, long=True)
+ display_specs(uninstall_list, **display_args)
print('')
ask_for_confirmation('Do you want to proceed ? ')
diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py
new file mode 100644
index 0000000000..8f1fc9be74
--- /dev/null
+++ b/lib/spack/spack/cmd/view.py
@@ -0,0 +1,295 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+'''Produce a "view" of a Spack DAG.
+
+A "view" is file hierarchy representing the union of a number of
+Spack-installed package file hierarchies. The union is formed from:
+
+- specs resolved from the package names given by the user (the seeds)
+
+- all depenencies of the seeds unless user specifies `--no-depenencies`
+
+- less any specs with names matching the regular expressions given by
+ `--exclude`
+
+The `view` can be built and tore down via a number of methods (the "actions"):
+
+- symlink :: a file system view which is a directory hierarchy that is
+ the union of the hierarchies of the installed packages in the DAG
+ where installed files are referenced via symlinks.
+
+- hardlink :: like the symlink view but hardlinks are used.
+
+- statlink :: a view producing a status report of a symlink or
+ hardlink view.
+
+The file system view concept is imspired by Nix, implemented by
+brett.viren@gmail.com ca 2016.
+
+'''
+# Implementation notes:
+#
+# This is implemented as a visitor pattern on the set of package specs.
+#
+# The command line ACTION maps to a visitor_*() function which takes
+# the set of package specs and any args which may be specific to the
+# ACTION.
+#
+# To add a new view:
+# 1. add a new cmd line args sub parser ACTION
+# 2. add any action-specific options/arguments, most likely a list of specs.
+# 3. add a visitor_MYACTION() function
+# 4. add any visitor_MYALIAS assignments to match any command line aliases
+
+import os
+import re
+import spack
+import spack.cmd
+import llnl.util.tty as tty
+
+description = "Produce a single-rooted directory view of a spec."
+
+
+def setup_parser(sp):
+ setup_parser.parser = sp
+
+ sp.add_argument(
+ '-v', '--verbose', action='store_true', default=False,
+ help="Display verbose output.")
+ sp.add_argument(
+ '-e', '--exclude', action='append', default=[],
+ help="Exclude packages with names matching the given regex pattern.")
+ sp.add_argument(
+ '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'],
+ default='true',
+ help="Follow dependencies.")
+
+ ssp = sp.add_subparsers(metavar='ACTION', dest='action')
+
+ specs_opts = dict(metavar='spec', nargs='+',
+ help="Seed specs of the packages to view.")
+
+ # The action parameterizes the command but in keeping with Spack
+ # patterns we make it a subcommand.
+ file_system_view_actions = [
+ ssp.add_parser(
+ 'symlink', aliases=['add', 'soft'],
+ help='Add package files to a filesystem view via symbolic links.'),
+ ssp.add_parser(
+ 'hardlink', aliases=['hard'],
+ help='Add packages files to a filesystem via via hard links.'),
+ ssp.add_parser(
+ 'remove', aliases=['rm'],
+ help='Remove packages from a filesystem view.'),
+ ssp.add_parser(
+ 'statlink', aliases=['status', 'check'],
+ help='Check status of packages in a filesystem view.')
+ ]
+ # All these options and arguments are common to every action.
+ for act in file_system_view_actions:
+ act.add_argument('path', nargs=1,
+ help="Path to file system view directory.")
+ act.add_argument('specs', **specs_opts)
+
+ return
+
+
+def assuredir(path):
+ 'Assure path exists as a directory'
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+
+def relative_to(prefix, path):
+ 'Return end of `path` relative to `prefix`'
+ assert 0 == path.find(prefix)
+ reldir = path[len(prefix):]
+ if reldir.startswith('/'):
+ reldir = reldir[1:]
+ return reldir
+
+
+def transform_path(spec, path, prefix=None):
+ 'Return the a relative path corresponding to given path spec.prefix'
+ if os.path.isabs(path):
+ path = relative_to(spec.prefix, path)
+ subdirs = path.split(os.path.sep)
+ if subdirs[0] == '.spack':
+ lst = ['.spack', spec.name] + subdirs[1:]
+ path = os.path.join(*lst)
+ if prefix:
+ path = os.path.join(prefix, path)
+ return path
+
+
+def purge_empty_directories(path):
+ '''Ascend up from the leaves accessible from `path`
+ and remove empty directories.'''
+ for dirpath, subdirs, files in os.walk(path, topdown=False):
+ for sd in subdirs:
+ sdp = os.path.join(dirpath, sd)
+ try:
+ os.rmdir(sdp)
+ except OSError:
+ pass
+
+
+def filter_exclude(specs, exclude):
+ 'Filter specs given sequence of exclude regex'
+ to_exclude = [re.compile(e) for e in exclude]
+
+ def exclude(spec):
+ for e in to_exclude:
+ if e.match(spec.name):
+ return True
+ return False
+ return [s for s in specs if not exclude(s)]
+
+
+def flatten(seeds, descend=True):
+ 'Normalize and flattend seed specs and descend hiearchy'
+ flat = set()
+ for spec in seeds:
+ if not descend:
+ flat.add(spec)
+ continue
+ flat.update(spec.normalized().traverse())
+ return flat
+
+
+def check_one(spec, path, verbose=False):
+ 'Check status of view in path against spec'
+ dotspack = os.path.join(path, '.spack', spec.name)
+ if os.path.exists(os.path.join(dotspack)):
+ tty.info('Package in view: "%s"' % spec.name)
+ return
+ tty.info('Package not in view: "%s"' % spec.name)
+ return
+
+
+def remove_one(spec, path, verbose=False):
+ 'Remove any files found in `spec` from `path` and purge empty directories.'
+
+ if not os.path.exists(path):
+ return # done, short circuit
+
+ dotspack = transform_path(spec, '.spack', path)
+ if not os.path.exists(dotspack):
+ if verbose:
+ tty.info('Skipping nonexistent package: "%s"' % spec.name)
+ return
+
+ if verbose:
+ tty.info('Removing package: "%s"' % spec.name)
+ for dirpath, dirnames, filenames in os.walk(spec.prefix):
+ if not filenames:
+ continue
+ targdir = transform_path(spec, dirpath, path)
+ for fname in filenames:
+ dst = os.path.join(targdir, fname)
+ if not os.path.exists(dst):
+ continue
+ os.unlink(dst)
+
+
+def link_one(spec, path, link=os.symlink, verbose=False):
+ 'Link all files in `spec` into directory `path`.'
+
+ dotspack = transform_path(spec, '.spack', path)
+ if os.path.exists(dotspack):
+ tty.warn('Skipping existing package: "%s"' % spec.name)
+ return
+
+ if verbose:
+ tty.info('Linking package: "%s"' % spec.name)
+ for dirpath, dirnames, filenames in os.walk(spec.prefix):
+ if not filenames:
+ continue # avoid explicitly making empty dirs
+
+ targdir = transform_path(spec, dirpath, path)
+ assuredir(targdir)
+
+ for fname in filenames:
+ src = os.path.join(dirpath, fname)
+ dst = os.path.join(targdir, fname)
+ if os.path.exists(dst):
+ if '.spack' in dst.split(os.path.sep):
+ continue # silence these
+ tty.warn("Skipping existing file: %s" % dst)
+ continue
+ link(src, dst)
+
+
+def visitor_symlink(specs, args):
+ 'Symlink all files found in specs'
+ path = args.path[0]
+ assuredir(path)
+ for spec in specs:
+ link_one(spec, path, verbose=args.verbose)
+visitor_add = visitor_symlink
+visitor_soft = visitor_symlink
+
+
+def visitor_hardlink(specs, args):
+ 'Hardlink all files found in specs'
+ path = args.path[0]
+ assuredir(path)
+ for spec in specs:
+ link_one(spec, path, os.link, verbose=args.verbose)
+visitor_hard = visitor_hardlink
+
+
+def visitor_remove(specs, args):
+ 'Remove all files and directories found in specs from args.path'
+ path = args.path[0]
+ for spec in specs:
+ remove_one(spec, path, verbose=args.verbose)
+ purge_empty_directories(path)
+visitor_rm = visitor_remove
+
+
+def visitor_statlink(specs, args):
+ 'Give status of view in args.path relative to specs'
+ path = args.path[0]
+ for spec in specs:
+ check_one(spec, path, verbose=args.verbose)
+visitor_status = visitor_statlink
+visitor_check = visitor_statlink
+
+
+def view(parser, args):
+ 'Produce a view of a set of packages.'
+
+ # Process common args
+ seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs]
+ specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true'])
+ specs = filter_exclude(specs, args.exclude)
+
+ # Execute the visitation.
+ try:
+ visitor = globals()['visitor_' + args.action]
+ except KeyError:
+ tty.error('Unknown action: "%s"' % args.action)
+ visitor(specs, args)
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index e2da272212..ce4555bc56 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -33,6 +33,7 @@ from llnl.util.filesystem import join_path
import spack.error
import spack.spec
+import spack.architecture
from spack.util.multiproc import parmap
from spack.util.executable import *
from spack.util.environment import get_path
@@ -107,22 +108,46 @@ class Compiler(object):
@property
def fc_rpath_arg(self):
return '-Wl,-rpath,'
+ # Cray PrgEnv name that can be used to load this compiler
+ PrgEnv = None
+ # Name of module used to switch versions of this compiler
+ PrgEnv_compiler = None
-
- def __init__(self, cspec, cc, cxx, f77, fc):
+ def __init__(self, cspec, operating_system,
+ paths, modules=[], alias=None, **kwargs):
def check(exe):
if exe is None:
return None
_verify_executables(exe)
return exe
- self.cc = check(cc)
- self.cxx = check(cxx)
- self.f77 = check(f77)
- self.fc = check(fc)
-
+ self.cc = check(paths[0])
+ self.cxx = check(paths[1])
+ if len(paths) > 2:
+ self.f77 = check(paths[2])
+ if len(paths) == 3:
+ self.fc = self.f77
+ else:
+ self.fc = check(paths[3])
+
+ #self.cc = check(cc)
+ #self.cxx = check(cxx)
+ #self.f77 = check(f77)
+ #self.fc = check(fc)
+
+ # Unfortunately have to make sure these params are accepted
+ # in the same order they are returned by sorted(flags)
+ # in compilers/__init__.py
+ self.flags = {}
+ for flag in spack.spec.FlagMap.valid_compiler_flags():
+ value = kwargs.get(flag, None)
+ if value is not None:
+ self.flags[flag] = value.split()
+
+ self.operating_system = operating_system
self.spec = cspec
-
+ self.modules = modules
+ self.alias = alias
@property
def version(self):
@@ -188,7 +213,6 @@ class Compiler(object):
def fc_version(cls, fc):
return cls.default_version(fc)
-
@classmethod
def _find_matches_in_path(cls, compiler_names, detect_version, *path):
"""Finds compilers in the paths supplied.
@@ -250,57 +274,6 @@ class Compiler(object):
successful.reverse()
return dict(((v, p, s), path) for v, p, s, path in successful)
- @classmethod
- def find(cls, *path):
- """Try to find this type of compiler in the user's
- environment. For each set of compilers found, this returns
- compiler objects with the cc, cxx, f77, fc paths and the
- version filled in.
-
- This will search for compilers with the names in cc_names,
- cxx_names, etc. and it will group them if they have common
- prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would
- be grouped with g++-mp-4.7 and gfortran-mp-4.7.
- """
- dicts = parmap(
- lambda t: cls._find_matches_in_path(*t),
- [(cls.cc_names, cls.cc_version) + tuple(path),
- (cls.cxx_names, cls.cxx_version) + tuple(path),
- (cls.f77_names, cls.f77_version) + tuple(path),
- (cls.fc_names, cls.fc_version) + tuple(path)])
-
- all_keys = set()
- for d in dicts:
- all_keys.update(d)
-
- compilers = {}
- for k in all_keys:
- ver, pre, suf = k
-
- # Skip compilers with unknown version.
- if ver == 'unknown':
- continue
-
- paths = tuple(pn[k] if k in pn else None for pn in dicts)
- spec = spack.spec.CompilerSpec(cls.name, ver)
-
- if ver in compilers:
- prev = compilers[ver]
-
- # prefer the one with more compilers.
- prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
- newcount = len([p for p in paths if p is not None])
- prevcount = len([p for p in prev_paths if p is not None])
-
- # Don't add if it's not an improvement over prev compiler.
- if newcount <= prevcount:
- continue
-
- compilers[ver] = cls(spec, *paths)
-
- return list(compilers.values())
-
-
def __repr__(self):
"""Return a string representation of the compiler toolchain."""
return self.__str__()
@@ -309,7 +282,7 @@ class Compiler(object):
def __str__(self):
"""Return a string representation of the compiler toolchain."""
return "%s(%s)" % (
- self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc))))
+ self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system)))))
class CompilerAccessError(spack.error.SpackError):
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index 692e5518aa..ae72b743b2 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -28,6 +28,11 @@ system and configuring Spack to use multiple compilers.
import imp
import os
import platform
+import copy
+import hashlib
+import base64
+import yaml
+import sys
from llnl.util.lang import memoized, list_modules
from llnl.util.filesystem import join_path
@@ -45,7 +50,9 @@ from spack.util.naming import mod_to_class
from spack.util.environment import get_path
_imported_compilers_module = 'spack.compilers'
-_required_instance_vars = ['cc', 'cxx', 'f77', 'fc']
+_path_instance_vars = ['cc', 'cxx', 'f77', 'fc']
+_other_instance_vars = ['modules', 'operating_system']
+_cache_config_file = []
# TODO: customize order in config file
if platform.system() == 'Darwin':
@@ -64,107 +71,105 @@ def _auto_compiler_spec(function):
def _to_dict(compiler):
"""Return a dict version of compiler suitable to insert in YAML."""
- return {
- str(compiler.spec) : dict(
- (attr, getattr(compiler, attr, None))
- for attr in _required_instance_vars)
- }
+ d = {}
+ d['spec'] = str(compiler.spec)
+ d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars )
+ d['operating_system'] = str(compiler.operating_system)
+ d['modules'] = compiler.modules if compiler.modules else []
+ if compiler.alias:
+ d['alias'] = compiler.alias
-def get_compiler_config(arch=None, scope=None):
+ return {'compiler': d}
+
+
+def get_compiler_config(scope=None):
"""Return the compiler configuration for the specified architecture.
"""
- # Check whether we're on a front-end (native) architecture.
- my_arch = spack.architecture.sys_type()
- if arch is None:
- arch = my_arch
-
def init_compiler_config():
"""Compiler search used when Spack has no compilers."""
- config[arch] = {}
- compilers = find_compilers(*get_path('PATH'))
+ compilers = find_compilers()
+ compilers_dict = []
for compiler in compilers:
- config[arch].update(_to_dict(compiler))
- spack.config.update_config('compilers', config, scope=scope)
+ compilers_dict.append(_to_dict(compiler))
+ spack.config.update_config('compilers', compilers_dict, scope=scope)
config = spack.config.get_config('compilers', scope=scope)
-
# Update the configuration if there are currently no compilers
# configured. Avoid updating automatically if there ARE site
# compilers configured but no user ones.
- if arch == my_arch and arch not in config:
+ if not config:
if scope is None:
# We know no compilers were configured in any scope.
init_compiler_config()
+ config = spack.config.get_config('compilers', scope=scope)
elif scope == 'user':
# Check the site config and update the user config if
# nothing is configured at the site level.
site_config = spack.config.get_config('compilers', scope='site')
if not site_config:
init_compiler_config()
-
- return config[arch] if arch in config else {}
+ config = spack.config.get_config('compilers', scope=scope)
+ return config
+ elif config:
+ return config
+ else:
+ return [] # Return empty list which we will later append to.
-def add_compilers_to_config(compilers, arch=None, scope=None):
+def add_compilers_to_config(compilers, scope=None):
"""Add compilers to the config for the specified architecture.
Arguments:
- compilers: a list of Compiler objects.
- - arch: arch to add compilers for.
- scope: configuration scope to modify.
"""
- if arch is None:
- arch = spack.architecture.sys_type()
-
- compiler_config = get_compiler_config(arch, scope)
+ compiler_config = get_compiler_config(scope)
for compiler in compilers:
- compiler_config[str(compiler.spec)] = dict(
- (c, getattr(compiler, c, "None"))
- for c in _required_instance_vars)
-
- update = { arch : compiler_config }
- spack.config.update_config('compilers', update, scope)
+ compiler_config.append(_to_dict(compiler))
+ global _cache_config_file
+ _cache_config_file = compiler_config
+ spack.config.update_config('compilers', compiler_config, scope)
@_auto_compiler_spec
-def remove_compiler_from_config(compiler_spec, arch=None, scope=None):
+def remove_compiler_from_config(compiler_spec, scope=None):
"""Remove compilers from the config, by spec.
Arguments:
- compiler_specs: a list of CompilerSpec objects.
- - arch: arch to add compilers for.
- scope: configuration scope to modify.
"""
- if arch is None:
- arch = spack.architecture.sys_type()
-
- compiler_config = get_compiler_config(arch, scope)
- del compiler_config[str(compiler_spec)]
- update = { arch : compiler_config }
-
- spack.config.update_config('compilers', update, scope)
-
-
-def all_compilers_config(arch=None, scope=None):
+ compiler_config = get_compiler_config(scope)
+ config_length = len(compiler_config)
+
+ filtered_compiler_config = [comp for comp in compiler_config
+ if spack.spec.CompilerSpec(comp['compiler']['spec']) != compiler_spec]
+ # Need a better way for this
+ global _cache_config_file
+ _cache_config_file = filtered_compiler_config # Update the cache for changes
+ if len(filtered_compiler_config) == config_length: # No items removed
+ CompilerSpecInsufficientlySpecificError(compiler_spec)
+ spack.config.update_config('compilers', filtered_compiler_config, scope)
+
+
+def all_compilers_config(scope=None):
"""Return a set of specs for all the compiler versions currently
available to build with. These are instances of CompilerSpec.
"""
# Get compilers for this architecture.
- arch_config = get_compiler_config(arch, scope)
-
- # Merge 'all' compilers with arch-specific ones.
- # Arch-specific compilers have higher precedence.
- merged_config = get_compiler_config('all', scope=scope)
- merged_config = spack.config._merge_yaml(merged_config, arch_config)
-
- return merged_config
+ global _cache_config_file #Create a cache of the config file so we don't load all the time.
+ if not _cache_config_file:
+ _cache_config_file = get_compiler_config(scope)
+ return _cache_config_file
+ else:
+ return _cache_config_file
-def all_compilers(arch=None, scope=None):
+def all_compilers(scope=None):
# Return compiler specs from the merged config.
- return [spack.spec.CompilerSpec(s)
- for s in all_compilers_config(arch, scope)]
+ return [spack.spec.CompilerSpec(s['compiler']['spec'])
+ for s in all_compilers_config(scope)]
def default_compiler():
@@ -179,36 +184,18 @@ def default_compiler():
return sorted(versions)[-1]
-def find_compilers(*path):
+def find_compilers(*paths):
"""Return a list of compilers found in the suppied paths.
- This invokes the find() method for each Compiler class,
- and appends the compilers detected to a list.
+ This invokes the find_compilers() method for each operating
+ system associated with the host platform, and appends
+ the compilers detected to a list.
"""
- # Make sure path elements exist, and include /bin directories
- # under prefixes.
- filtered_path = []
- for p in path:
- # Eliminate symlinks and just take the real directories.
- p = os.path.realpath(p)
- if not os.path.isdir(p):
- continue
- filtered_path.append(p)
-
- # Check for a bin directory, add it if it exists
- bin = join_path(p, 'bin')
- if os.path.isdir(bin):
- filtered_path.append(os.path.realpath(bin))
-
- # Once the paths are cleaned up, do a search for each type of
- # compiler. We can spawn a bunch of parallel searches to reduce
- # the overhead of spelunking all these directories.
- types = all_compiler_types()
- compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types)
-
- # ensure all the version calls we made are cached in the parent
- # process, as well. This speeds up Spack a lot.
- clist = reduce(lambda x,y: x+y, compiler_lists)
- return clist
+ # Find compilers for each operating system class
+ oss = all_os_classes()
+ compiler_lists = []
+ for o in oss:
+ compiler_lists.extend(o.find_compilers(*paths))
+ return compiler_lists
def supported_compilers():
@@ -227,47 +214,83 @@ def supported(compiler_spec):
@_auto_compiler_spec
-def find(compiler_spec, arch=None, scope=None):
+def find(compiler_spec, scope=None):
"""Return specs of available compilers that match the supplied
compiler spec. Return an list if nothing found."""
- return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)]
+ return [c for c in all_compilers(scope) if c.satisfies(compiler_spec)]
@_auto_compiler_spec
-def compilers_for_spec(compiler_spec, arch=None, scope=None):
+def compilers_for_spec(compiler_spec, scope=None, **kwargs):
"""This gets all compilers that satisfy the supplied CompilerSpec.
Returns an empty list if none are found.
"""
- config = all_compilers_config(arch, scope)
+ platform = kwargs.get("platform", None)
+ config = all_compilers_config(scope)
+
+ def get_compilers(cspec):
+ compilers = []
- def get_compiler(cspec):
- items = config[str(cspec)]
+ for items in config:
+ if items['compiler']['spec'] != str(cspec):
+ continue
+ items = items['compiler']
- if not all(n in items for n in _required_instance_vars):
- raise InvalidCompilerConfigurationError(cspec)
+ if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)):
+ raise InvalidCompilerConfigurationError(cspec)
- cls = class_for_compiler_name(cspec.name)
- compiler_paths = []
- for c in _required_instance_vars:
- compiler_path = items[c]
- if compiler_path != "None":
- compiler_paths.append(compiler_path)
+ cls = class_for_compiler_name(cspec.name)
+
+ compiler_paths = []
+ for c in _path_instance_vars:
+ compiler_path = items['paths'][c]
+ if compiler_path != "None":
+ compiler_paths.append(compiler_path)
+ else:
+ compiler_paths.append(None)
+
+ mods = items.get('modules')
+ if mods == 'None':
+ mods = []
+
+ if 'operating_system' in items:
+ operating_system = spack.architecture._operating_system_from_dict(items['operating_system'], platform)
else:
- compiler_paths.append(None)
+ operating_system = None
+
- return cls(cspec, *compiler_paths)
+ alias = items['alias'] if 'alias' in items else None
- matches = find(compiler_spec, arch, scope)
- return [get_compiler(cspec) for cspec in matches]
+ flags = {}
+ for f in spack.spec.FlagMap.valid_compiler_flags():
+ if f in items:
+ flags[f] = items[f]
+
+ compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags))
+
+ return compilers
+
+ matches = set(find(compiler_spec, scope))
+ compilers = []
+ for cspec in matches:
+ compilers.extend(get_compilers(cspec))
+ return compilers
+# return [get_compilers(cspec) for cspec in matches]
@_auto_compiler_spec
-def compiler_for_spec(compiler_spec):
+def compiler_for_spec(compiler_spec, arch):
"""Get the compiler that satisfies compiler_spec. compiler_spec must
be concrete."""
+ operating_system = arch.platform_os
assert(compiler_spec.concrete)
- compilers = compilers_for_spec(compiler_spec)
- assert(len(compilers) == 1)
+
+ compilers = [c for c in compilers_for_spec(compiler_spec, platform=arch.platform)
+ if c.operating_system == operating_system]
+ if len(compilers) < 1:
+ raise NoCompilerForSpecError(compiler_spec, operating_system)
+ if len(compilers) > 1:
+ raise CompilerSpecInsufficientlySpecificError(compiler_spec)
return compilers[0]
@@ -285,6 +308,19 @@ def class_for_compiler_name(compiler_name):
return cls
+def all_os_classes():
+ """
+ Return the list of classes for all operating systems available on
+ this platform
+ """
+ classes = []
+
+ platform = spack.architecture.sys_type()
+ for os_class in platform.operating_sys.values():
+ classes.append(os_class)
+
+ return classes
+
def all_compiler_types():
return [class_for_compiler_name(c) for c in supported_compilers()]
@@ -294,9 +330,19 @@ class InvalidCompilerConfigurationError(spack.error.SpackError):
super(InvalidCompilerConfigurationError, self).__init__(
"Invalid configuration for [compiler \"%s\"]: " % compiler_spec,
"Compiler configuration must contain entries for all compilers: %s"
- % _required_instance_vars)
+ % _path_instance_vars)
class NoCompilersError(spack.error.SpackError):
def __init__(self):
super(NoCompilersError, self).__init__("Spack could not find any compilers!")
+
+class NoCompilerForSpecError(spack.error.SpackError):
+ def __init__(self, compiler_spec, target):
+ super(NoCompilerForSpecError, self).__init__("No compilers for operating system %s satisfy spec %s" % (
+ target, compiler_spec))
+
+class CompilerSpecInsufficientlySpecificError(spack.error.SpackError):
+ def __init__(self, compiler_spec):
+ super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s",
+ compiler_spec)
diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py
index 072bcd065f..00b406d820 100644
--- a/lib/spack/spack/compilers/clang.py
+++ b/lib/spack/spack/compilers/clang.py
@@ -73,7 +73,7 @@ class Clang(Compiler):
return "-std=c++11"
@classmethod
- def default_version(self, comp):
+ def default_version(cls, comp):
"""The '--version' option works for clang compilers.
On most platforms, output looks like this::
diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py
new file mode 100644
index 0000000000..4ba8b110ec
--- /dev/null
+++ b/lib/spack/spack/compilers/craype.py
@@ -0,0 +1,58 @@
+##############################################################################}
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import llnl.util.tty as tty
+
+#from spack.build_environment import load_module
+from spack.compiler import *
+#from spack.version import ver
+
+class Craype(Compiler):
+ # Subclasses use possible names of C compiler
+ cc_names = ['cc']
+
+ # Subclasses use possible names of C++ compiler
+ cxx_names = ['CC']
+
+ # Subclasses use possible names of Fortran 77 compiler
+ f77_names = ['ftn']
+
+ # Subclasses use possible names of Fortran 90 compiler
+ fc_names = ['ftn']
+
+ # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
+ suffixes = [r'-mp-\d\.\d']
+
+ PrgEnv = 'PrgEnv-cray'
+ PrgEnv_compiler = 'craype'
+
+ link_paths = { 'cc' : 'cc',
+ 'cxx' : 'c++',
+ 'f77' : 'f77',
+ 'fc' : 'fc'}
+
+ @classmethod
+ def default_version(cls, comp):
+ return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)')
+
diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py
index 164bddeb3f..3f552eaece 100644
--- a/lib/spack/spack/compilers/gcc.py
+++ b/lib/spack/spack/compilers/gcc.py
@@ -49,6 +49,9 @@ class Gcc(Compiler):
'f77' : 'gcc/gfortran',
'fc' : 'gcc/gfortran' }
+ PrgEnv = 'PrgEnv-gnu'
+ PrgEnv_compiler = 'gcc'
+
@property
def openmp_flag(self):
return "-fopenmp"
@@ -74,9 +77,9 @@ class Gcc(Compiler):
return get_compiler_version(
fc, '-dumpversion',
# older gfortran versions don't have simple dumpversion output.
- r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)')
+ r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module)
@classmethod
def f77_version(cls, f77):
- return cls.fc_version(f77)
+ return cls.fc_version(f77, module)
diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py
index 5007ece645..6cad03ff47 100644
--- a/lib/spack/spack/compilers/intel.py
+++ b/lib/spack/spack/compilers/intel.py
@@ -45,6 +45,9 @@ class Intel(Compiler):
'f77' : 'intel/ifort',
'fc' : 'intel/ifort' }
+ PrgEnv = 'PrgEnv-intel'
+ PrgEnv_compiler = 'intel'
+
@property
def openmp_flag(self):
if self.version < ver('16.0'):
diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py
index d42148dc49..6d36d8bfa6 100644
--- a/lib/spack/spack/compilers/pgi.py
+++ b/lib/spack/spack/compilers/pgi.py
@@ -44,6 +44,12 @@ class Pgi(Compiler):
'f77' : 'pgi/pgfortran',
'fc' : 'pgi/pgfortran' }
+
+
+ PrgEnv = 'PrgEnv-pgi'
+ PrgEnv_compiler = 'pgi'
+
+
@property
def openmp_flag(self):
return "-mp"
@@ -52,7 +58,6 @@ class Pgi(Compiler):
def cxx11_flag(self):
return "-std=c++11"
-
@classmethod
def default_version(cls, comp):
"""The '-V' option works for all the PGI compilers.
diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py
index bda2de4b87..b1431436ad 100644
--- a/lib/spack/spack/compilers/xl.py
+++ b/lib/spack/spack/compilers/xl.py
@@ -56,8 +56,9 @@ class Xl(Compiler):
else:
return "-qlanglvl=extended0x"
+
@classmethod
- def default_version(self, comp):
+ def default_version(cls, comp):
"""The '-qversion' is the standard option fo XL compilers.
Output looks like this::
@@ -83,6 +84,7 @@ class Xl(Compiler):
return get_compiler_version(
comp, '-qversion',r'([0-9]?[0-9]\.[0-9])')
+
@classmethod
def fc_version(cls, fc):
"""The fortran and C/C++ versions of the XL compiler are always two units apart.
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index f5e1c10b48..1f37455c77 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -44,6 +44,7 @@ from spec import DependencyMap
from itertools import chain
from spack.config import *
+
class DefaultConcretizer(object):
"""This class doesn't have any state, it just provides some methods for
concretization. You can subclass it to override just some of the
@@ -83,7 +84,8 @@ class DefaultConcretizer(object):
raise NoBuildError(spec)
def cmp_externals(a, b):
- if a.name != b.name:
+ if a.name != b.name and (not a.external or a.external_module and
+ not b.external and b.external_module):
# We're choosing between different providers, so
# maintain order from provider sort
return candidates.index(a) - candidates.index(b)
@@ -186,31 +188,64 @@ class DefaultConcretizer(object):
return True # Things changed
+ def _concretize_operating_system(self, spec):
+ platform = spec.architecture.platform
+ if spec.architecture.platform_os is not None and isinstance(
+ spec.architecture.platform_os,spack.architecture.OperatingSystem):
+ return False
- def concretize_architecture(self, spec):
- """If the spec already had an architecture, return. Otherwise if
- the root of the DAG has an architecture, then use that.
- Otherwise take the system's default architecture.
-
- Intuition: Architectures won't be set a lot, and generally you
- want the host system's architecture. When architectures are
- mised in a spec, it is likely because the tool requries a
- cross-compiled component, e.g. for tools that run on BlueGene
- or Cray machines. These constraints will likely come directly
- from packages, so require the user to be explicit if they want
- to mess with the architecture, and revert to the default when
- they're not explicit.
- """
- if spec.architecture is not None:
+ if spec.root.architecture and spec.root.architecture.platform_os:
+ if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem):
+ spec.architecture.platform_os = spec.root.architecture.platform_os
+ else:
+ spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os')
+ return True #changed
+
+ def _concretize_target(self, spec):
+ platform = spec.architecture.platform
+ if spec.architecture.target is not None and isinstance(
+ spec.architecture.target, spack.architecture.Target):
return False
+ if spec.root.architecture and spec.root.architecture.target:
+ if isinstance(spec.root.architecture.target,spack.architecture.Target):
+ spec.architecture.target = spec.root.architecture.target
+ else:
+ spec.architecture.target = spec.architecture.platform.target('default_target')
+ return True #changed
- if spec.root.architecture:
- spec.architecture = spec.root.architecture
+ def _concretize_platform(self, spec):
+ if spec.architecture.platform is not None and isinstance(
+ spec.architecture.platform, spack.architecture.Platform):
+ return False
+ if spec.root.architecture and spec.root.architecture.platform:
+ if isinstance(spec.root.architecture.platform,spack.architecture.Platform):
+ spec.architecture.platform = spec.root.architecture.platform
else:
- spec.architecture = spack.architecture.sys_type()
+ spec.architecture.platform = spack.architecture.sys_type()
+ return True #changed?
+
+ def concretize_architecture(self, spec):
+ """If the spec is empty provide the defaults of the platform. If the
+ architecture is not a basestring, then check if either the platform,
+ target or operating system are concretized. If any of the fields are
+ changed then return True. If everything is concretized (i.e the
+ architecture attribute is a namedtuple of classes) then return False.
+ If the target is a string type, then convert the string into a
+ concretized architecture. If it has no architecture and the root of the
+ DAG has an architecture, then use the root otherwise use the defaults
+ on the platform.
+ """
+ if spec.architecture is None:
+ # Set the architecture to all defaults
+ spec.architecture = spack.architecture.Arch()
+ return True
+
+ # Concretize the operating_system and target based of the spec
+ ret = any((self._concretize_platform(spec),
+ self._concretize_operating_system(spec),
+ self._concretize_target(spec)))
+ return ret
- assert(spec.architecture is not None)
- return True # changed
def concretize_variants(self, spec):
@@ -237,6 +272,23 @@ class DefaultConcretizer(object):
build with the compiler that will be used by libraries that
link to this one, to maximize compatibility.
"""
+ # Pass on concretizing the compiler if the target is not yet determined
+ if not spec.architecture.platform_os:
+ #Although this usually means changed, this means awaiting other changes
+ return True
+
+ # Only use a matching compiler if it is of the proper style
+ # Takes advantage of the proper logic already existing in compiler_for_spec
+ # Should think whether this can be more efficient
+ def _proper_compiler_style(cspec, arch):
+ platform = arch.platform
+ compilers = spack.compilers.compilers_for_spec(cspec,
+ platform=platform)
+ return filter(lambda c: c.operating_system ==
+ arch.platform_os, compilers)
+ #return compilers
+
+
all_compilers = spack.compilers.all_compilers()
if (spec.compiler and
@@ -246,6 +298,7 @@ class DefaultConcretizer(object):
#Find the another spec that has a compiler, or the root if none do
other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler)
+
if not other_spec:
other_spec = spec.root
other_compiler = other_spec.compiler
@@ -264,11 +317,75 @@ class DefaultConcretizer(object):
raise UnavailableCompilerVersionError(other_compiler)
# copy concrete version into other_compiler
- spec.compiler = matches[0].copy()
+ index = 0
+ while not _proper_compiler_style(matches[index], spec.architecture):
+ index += 1
+ if index == len(matches) - 1:
+ raise NoValidVersionError(spec)
+ spec.compiler = matches[index].copy()
assert(spec.compiler.concrete)
return True # things changed.
+ def concretize_compiler_flags(self, spec):
+ """
+ The compiler flags are updated to match those of the spec whose
+ compiler is used, defaulting to no compiler flags in the spec.
+ Default specs set at the compiler level will still be added later.
+ """
+
+
+ if not spec.architecture.platform_os:
+ #Although this usually means changed, this means awaiting other changes
+ return True
+
+ ret = False
+ for flag in spack.spec.FlagMap.valid_compiler_flags():
+ try:
+ nearest = next(p for p in spec.traverse(direction='parents')
+ if ((p.compiler == spec.compiler and p is not spec)
+ and flag in p.compiler_flags))
+ if not flag in spec.compiler_flags or \
+ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])):
+ if flag in spec.compiler_flags:
+ spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) |
+ set(nearest.compiler_flags[flag]))
+ else:
+ spec.compiler_flags[flag] = nearest.compiler_flags[flag]
+ ret = True
+
+ except StopIteration:
+ if (flag in spec.root.compiler_flags and ((not flag in spec.compiler_flags) or
+ sorted(spec.compiler_flags[flag]) != sorted(spec.root.compiler_flags[flag]))):
+ if flag in spec.compiler_flags:
+ spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) |
+ set(spec.root.compiler_flags[flag]))
+ else:
+ spec.compiler_flags[flag] = spec.root.compiler_flags[flag]
+ ret = True
+ else:
+ if not flag in spec.compiler_flags:
+ spec.compiler_flags[flag] = []
+
+ # Include the compiler flag defaults from the config files
+ # This ensures that spack will detect conflicts that stem from a change
+ # in default compiler flags.
+ compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture)
+ for flag in compiler.flags:
+ if flag not in spec.compiler_flags:
+ spec.compiler_flags[flag] = compiler.flags[flag]
+ if compiler.flags[flag] != []:
+ ret = True
+ else:
+ if ((sorted(spec.compiler_flags[flag]) != sorted(compiler.flags[flag])) and
+ (not set(spec.compiler_flags[flag]) >= set(compiler.flags[flag]))):
+ ret = True
+ spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) |
+ set(compiler.flags[flag]))
+
+ return ret
+
+
def find_spec(spec, condition):
"""Searches the dag from spec in an intelligent order and looks
for a spec that matches a condition"""
@@ -330,7 +447,6 @@ def cmp_specs(lhs, rhs):
return 0
-
class UnavailableCompilerVersionError(spack.error.SpackError):
"""Raised when there is no available compiler that satisfies a
compiler spec."""
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index ec37bd290c..db0787edc6 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -135,7 +135,7 @@ from yaml.error import MarkedYAMLError
# Hacked yaml for configuration files preserves line numbers.
import spack.util.spack_yaml as syaml
-
+from spack.build_environment import get_path_from_module
"""Dict from section names -> schema for that section."""
section_schemas = {
@@ -146,18 +146,17 @@ section_schemas = {
'additionalProperties': False,
'patternProperties': {
'compilers:?': { # optional colon for overriding site config.
- 'type': 'object',
- 'default': {},
- 'additionalProperties': False,
- 'patternProperties': {
- r'\w[\w-]*': { # architecture
+ 'type': 'array',
+ 'items': {
+ 'compiler': {
'type': 'object',
'additionalProperties': False,
- 'patternProperties': {
- r'\w[\w-]*@\w[\w-]*': { # compiler spec
+ 'required': ['paths', 'spec', 'modules', 'operating_system'],
+ 'properties': {
+ 'paths': {
'type': 'object',
- 'additionalProperties': False,
'required': ['cc', 'cxx', 'f77', 'fc'],
+ 'additionalProperties': False,
'properties': {
'cc': { 'anyOf': [ {'type' : 'string' },
{'type' : 'null' }]},
@@ -167,8 +166,27 @@ section_schemas = {
{'type' : 'null' }]},
'fc': { 'anyOf': [ {'type' : 'string' },
{'type' : 'null' }]},
- },},},},},},},},
-
+ 'cflags': { 'anyOf': [ {'type' : 'string' },
+ {'type' : 'null' }]},
+ 'cxxflags': { 'anyOf': [ {'type' : 'string' },
+ {'type' : 'null' }]},
+ 'fflags': { 'anyOf': [ {'type' : 'string' },
+ {'type' : 'null' }]},
+ 'cppflags': { 'anyOf': [ {'type' : 'string' },
+ {'type' : 'null' }]},
+ 'ldflags': { 'anyOf': [ {'type' : 'string' },
+ {'type' : 'null' }]},
+ 'ldlibs': { 'anyOf': [ {'type' : 'string' },
+ {'type' : 'null' }]}}},
+ 'spec': { 'type': 'string'},
+ 'operating_system': { 'type': 'string'},
+ 'alias': { 'anyOf': [ {'type' : 'string'},
+ {'type' : 'null' }]},
+ 'modules': { 'anyOf': [ {'type' : 'string'},
+ {'type' : 'null' },
+ {'type': 'array'},
+ ]}
+ },},},},},},
'mirrors': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack mirror configuration file schema',
@@ -194,7 +212,6 @@ section_schemas = {
'default': [],
'items': {
'type': 'string'},},},},
-
'packages': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack package configuration file schema',
@@ -224,6 +241,10 @@ section_schemas = {
'type': 'boolean',
'default': True,
},
+ 'modules': {
+ 'type' : 'object',
+ 'default' : {},
+ },
'providers': {
'type': 'object',
'default': {},
@@ -563,8 +584,7 @@ def _merge_yaml(dest, source):
# Source list is prepended (for precedence)
if they_are(list):
- seen = set(source)
- dest[:] = source + [x for x in dest if x not in seen]
+ dest[:] = source + [x for x in dest if x not in source]
return dest
# Source dict is merged into dest.
@@ -667,7 +687,8 @@ def spec_externals(spec):
external_specs = []
pkg_paths = allpkgs.get(name, {}).get('paths', None)
- if not pkg_paths:
+ pkg_modules = allpkgs.get(name, {}).get('modules', None)
+ if (not pkg_paths) and (not pkg_modules):
return []
for external_spec, path in pkg_paths.iteritems():
@@ -678,6 +699,17 @@ def spec_externals(spec):
external_spec = spack.spec.Spec(external_spec, external=path)
if external_spec.satisfies(spec):
external_specs.append(external_spec)
+
+ for external_spec, module in pkg_modules.iteritems():
+ if not module:
+ continue
+
+ path = get_path_from_module(module)
+
+ external_spec = spack.spec.Spec(external_spec, external=path, external_module=module)
+ if external_spec.satisfies(spec):
+ external_specs.append(external_spec)
+
return external_specs
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py
index ee4473e079..f941346bb1 100644
--- a/lib/spack/spack/database.py
+++ b/lib/spack/spack/database.py
@@ -40,7 +40,6 @@ filesystem.
"""
import os
-import time
import socket
import yaml
@@ -56,11 +55,12 @@ from spack.spec import Spec
from spack.error import SpackError
from spack.repository import UnknownPackageError
+
# DB goes in this directory underneath the root
_db_dirname = '.spack-db'
# DB version. This is stuck in the DB file to track changes in format.
-_db_version = Version('0.9')
+_db_version = Version('0.9.1')
# Default timeout for spack database locks is 5 min.
_db_lock_timeout = 60
@@ -69,10 +69,12 @@ _db_lock_timeout = 60
def _autospec(function):
"""Decorator that automatically converts the argument of a single-arg
function to a Spec."""
+
def converter(self, spec_like, *args, **kwargs):
if not isinstance(spec_like, spack.spec.Spec):
spec_like = spack.spec.Spec(spec_like)
return function(self, spec_like, *args, **kwargs)
+
return converter
@@ -92,22 +94,28 @@ class InstallRecord(object):
dependents left.
"""
- def __init__(self, spec, path, installed, ref_count=0):
+
+ def __init__(self, spec, path, installed, ref_count=0, explicit=False):
self.spec = spec
self.path = str(path)
self.installed = bool(installed)
self.ref_count = ref_count
+ self.explicit = explicit
def to_dict(self):
- return { 'spec' : self.spec.to_node_dict(),
- 'path' : self.path,
- 'installed' : self.installed,
- 'ref_count' : self.ref_count }
+ return {
+ 'spec': self.spec.to_node_dict(),
+ 'path': self.path,
+ 'installed': self.installed,
+ 'ref_count': self.ref_count,
+ 'explicit': self.explicit
+ }
@classmethod
def from_dict(cls, spec, dictionary):
d = dictionary
- return InstallRecord(spec, d['path'], d['installed'], d['ref_count'])
+ return InstallRecord(spec, d['path'], d['installed'], d['ref_count'],
+ d.get('explicit', False))
class Database(object):
@@ -142,7 +150,7 @@ class Database(object):
# Set up layout of database files within the db dir
self._index_path = join_path(self._db_dir, 'index.yaml')
- self._lock_path = join_path(self._db_dir, 'lock')
+ self._lock_path = join_path(self._db_dir, 'lock')
# Create needed directories and files
if not os.path.exists(self._db_dir):
@@ -155,17 +163,14 @@ class Database(object):
self.lock = Lock(self._lock_path)
self._data = {}
-
def write_transaction(self, timeout=_db_lock_timeout):
"""Get a write lock context manager for use in a `with` block."""
return WriteTransaction(self, self._read, self._write, timeout)
-
def read_transaction(self, timeout=_db_lock_timeout):
"""Get a read lock context manager for use in a `with` block."""
return ReadTransaction(self, self._read, None, timeout)
-
def _write_to_yaml(self, stream):
"""Write out the databsae to a YAML file.
@@ -181,9 +186,9 @@ class Database(object):
# different paths, it can't differentiate.
# TODO: fix this before we support multiple install locations.
database = {
- 'database' : {
- 'installs' : installs,
- 'version' : str(_db_version)
+ 'database': {
+ 'installs': installs,
+ 'version': str(_db_version)
}
}
@@ -192,32 +197,33 @@ class Database(object):
except YAMLError as e:
raise SpackYAMLError("error writing YAML database:", str(e))
-
def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database.
Does not do any locking.
"""
- if hash_key not in installs:
- parent = read_spec(installs[parent_key]['path'])
-
spec_dict = installs[hash_key]['spec']
+ # Install records don't include hash with spec, so we add it in here
+ # to ensure it is read properly.
+ for name in spec_dict:
+ spec_dict[name]['hash'] = hash_key
+
# Build spec from dict first.
spec = Spec.from_node_dict(spec_dict)
# Add dependencies from other records in the install DB to
# form a full spec.
- for dep_hash in spec_dict[spec.name]['dependencies'].values():
- child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
- spec._add_dependency(child)
+ if 'dependencies' in spec_dict[spec.name]:
+ for dep_hash in spec_dict[spec.name]['dependencies'].values():
+ child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
+ spec._add_dependency(child)
# Specs from the database need to be marked concrete because
# they represent actual installations.
spec._mark_concrete()
return spec
-
def _read_from_yaml(self, stream):
"""
Fill database from YAML, do not maintain old data
@@ -239,22 +245,27 @@ class Database(object):
return
def check(cond, msg):
- if not cond: raise CorruptDatabaseError(self._index_path, msg)
+ if not cond:
+ raise CorruptDatabaseError(self._index_path, msg)
check('database' in yfile, "No 'database' attribute in YAML.")
# High-level file checks
db = yfile['database']
check('installs' in db, "No 'installs' in YAML DB.")
- check('version' in db, "No 'version' in YAML DB.")
+ check('version' in db, "No 'version' in YAML DB.")
+
+ installs = db['installs']
# TODO: better version checking semantics.
version = Version(db['version'])
- if version != _db_version:
+ if version > _db_version:
raise InvalidDatabaseVersionError(_db_version, version)
+ elif version < _db_version:
+ self.reindex(spack.install_layout)
+ installs = dict((k, v.to_dict()) for k, v in self._data.items())
# Iterate through database and check each record.
- installs = db['installs']
data = {}
for hash_key, rec in installs.items():
try:
@@ -265,25 +276,26 @@ class Database(object):
# hashes are the same.
spec_hash = spec.dag_hash()
if not spec_hash == hash_key:
- tty.warn("Hash mismatch in database: %s -> spec with hash %s"
- % (hash_key, spec_hash))
- continue # TODO: is skipping the right thing to do?
+ tty.warn(
+ "Hash mismatch in database: %s -> spec with hash %s" %
+ (hash_key, spec_hash))
+ continue # TODO: is skipping the right thing to do?
# Insert the brand new spec in the database. Each
# spec has its own copies of its dependency specs.
- # TODO: would a more immmutable spec implementation simplify this?
+ # TODO: would a more immmutable spec implementation simplify
+ # this?
data[hash_key] = InstallRecord.from_dict(spec, rec)
except Exception as e:
tty.warn("Invalid database reecord:",
"file: %s" % self._index_path,
"hash: %s" % hash_key,
- "cause: %s" % str(e))
+ "cause: %s: %s" % (type(e).__name__, str(e)))
raise
self._data = data
-
def reindex(self, directory_layout):
"""Build database index from scratch based from a directory layout.
@@ -299,7 +311,11 @@ class Database(object):
for spec in directory_layout.all_specs():
# Create a spec for each known package and add it.
path = directory_layout.path_for_spec(spec)
- self._add(spec, path, directory_layout)
+ old_info = old_data.get(spec.dag_hash())
+ explicit = False
+ if old_info is not None:
+ explicit = old_info.explicit
+ self._add(spec, path, directory_layout, explicit=explicit)
self._check_ref_counts()
@@ -308,7 +324,6 @@ class Database(object):
self._data = old_data
raise
-
def _check_ref_counts(self):
"""Ensure consistency of reference counts in the DB.
@@ -330,9 +345,8 @@ class Database(object):
found = rec.ref_count
if not expected == found:
raise AssertionError(
- "Invalid ref_count: %s: %d (expected %d), in DB %s"
- % (key, found, expected, self._index_path))
-
+ "Invalid ref_count: %s: %d (expected %d), in DB %s" %
+ (key, found, expected, self._index_path))
def _write(self):
"""Write the in-memory database index to its file path.
@@ -354,7 +368,6 @@ class Database(object):
os.remove(temp_file)
raise
-
def _read(self):
"""Re-read Database from the data in the set location.
@@ -369,8 +382,7 @@ class Database(object):
# reindex() takes its own write lock, so no lock here.
self.reindex(spack.install_layout)
-
- def _add(self, spec, path, directory_layout=None):
+ def _add(self, spec, path, directory_layout=None, explicit=False):
"""Add an install record for spec at path to the database.
This assumes that the spec is not already installed. It
@@ -392,11 +404,11 @@ class Database(object):
rec.path = path
else:
- self._data[key] = InstallRecord(spec, path, True)
+ self._data[key] = InstallRecord(spec, path, True,
+ explicit=explicit)
for dep in spec.dependencies.values():
self._increment_ref_count(dep, directory_layout)
-
def _increment_ref_count(self, spec, directory_layout=None):
"""Recursively examine dependencies and update their DB entries."""
key = spec.dag_hash()
@@ -415,7 +427,7 @@ class Database(object):
self._data[key].ref_count += 1
@_autospec
- def add(self, spec, path):
+ def add(self, spec, path, explicit=False):
"""Add spec at path to database, locking and reading DB to sync.
``add()`` will lock and read from the DB on disk.
@@ -424,30 +436,27 @@ class Database(object):
# TODO: ensure that spec is concrete?
# Entire add is transactional.
with self.write_transaction():
- self._add(spec, path)
-
+ self._add(spec, path, explicit=explicit)
def _get_matching_spec_key(self, spec, **kwargs):
"""Get the exact spec OR get a single spec that matches."""
key = spec.dag_hash()
- if not key in self._data:
+ if key not in self._data:
match = self.query_one(spec, **kwargs)
if match:
return match.dag_hash()
raise KeyError("No such spec in database! %s" % spec)
return key
-
@_autospec
def get_record(self, spec, **kwargs):
key = self._get_matching_spec_key(spec, **kwargs)
return self._data[key]
-
def _decrement_ref_count(self, spec):
key = spec.dag_hash()
- if not key in self._data:
+ if key not in self._data:
# TODO: print something here? DB is corrupt, but
# not much we can do.
return
@@ -460,7 +469,6 @@ class Database(object):
for dep in spec.dependencies.values():
self._decrement_ref_count(dep)
-
def _remove(self, spec):
"""Non-locking version of remove(); does real work.
"""
@@ -479,7 +487,6 @@ class Database(object):
# query spec was passed in.
return rec.spec
-
@_autospec
def remove(self, spec):
"""Removes a spec from the database. To be called on uninstall.
@@ -496,7 +503,6 @@ class Database(object):
with self.write_transaction():
return self._remove(spec)
-
@_autospec
def installed_extensions_for(self, extendee_spec):
"""
@@ -507,13 +513,12 @@ class Database(object):
try:
if s.package.extends(extendee_spec):
yield s.package
- except UnknownPackageError as e:
+ except UnknownPackageError:
continue
# skips unknown packages
# TODO: conditional way to do this instead of catching exceptions
-
- def query(self, query_spec=any, known=any, installed=True):
+ def query(self, query_spec=any, known=any, installed=True, explicit=any):
"""Run a query on the database.
``query_spec``
@@ -553,14 +558,16 @@ class Database(object):
for key, rec in self._data.items():
if installed is not any and rec.installed != installed:
continue
- if known is not any and spack.repo.exists(rec.spec.name) != known:
+ if explicit is not any and rec.explicit != explicit:
+ continue
+ if known is not any and spack.repo.exists(
+ rec.spec.name) != known:
continue
if query_spec is any or rec.spec.satisfies(query_spec):
results.append(rec.spec)
return sorted(results)
-
def query_one(self, query_spec, known=any, installed=True):
"""Query for exactly one spec that matches the query spec.
@@ -572,10 +579,9 @@ class Database(object):
assert len(concrete_specs) <= 1
return concrete_specs[0] if concrete_specs else None
-
def missing(self, spec):
with self.read_transaction():
- key = spec.dag_hash()
+ key = spec.dag_hash()
return key in self._data and not self._data[key].installed
@@ -587,7 +593,10 @@ class _Transaction(object):
Timeout for lock is customizable.
"""
- def __init__(self, db, acquire_fn=None, release_fn=None,
+
+ def __init__(self, db,
+ acquire_fn=None,
+ release_fn=None,
timeout=_db_lock_timeout):
self._db = db
self._timeout = timeout
@@ -622,11 +631,11 @@ class WriteTransaction(_Transaction):
class CorruptDatabaseError(SpackError):
def __init__(self, path, msg=''):
super(CorruptDatabaseError, self).__init__(
- "Spack database is corrupt: %s. %s" %(path, msg))
+ "Spack database is corrupt: %s. %s" % (path, msg))
class InvalidDatabaseVersionError(SpackError):
def __init__(self, expected, found):
super(InvalidDatabaseVersionError, self).__init__(
- "Expected database version %s but found version %s"
- % (expected, found))
+ "Expected database version %s but found version %s" %
+ (expected, found))
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 74ee7b0add..ca8f21dc08 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -45,11 +45,8 @@ The available directives are:
* ``resource``
"""
-__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
- 'variant', 'resource']
import re
-import inspect
import os.path
import functools
@@ -67,6 +64,9 @@ from spack.spec import Spec, parse_anonymous_spec
from spack.resource import Resource
from spack.fetch_strategy import from_kwargs
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', 'variant',
+ 'resource']
+
#
# This is a list of all directives, built up as they are defined in
# this file.
@@ -122,15 +122,14 @@ class directive(object):
def __init__(self, dicts=None):
if isinstance(dicts, basestring):
- dicts = (dicts,)
+ dicts = (dicts, )
elif type(dicts) not in (list, tuple):
raise TypeError(
- "dicts arg must be list, tuple, or string. Found %s"
- % type(dicts))
+ "dicts arg must be list, tuple, or string. Found %s" %
+ type(dicts))
self.dicts = dicts
-
def ensure_dicts(self, pkg):
"""Ensure that a package has the dicts required by this directive."""
for d in self.dicts:
@@ -142,7 +141,6 @@ class directive(object):
raise spack.error.SpackError(
"Package %s has non-dict %s attribute!" % (pkg, d))
-
def __call__(self, directive_function):
directives[directive_function.__name__] = self
@@ -259,11 +257,12 @@ def variant(pkg, name, default=False, description=""):
"""Define a variant for the package. Packager can specify a default
value (on or off) as well as a text description."""
- default = bool(default)
+ default = default
description = str(description).strip()
if not re.match(spack.spec.identifier_re, name):
- raise DirectiveError("Invalid variant name in %s: '%s'" % (pkg.name, name))
+ raise DirectiveError("Invalid variant name in %s: '%s'" %
+ (pkg.name, name))
pkg.variants[name] = Variant(default, description)
@@ -271,31 +270,37 @@ def variant(pkg, name, default=False, description=""):
@directive('resources')
def resource(pkg, **kwargs):
"""
- Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
- dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their
- own folder inside spack stage area, and then linked into the stage area of the package that needs them.
+ Define an external resource to be fetched and staged when building the
+ package. Based on the keywords present in the dictionary the appropriate
+ FetchStrategy will be used for the resource. Resources are fetched and
+ staged in their own folder inside spack stage area, and then linked into
+ the stage area of the package that needs them.
List of recognized keywords:
- * 'when' : (optional) represents the condition upon which the resource is needed
- * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage
- area.
- * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage
- area.
+ * 'when' : (optional) represents the condition upon which the resource is
+ needed
+ * 'destination' : (optional) path where to link the resource. This path
+ must be relative to the main package stage area.
+ * 'placement' : (optional) gives the possibility to fine tune how the
+ resource is linked into the main package stage area.
"""
when = kwargs.get('when', pkg.name)
destination = kwargs.get('destination', "")
placement = kwargs.get('placement', None)
# Check if the path is relative
if os.path.isabs(destination):
- message = "The destination keyword of a resource directive can't be an absolute path.\n"
+ message = "The destination keyword of a resource directive can't be"
+ " an absolute path.\n"
message += "\tdestination : '{dest}\n'".format(dest=destination)
raise RuntimeError(message)
# Check if the path falls within the main package stage area
- test_path = 'stage_folder_root/'
- normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path
+ test_path = 'stage_folder_root'
+ normalized_destination = os.path.normpath(join_path(test_path, destination)
+ ) # Normalized absolute path
if test_path not in normalized_destination:
- message = "The destination folder of a resource must fall within the main package stage directory.\n"
+ message = "The destination folder of a resource must fall within the"
+ " main package stage directory.\n"
message += "\tdestination : '{dest}'\n".format(dest=destination)
raise RuntimeError(message)
when_spec = parse_anonymous_spec(when, pkg.name)
@@ -307,6 +312,7 @@ def resource(pkg, **kwargs):
class DirectiveError(spack.error.SpackError):
"""This is raised when something is wrong with a package directive."""
+
def __init__(self, directive, message):
super(DirectiveError, self).__init__(message)
self.directive = directive
@@ -314,6 +320,7 @@ class DirectiveError(spack.error.SpackError):
class CircularReferenceError(DirectiveError):
"""This is raised when something depends on itself."""
+
def __init__(self, directive, package):
super(CircularReferenceError, self).__init__(
directive,
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 32d27d7bd0..7e20365b0f 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -165,7 +165,7 @@ class DirectoryLayout(object):
class YamlDirectoryLayout(DirectoryLayout):
"""Lays out installation directories like this::
<install root>/
- <architecture>/
+ <target>/
<compiler>-<compiler version>/
<name>-<version>-<variants>-<hash>
@@ -207,8 +207,7 @@ class YamlDirectoryLayout(DirectoryLayout):
spec.version,
spec.dag_hash(self.hash_len))
- path = join_path(
- spec.architecture,
+ path = join_path(spec.architecture,
"%s-%s" % (spec.compiler.name, spec.compiler.version),
dir_name)
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 11998ad8d2..30c6228ca4 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -1,4 +1,4 @@
-##############################################################################
+#
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
@@ -21,14 +21,17 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
+#
import collections
import inspect
+import json
import os
import os.path
+import subprocess
class NameModifier(object):
+
def __init__(self, name, **kwargs):
self.name = name
self.args = {'name': name}
@@ -36,57 +39,68 @@ class NameModifier(object):
class NameValueModifier(object):
+
def __init__(self, name, value, **kwargs):
self.name = name
self.value = value
- self.args = {'name': name, 'value': value}
+ self.separator = kwargs.get('separator', ':')
+ self.args = {'name': name, 'value': value, 'delim': self.separator}
self.args.update(kwargs)
class SetEnv(NameValueModifier):
+
def execute(self):
os.environ[self.name] = str(self.value)
class UnsetEnv(NameModifier):
+
def execute(self):
# Avoid throwing if the variable was not set
os.environ.pop(self.name, None)
class SetPath(NameValueModifier):
+
def execute(self):
- string_path = concatenate_paths(self.value)
+ string_path = concatenate_paths(self.value, separator=self.separator)
os.environ[self.name] = string_path
class AppendPath(NameValueModifier):
+
def execute(self):
environment_value = os.environ.get(self.name, '')
- directories = environment_value.split(':') if environment_value else []
+ directories = environment_value.split(
+ self.separator) if environment_value else []
directories.append(os.path.normpath(self.value))
- os.environ[self.name] = ':'.join(directories)
+ os.environ[self.name] = self.separator.join(directories)
class PrependPath(NameValueModifier):
+
def execute(self):
environment_value = os.environ.get(self.name, '')
- directories = environment_value.split(':') if environment_value else []
+ directories = environment_value.split(
+ self.separator) if environment_value else []
directories = [os.path.normpath(self.value)] + directories
- os.environ[self.name] = ':'.join(directories)
+ os.environ[self.name] = self.separator.join(directories)
class RemovePath(NameValueModifier):
+
def execute(self):
environment_value = os.environ.get(self.name, '')
- directories = environment_value.split(':') if environment_value else []
- directories = [os.path.normpath(x)
- for x in directories
+ directories = environment_value.split(
+ self.separator) if environment_value else []
+ directories = [os.path.normpath(x) for x in directories
if x != os.path.normpath(self.value)]
- os.environ[self.name] = ':'.join(directories)
+ os.environ[self.name] = self.separator.join(directories)
class EnvironmentModifications(object):
+
"""
Keeps track of requests to modify the current environment.
@@ -237,18 +251,140 @@ class EnvironmentModifications(object):
for x in actions:
x.execute()
+ @staticmethod
+ def from_sourcing_files(*args, **kwargs):
+ """
+ Creates an instance of EnvironmentModifications that, if executed,
+ has the same effect on the environment as sourcing the files passed as
+ parameters
+
+ Args:
+ *args: list of files to be sourced
-def concatenate_paths(paths):
+ Returns:
+ instance of EnvironmentModifications
+ """
+ env = EnvironmentModifications()
+ # Check if the files are actually there
+ if not all(os.path.isfile(file) for file in args):
+ raise RuntimeError('trying to source non-existing files')
+ # Relevant kwd parameters and formats
+ info = dict(kwargs)
+ info.setdefault('shell', '/bin/bash')
+ info.setdefault('shell_options', '-c')
+ info.setdefault('source_command', 'source')
+ info.setdefault('suppress_output', '&> /dev/null')
+ info.setdefault('concatenate_on_success', '&&')
+
+ shell = '{shell}'.format(**info)
+ shell_options = '{shell_options}'.format(**info)
+ source_file = '{source_command} {file} {concatenate_on_success}'
+ dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' # NOQA: ignore=E501
+ # Construct the command that will be executed
+ command = [source_file.format(file=file, **info) for file in args]
+ command.append(dump_environment)
+ command = ' '.join(command)
+ command = [
+ shell,
+ shell_options,
+ command
+ ]
+
+ # Try to source all the files,
+ proc = subprocess.Popen(
+ command, stdout=subprocess.PIPE, env=os.environ)
+ proc.wait()
+ if proc.returncode != 0:
+ raise RuntimeError('sourcing files returned a non-zero exit code')
+ output = ''.join([line for line in proc.stdout])
+ # Construct a dictionary with all the variables in the new environment
+ after_source_env = dict(json.loads(output))
+ this_environment = dict(os.environ)
+
+ # Filter variables that are not related to sourcing a file
+ to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD'
+ for d in after_source_env, this_environment:
+ for name in to_be_filtered:
+ d.pop(name, None)
+
+ # Fill the EnvironmentModifications instance
+
+ # New variables
+ new_variables = set(after_source_env) - set(this_environment)
+ for x in new_variables:
+ env.set(x, after_source_env[x])
+ # Variables that have been unset
+ unset_variables = set(this_environment) - set(after_source_env)
+ for x in unset_variables:
+ env.unset(x)
+ # Variables that have been modified
+ common_variables = set(this_environment).intersection(set(after_source_env)) # NOQA: ignore=E501
+ modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501
+
+ def return_separator_if_any(first_value, second_value):
+ separators = ':', ';'
+ for separator in separators:
+ if separator in first_value and separator in second_value:
+ return separator
+ return None
+
+ for x in modified_variables:
+ current = this_environment[x]
+ modified = after_source_env[x]
+ sep = return_separator_if_any(current, modified)
+ if sep is None:
+ # We just need to set the variable to the new value
+ env.set(x, after_source_env[x])
+ else:
+ current_list = current.split(sep)
+ modified_list = modified.split(sep)
+ # Paths that have been removed
+ remove_list = [
+ ii for ii in current_list if ii not in modified_list]
+ # Check that nothing has been added in the middle of vurrent
+ # list
+ remaining_list = [
+ ii for ii in current_list if ii in modified_list]
+ start = modified_list.index(remaining_list[0])
+ end = modified_list.index(remaining_list[-1])
+ search = sep.join(modified_list[start:end + 1])
+ if search not in current:
+ # We just need to set the variable to the new value
+ env.set(x, after_source_env[x])
+ break
+ else:
+ try:
+ prepend_list = modified_list[:start]
+ except KeyError:
+ prepend_list = []
+ try:
+ append_list = modified_list[end + 1:]
+ except KeyError:
+ append_list = []
+
+ for item in remove_list:
+ env.remove_path(x, item)
+ for item in append_list:
+ env.append_path(x, item)
+ for item in prepend_list:
+ env.prepend_path(x, item)
+
+ return env
+
+
+def concatenate_paths(paths, separator=':'):
"""
- Concatenates an iterable of paths into a string of column separated paths
+ Concatenates an iterable of paths into a string of paths separated by
+ separator, defaulting to colon
Args:
paths: iterable of paths
+ separator: the separator to use, default ':'
Returns:
string
"""
- return ':'.join(str(item) for item in paths)
+ return separator.join(str(item) for item in paths)
def set_or_unset_not_first(variable, changes, errstream):
@@ -256,16 +392,13 @@ def set_or_unset_not_first(variable, changes, errstream):
Check if we are going to set or unset something after other modifications
have already been requested
"""
- indexes = [ii
- for ii, item in enumerate(changes)
+ indexes = [ii for ii, item in enumerate(changes)
if ii != 0 and type(item) in [SetEnv, UnsetEnv]]
if indexes:
good = '\t \t{context} at {filename}:{lineno}'
nogood = '\t--->\t{context} at {filename}:{lineno}'
message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501
- errstream(
- message.format(
- var=variable))
+ errstream(message.format(var=variable))
for ii, item in enumerate(changes):
print_format = nogood if ii in indexes else good
errstream(print_format.format(**item.args))
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index e05cb13c1e..1953d7c1b3 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -1,4 +1,4 @@
-##############################################################################
+#
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
@@ -21,7 +21,7 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
+#
"""
Fetch strategies are used to download source code into a staging area
in order to build it. They need to define the following methods:
@@ -57,7 +57,6 @@ from spack.version import Version, ver
from spack.util.compression import decompressor_for, extension
import spack.util.pattern as pattern
-
"""List of all fetch strategies, created by FetchStrategy metaclass."""
all_strategies = []
@@ -76,19 +75,24 @@ def _needs_stage(fun):
class FetchStrategy(object):
+
"""Superclass of all fetch strategies."""
enabled = False # Non-abstract subclasses should be enabled.
required_attributes = None # Attributes required in version() args.
class __metaclass__(type):
+
"""This metaclass registers all fetch strategies in a list."""
+
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
- if cls.enabled: all_strategies.append(cls)
+ if cls.enabled:
+ all_strategies.append(cls)
def __init__(self):
- # The stage is initialized late, so that fetch strategies can be constructed
- # at package construction time. This is where things will be fetched.
+ # The stage is initialized late, so that fetch strategies can be
+ # constructed at package construction time. This is where things
+ # will be fetched.
self.stage = None
def set_stage(self, stage):
@@ -97,15 +101,20 @@ class FetchStrategy(object):
self.stage = stage
# Subclasses need to implement these methods
- def fetch(self): pass # Return True on success, False on fail.
+ def fetch(self):
+ pass # Return True on success, False on fail.
- def check(self): pass # Do checksum.
+ def check(self):
+ pass # Do checksum.
- def expand(self): pass # Expand archive.
+ def expand(self):
+ pass # Expand archive.
- def reset(self): pass # Revert to freshly downloaded state.
+ def reset(self):
+ pass # Revert to freshly downloaded state.
- def archive(self, destination): pass # Used to create tarball for mirror.
+ def archive(self, destination):
+ pass # Used to create tarball for mirror.
def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___"
@@ -119,6 +128,7 @@ class FetchStrategy(object):
@pattern.composite(interface=FetchStrategy)
class FetchStrategyComposite(object):
+
"""
Composite for a FetchStrategy object. Implements the GoF composite pattern.
"""
@@ -127,6 +137,7 @@ class FetchStrategyComposite(object):
class URLFetchStrategy(FetchStrategy):
+
"""FetchStrategy that pulls source code from a URL for an archive,
checks the archive against a checksum,and decompresses the archive.
"""
@@ -139,10 +150,12 @@ class URLFetchStrategy(FetchStrategy):
# If URL or digest are provided in the kwargs, then prefer
# those values.
self.url = kwargs.get('url', None)
- if not self.url: self.url = url
+ if not self.url:
+ self.url = url
self.digest = kwargs.get('md5', None)
- if not self.digest: self.digest = digest
+ if not self.digest:
+ self.digest = digest
self.expand_archive = kwargs.get('expand', True)
@@ -167,16 +180,20 @@ class URLFetchStrategy(FetchStrategy):
tty.msg("Trying to fetch from %s" % self.url)
if partial_file:
- save_args = ['-C', '-', # continue partial downloads
- '-o', partial_file] # use a .part file
+ save_args = ['-C',
+ '-', # continue partial downloads
+ '-o',
+ partial_file] # use a .part file
else:
save_args = ['-O']
curl_args = save_args + [
- '-f', # fail on >400 errors
- '-D', '-', # print out HTML headers
- '-L', # resolve 3xx redirects
- self.url, ]
+ '-f', # fail on >400 errors
+ '-D',
+ '-', # print out HTML headers
+ '-L', # resolve 3xx redirects
+ self.url,
+ ]
if sys.stdout.isatty():
curl_args.append('-#') # status bar when using a tty
@@ -184,8 +201,7 @@ class URLFetchStrategy(FetchStrategy):
curl_args.append('-sS') # just errors when not.
# Run curl but grab the mime type from the http headers
- headers = spack.curl(
- *curl_args, output=str, fail_on_error=False)
+ headers = spack.curl(*curl_args, output=str, fail_on_error=False)
if spack.curl.returncode != 0:
# clean up archive on failure.
@@ -198,34 +214,38 @@ class URLFetchStrategy(FetchStrategy):
if spack.curl.returncode == 22:
# This is a 404. Curl will print the error.
raise FailedDownloadError(
- self.url, "URL %s was not found!" % self.url)
+ self.url, "URL %s was not found!" % self.url)
elif spack.curl.returncode == 60:
# This is a certificate error. Suggest spack -k
raise FailedDownloadError(
- self.url,
- "Curl was unable to fetch due to invalid certificate. "
- "This is either an attack, or your cluster's SSL configuration "
- "is bad. If you believe your SSL configuration is bad, you "
- "can try running spack -k, which will not check SSL certificates."
- "Use this at your own risk.")
+ self.url,
+ "Curl was unable to fetch due to invalid certificate. "
+ "This is either an attack, or your cluster's SSL "
+ "configuration is bad. If you believe your SSL "
+ "configuration is bad, you can try running spack -k, "
+ "which will not check SSL certificates."
+ "Use this at your own risk.")
else:
# This is some other curl error. Curl will print the
# error, but print a spack message too
raise FailedDownloadError(
- self.url, "Curl failed with error %d" % spack.curl.returncode)
+ self.url,
+ "Curl failed with error %d" % spack.curl.returncode)
# Check if we somehow got an HTML file rather than the archive we
# asked for. We only look at the last content type, to handle
# redirects properly.
content_types = re.findall(r'Content-Type:[^\r\n]+', headers)
if content_types and 'text/html' in content_types[-1]:
- tty.warn("The contents of " + self.archive_file + " look like HTML.",
+ tty.warn("The contents of ",
+ (self.archive_file if self.archive_file is not None
+ else "the archive"),
+ " look like HTML.",
"The checksum will likely be bad. If it is, you can use",
- "'spack clean <package>' to remove the bad archive, then fix",
- "your internet gateway issue and install again.")
-
+ "'spack clean <package>' to remove the bad archive, then",
+ "fix your internet gateway issue and install again.")
if save_file:
os.rename(partial_file, save_file)
@@ -247,14 +267,16 @@ class URLFetchStrategy(FetchStrategy):
self.stage.chdir()
if not self.archive_file:
- raise NoArchiveFileError("URLFetchStrategy couldn't find archive file",
- "Failed on expand() for URL %s" % self.url)
+ raise NoArchiveFileError(
+ "URLFetchStrategy couldn't find archive file",
+ "Failed on expand() for URL %s" % self.url)
decompress = decompressor_for(self.archive_file)
# Expand all tarballs in their own directory to contain
# exploding tarballs.
- tarball_container = os.path.join(self.stage.path, "spack-expanded-archive")
+ tarball_container = os.path.join(self.stage.path,
+ "spack-expanded-archive")
mkdirp(tarball_container)
os.chdir(tarball_container)
decompress(self.archive_file)
@@ -295,20 +317,25 @@ class URLFetchStrategy(FetchStrategy):
"""Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository."""
if not self.digest:
- raise NoDigestError("Attempt to check URLFetchStrategy with no digest.")
+ raise NoDigestError(
+ "Attempt to check URLFetchStrategy with no digest.")
checker = crypto.Checker(self.digest)
if not checker.check(self.archive_file):
raise ChecksumError(
- "%s checksum failed for %s" % (checker.hash_name, self.archive_file),
- "Expected %s but got %s" % (self.digest, checker.sum))
+ "%s checksum failed for %s" %
+ (checker.hash_name, self.archive_file),
+ "Expected %s but got %s" % (self.digest, checker.sum))
@_needs_stage
def reset(self):
- """Removes the source path if it exists, then re-expands the archive."""
+ """
+ Removes the source path if it exists, then re-expands the archive.
+ """
if not self.archive_file:
- raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching",
- "Failed on reset() for URL %s" % self.url)
+ raise NoArchiveFileError(
+ "Tried to reset URLFetchStrategy before fetching",
+ "Failed on reset() for URL %s" % self.url)
# Remove everythigng but the archive from the stage
for filename in os.listdir(self.stage.path):
@@ -331,20 +358,23 @@ class URLFetchStrategy(FetchStrategy):
class VCSFetchStrategy(FetchStrategy):
+
def __init__(self, name, *rev_types, **kwargs):
super(VCSFetchStrategy, self).__init__()
self.name = name
# Set a URL based on the type of fetch strategy.
self.url = kwargs.get(name, None)
- if not self.url: raise ValueError(
+ if not self.url:
+ raise ValueError(
"%s requires %s argument." % (self.__class__, name))
# Ensure that there's only one of the rev_types
if sum(k in kwargs for k in rev_types) > 1:
raise FetchStrategyError(
- "Supply only one of %s to fetch with %s" % (
- comma_or(rev_types), name))
+ "Supply only one of %s to fetch with %s" % (
+ comma_or(rev_types), name
+ ))
# Set attributes for each rev type.
for rt in rev_types:
@@ -382,32 +412,95 @@ class VCSFetchStrategy(FetchStrategy):
return "%s<%s>" % (self.__class__, self.url)
+class GoFetchStrategy(VCSFetchStrategy):
+
+ """
+ Fetch strategy that employs the `go get` infrastructure
+ Use like this in a package:
+
+ version('name',
+ go='github.com/monochromegane/the_platinum_searcher/...')
+
+ Go get does not natively support versions, they can be faked with git
+ """
+ enabled = True
+ required_attributes = ('go', )
+
+ def __init__(self, **kwargs):
+ # Discards the keywords in kwargs that may conflict with the next
+ # call to __init__
+ forwarded_args = copy.copy(kwargs)
+ forwarded_args.pop('name', None)
+
+ super(GoFetchStrategy, self).__init__('go', **forwarded_args)
+ self._go = None
+
+ @property
+ def go_version(self):
+ vstring = self.go('version', output=str).split(' ')[2]
+ return Version(vstring)
+
+ @property
+ def go(self):
+ if not self._go:
+ self._go = which('go', required=True)
+ return self._go
+
+ @_needs_stage
+ def fetch(self):
+ self.stage.chdir()
+
+ tty.msg("Trying to get go resource:", self.url)
+
+ try:
+ os.mkdir('go')
+ except OSError:
+ pass
+ env = dict(os.environ)
+ env['GOPATH'] = os.path.join(os.getcwd(), 'go')
+ self.go('get', '-v', '-d', self.url, env=env)
+
+ def archive(self, destination):
+ super(GoFetchStrategy, self).archive(destination, exclude='.git')
+
+ @_needs_stage
+ def reset(self):
+ self.stage.chdir_to_source()
+ self.go('clean')
+
+ def __str__(self):
+ return "[go] %s" % self.url
+
+
class GitFetchStrategy(VCSFetchStrategy):
- """Fetch strategy that gets source code from a git repository.
- Use like this in a package:
- version('name', git='https://github.com/project/repo.git')
+ """
+ Fetch strategy that gets source code from a git repository.
+ Use like this in a package:
+
+ version('name', git='https://github.com/project/repo.git')
- Optionally, you can provide a branch, or commit to check out, e.g.:
+ Optionally, you can provide a branch, or commit to check out, e.g.:
- version('1.1', git='https://github.com/project/repo.git', tag='v1.1')
+ version('1.1', git='https://github.com/project/repo.git', tag='v1.1')
- You can use these three optional attributes in addition to ``git``:
+ You can use these three optional attributes in addition to ``git``:
- * ``branch``: Particular branch to build from (default is master)
- * ``tag``: Particular tag to check out
- * ``commit``: Particular commit hash in the repo
+ * ``branch``: Particular branch to build from (default is master)
+ * ``tag``: Particular tag to check out
+ * ``commit``: Particular commit hash in the repo
"""
enabled = True
- required_attributes = ('git',)
+ required_attributes = ('git', )
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next call
+ # to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(GitFetchStrategy, self).__init__(
- 'git', 'tag', 'branch', 'commit', **forwarded_args)
+ 'git', 'tag', 'branch', 'commit', **forwarded_args)
self._git = None
@property
@@ -501,6 +594,7 @@ class GitFetchStrategy(VCSFetchStrategy):
class SvnFetchStrategy(VCSFetchStrategy):
+
"""Fetch strategy that gets source code from a subversion repository.
Use like this in a package:
@@ -515,12 +609,13 @@ class SvnFetchStrategy(VCSFetchStrategy):
required_attributes = ['svn']
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next call
+ # to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(SvnFetchStrategy, self).__init__(
- 'svn', 'revision', **forwarded_args)
+ 'svn', 'revision', **forwarded_args)
self._svn = None
if self.revision is not None:
self.revision = str(self.revision)
@@ -576,32 +671,36 @@ class SvnFetchStrategy(VCSFetchStrategy):
class HgFetchStrategy(VCSFetchStrategy):
- """Fetch strategy that gets source code from a Mercurial repository.
- Use like this in a package:
- version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2')
+ """
+ Fetch strategy that gets source code from a Mercurial repository.
+ Use like this in a package:
+
+ version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2')
- Optionally, you can provide a branch, or revision to check out, e.g.:
+ Optionally, you can provide a branch, or revision to check out, e.g.:
- version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus')
+ version('torus',
+ hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus')
- You can use the optional 'revision' attribute to check out a
- branch, tag, or particular revision in hg. To prevent
- non-reproducible builds, using a moving target like a branch is
- discouraged.
+ You can use the optional 'revision' attribute to check out a
+ branch, tag, or particular revision in hg. To prevent
+ non-reproducible builds, using a moving target like a branch is
+ discouraged.
- * ``revision``: Particular revision, branch, or tag.
+ * ``revision``: Particular revision, branch, or tag.
"""
enabled = True
required_attributes = ['hg']
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next call
+ # to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(HgFetchStrategy, self).__init__(
- 'hg', 'revision', **forwarded_args)
+ 'hg', 'revision', **forwarded_args)
self._hg = None
@property
@@ -675,7 +774,8 @@ def from_kwargs(**kwargs):
return fetcher(**kwargs)
# Raise an error in case we can't instantiate any known strategy
message = "Cannot instantiate any FetchStrategy"
- long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+ long_message = message + " from the given arguments : {arguments}".format(
+ srguments=kwargs)
raise FetchError(message, long_message)
@@ -687,7 +787,7 @@ def for_package_version(pkg, version):
"""Determine a fetch strategy based on the arguments supplied to
version() in the package description."""
# If it's not a known version, extrapolate one.
- if not version in pkg.versions:
+ if version not in pkg.versions:
url = pkg.url_for_version(version)
if not url:
raise InvalidArgsError(pkg, version)
@@ -716,37 +816,44 @@ def for_package_version(pkg, version):
class FetchError(spack.error.SpackError):
+
def __init__(self, msg, long_msg=None):
super(FetchError, self).__init__(msg, long_msg)
class FailedDownloadError(FetchError):
+
"""Raised wen a download fails."""
def __init__(self, url, msg=""):
super(FailedDownloadError, self).__init__(
- "Failed to fetch file from URL: %s" % url, msg)
+ "Failed to fetch file from URL: %s" % url, msg)
self.url = url
class NoArchiveFileError(FetchError):
+
def __init__(self, msg, long_msg):
super(NoArchiveFileError, self).__init__(msg, long_msg)
class NoDigestError(FetchError):
+
def __init__(self, msg, long_msg=None):
super(NoDigestError, self).__init__(msg, long_msg)
class InvalidArgsError(FetchError):
+
def __init__(self, pkg, version):
- msg = "Could not construct a fetch strategy for package %s at version %s"
+ msg = ("Could not construct a fetch strategy for package %s at "
+ "version %s")
msg %= (pkg.name, version)
super(InvalidArgsError, self).__init__(msg)
class ChecksumError(FetchError):
+
"""Raised when archive fails to checksum."""
def __init__(self, message, long_msg=None):
@@ -754,8 +861,10 @@ class ChecksumError(FetchError):
class NoStageError(FetchError):
+
"""Raised when fetch operations are called before set_stage()."""
def __init__(self, method):
super(NoStageError, self).__init__(
- "Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
+ "Must call FetchStrategy.set_stage() before calling %s" %
+ method.__name__)
diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py
index 0f63b0e05a..9010b84154 100644
--- a/lib/spack/spack/hooks/licensing.py
+++ b/lib/spack/spack/hooks/licensing.py
@@ -26,7 +26,7 @@ import os
import spack
import llnl.util.tty as tty
-from llnl.util.filesystem import join_path
+from llnl.util.filesystem import join_path, mkdirp
def pre_install(pkg):
@@ -154,6 +154,9 @@ def symlink_license(pkg):
target = pkg.global_license_file
for filename in pkg.license_files:
link_name = join_path(pkg.prefix, filename)
+ license_dir = os.path.dirname(link_name)
+ if not os.path.exists(license_dir):
+ mkdirp(license_dir)
if os.path.exists(target):
os.symlink(target, link_name)
tty.msg("Added local symlink %s to global license file" %
diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py
index 83d67ea225..cb0ad42b14 100644
--- a/lib/spack/spack/hooks/sbang.py
+++ b/lib/spack/spack/hooks/sbang.py
@@ -24,7 +24,6 @@
##############################################################################
import os
-from llnl.util.filesystem import *
import llnl.util.tty as tty
import spack
@@ -34,6 +33,7 @@ import spack.modules
# here, as it is the shortest I could find on a modern OS.
shebang_limit = 127
+
def shebang_too_long(path):
"""Detects whether a file has a shebang line that is too long."""
with open(path, 'r') as script:
@@ -57,16 +57,10 @@ def filter_shebang(path):
if original.startswith(new_sbang_line):
return
- backup = path + ".shebang.bak"
- os.rename(path, backup)
-
with open(path, 'w') as new_file:
new_file.write(new_sbang_line)
new_file.write(original)
- copy_mode(backup, path)
- unset_executable_mode(backup)
-
tty.warn("Patched overly long shebang in %s" % path)
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index a35e21c3db..d2b819e80a 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -485,9 +485,9 @@ class TclModule(EnvModule):
path = join_path(spack.share_path, "modules")
environment_modifications_formats = {
- PrependPath: 'prepend-path {name} \"{value}\"\n',
- AppendPath: 'append-path {name} \"{value}\"\n',
- RemovePath: 'remove-path {name} \"{value}\"\n',
+ PrependPath: 'prepend-path --delim "{delim}" {name} \"{value}\"\n',
+ AppendPath: 'append-path --delim "{delim}" {name} \"{value}\"\n',
+ RemovePath: 'remove-path --delim "{delim}" {name} \"{value}\"\n',
SetEnv: 'setenv {name} \"{value}\"\n',
UnsetEnv: 'unsetenv {name}\n'
}
diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py
index 5fda9328d6..170ef3cea2 100644
--- a/lib/spack/spack/multimethod.py
+++ b/lib/spack/spack/multimethod.py
@@ -146,12 +146,12 @@ class when(object):
def install(self, prefix):
# Do default install
- @when('=chaos_5_x86_64_ib')
+ @when('arch=chaos_5_x86_64_ib')
def install(self, prefix):
# This will be executed instead of the default install if
# the package's sys_type() is chaos_5_x86_64_ib.
- @when('=bgqos_0")
+ @when('arch=bgqos_0")
def install(self, prefix):
# This will be executed if the package's sys_type is bgqos_0
diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/operating_systems/__init__.py
diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py
new file mode 100644
index 0000000000..c160a60be8
--- /dev/null
+++ b/lib/spack/spack/operating_systems/cnl.py
@@ -0,0 +1,62 @@
+import re
+import os
+
+from spack.architecture import OperatingSystem
+from spack.util.executable import *
+import spack.spec
+from spack.util.multiproc import parmap
+import spack.compilers
+
+class Cnl(OperatingSystem):
+ """ Compute Node Linux (CNL) is the operating system used for the Cray XC
+ series super computers. It is a very stripped down version of GNU/Linux.
+ Any compilers found through this operating system will be used with
+ modules. If updated, user must make sure that version and name are
+ updated to indicate that OS has been upgraded (or downgraded)
+ """
+ def __init__(self):
+ name = 'CNL'
+ version = '10'
+ super(Cnl, self).__init__(name, version)
+
+
+ def find_compilers(self, *paths):
+ types = spack.compilers.all_compiler_types()
+ compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
+
+ # ensure all the version calls we made are cached in the parent
+ # process, as well. This speeds up Spack a lot.
+ clist = reduce(lambda x,y: x+y, compiler_lists)
+ return clist
+
+
+ def find_compiler(self, cmp_cls, *paths):
+ compilers = []
+ if cmp_cls.PrgEnv:
+ if not cmp_cls.PrgEnv_compiler:
+ tty.die('Must supply PrgEnv_compiler with PrgEnv')
+
+ modulecmd = which('modulecmd')
+ modulecmd.add_default_arg('python')
+
+ # Save the environment variable to restore later
+ old_modulepath = os.environ['MODULEPATH']
+ # if given any explicit paths, search them for module files too
+ if paths:
+ module_paths = ':' + ':'.join(p for p in paths)
+ os.environ['MODULEPATH'] = module_paths
+
+ output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str)
+ matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output)
+ for name, version in matches:
+ v = version
+ comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self,
+ ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v])
+
+ compilers.append(comp)
+
+ # Restore modulepath environment variable
+ if paths:
+ os.environ['MODULEPATH'] = old_modulepath
+
+ return compilers
diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py
new file mode 100644
index 0000000000..2e3c72719b
--- /dev/null
+++ b/lib/spack/spack/operating_systems/linux_distro.py
@@ -0,0 +1,22 @@
+import re
+import platform as py_platform
+from spack.architecture import OperatingSystem
+
+class LinuxDistro(OperatingSystem):
+ """ This class will represent the autodetected operating system
+ for a Linux System. Since there are many different flavors of
+ Linux, this class will attempt to encompass them all through
+ autodetection using the python module platform and the method
+ platform.dist()
+ """
+ def __init__(self):
+ distname, version, _ = py_platform.linux_distribution(
+ full_distribution_name=False)
+
+ # Grabs major version from tuple on redhat; on other platforms
+ # grab the first legal identifier in the version field. On
+ # debian you get things like 'wheezy/sid'; sid means unstable.
+ # We just record 'wheezy' and don't get quite so detailed.
+ version = re.split(r'[^\w-]', version)[0]
+
+ super(LinuxDistro, self).__init__(distname, version)
diff --git a/lib/spack/spack/operating_systems/mac_os.py b/lib/spack/spack/operating_systems/mac_os.py
new file mode 100644
index 0000000000..f35b3ca577
--- /dev/null
+++ b/lib/spack/spack/operating_systems/mac_os.py
@@ -0,0 +1,29 @@
+import platform as py_platform
+from spack.architecture import OperatingSystem
+
+class MacOs(OperatingSystem):
+ """This class represents the macOS operating system. This will be
+ auto detected using the python platform.mac_ver. The macOS
+ platform will be represented using the major version operating
+ system name, i.e el capitan, yosemite...etc.
+ """
+
+ def __init__(self):
+ """ Autodetects the mac version from a dictionary. Goes back as
+ far as 10.6 snowleopard. If the user has an older mac then
+ the version will just be a generic mac_os.
+ """
+ mac_releases = {'10.6': "snowleopard",
+ "10.7": "lion",
+ "10.8": "mountainlion",
+ "10.9": "mavericks",
+ "10.10": "yosemite",
+ "10.11": "elcapitan",
+ "10.12": "sierra"}
+
+ mac_ver = py_platform.mac_ver()[0][:-2]
+ name = mac_releases.get(mac_ver, "macos")
+ super(MacOs, self).__init__(name, mac_ver)
+
+ def __str__(self):
+ return self.name
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 8167341127..98fd51b262 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -398,13 +398,19 @@ class Package(object):
spack.repo.get(self.extendee_spec)._check_extendable()
@property
+ def global_license_dir(self):
+ """Returns the directory where global license files for all
+ packages are stored."""
+ spack_root = ancestor(__file__, 4)
+ return join_path(spack_root, 'etc', 'spack', 'licenses')
+
+ @property
def global_license_file(self):
- """Returns the path where a global license file should be stored."""
+ """Returns the path where a global license file for this
+ particular package should be stored."""
if not self.license_files:
return
- spack_root = ancestor(__file__, 4)
- global_license_dir = join_path(spack_root, 'etc', 'spack', 'licenses')
- return join_path(global_license_dir, self.name,
+ return join_path(self.global_license_dir, self.name,
os.path.basename(self.license_files[0]))
@property
@@ -676,11 +682,13 @@ class Package(object):
return self.spec.prefix
@property
+ #TODO: Change this to architecture
def compiler(self):
"""Get the spack.compiler.Compiler object used to build this package"""
if not self.spec.concrete:
raise ValueError("Can only get a compiler for a concrete package.")
- return spack.compilers.compiler_for_spec(self.spec.compiler)
+ return spack.compilers.compiler_for_spec(self.spec.compiler,
+ self.spec.architecture)
def url_version(self, version):
"""
@@ -857,7 +865,8 @@ class Package(object):
skip_patch=False,
verbose=False,
make_jobs=None,
- fake=False):
+ fake=False,
+ explicit=False):
"""Called by commands to install a package and its dependencies.
Package implementations should override install() to describe
@@ -887,6 +896,11 @@ class Package(object):
# Ensure package is not already installed
if spack.install_layout.check_installed(self.spec):
tty.msg("%s is already installed in %s" % (self.name, self.prefix))
+ rec = spack.installed_db.get_record(self.spec)
+ if (not rec.explicit) and explicit:
+ with spack.installed_db.write_transaction():
+ rec = spack.installed_db.get_record(self.spec)
+ rec.explicit = True
return
tty.msg("Installing %s" % self.name)
@@ -995,7 +1009,7 @@ class Package(object):
# note: PARENT of the build process adds the new package to
# the database, so that we don't need to re-read from file.
- spack.installed_db.add(self.spec, self.prefix)
+ spack.installed_db.add(self.spec, self.prefix, explicit=explicit)
def sanity_check_prefix(self):
"""This function checks whether install succeeded."""
diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/platforms/__init__.py
diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py
new file mode 100644
index 0000000000..e0eb76f336
--- /dev/null
+++ b/lib/spack/spack/platforms/bgq.py
@@ -0,0 +1,18 @@
+import os
+from spack.architecture import Platform, Target
+
+class Bgq(Platform):
+ priority = 30
+ front_end = 'power7'
+ back_end = 'powerpc'
+ default = 'powerpc'
+
+ def __init__(self):
+ super(Bgq, self).__init__('bgq')
+ self.add_target(self.front_end, Target(self.front_end))
+ self.add_target(self.back_end, Target(self.back_end,))
+
+ @classmethod
+ def detect(self):
+ return os.path.exists('/bgsys')
+
diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py
new file mode 100644
index 0000000000..e710303e23
--- /dev/null
+++ b/lib/spack/spack/platforms/cray_xc.py
@@ -0,0 +1,46 @@
+import os
+from spack.architecture import Platform, Target
+from spack.operating_systems.linux_distro import LinuxDistro
+from spack.operating_systems.cnl import Cnl
+
+class CrayXc(Platform):
+ priority = 20
+ front_end = 'sandybridge'
+ back_end = 'ivybridge'
+ default = 'ivybridge'
+
+ front_os = "SuSE11"
+ back_os = "CNL10"
+ default_os = "CNL10"
+
+ def __init__(self):
+ ''' Since cori doesn't have ivybridge as a front end it's better
+ if we use CRAY_CPU_TARGET as the default. This will ensure
+ that if we're on a XC-40 or XC-30 then we can detect the target
+ '''
+ super(CrayXc, self).__init__('cray_xc')
+
+ # Handle the default here so we can check for a key error
+ if 'CRAY_CPU_TARGET' in os.environ:
+ self.default = os.environ['CRAY_CPU_TARGET']
+
+ # Change the defaults to haswell if we're on an XC40
+ if self.default == 'haswell':
+ self.front_end = self.default
+ self.back_end = self.default
+
+ # Could switch to use modules and fe targets for front end
+ # Currently using compilers by path for front end.
+ self.add_target('sandybridge', Target('sandybridge'))
+ self.add_target('ivybridge',
+ Target('ivybridge', 'craype-ivybridge'))
+ self.add_target('haswell',
+ Target('haswell','craype-haswell'))
+
+ self.add_operating_system('SuSE11', LinuxDistro())
+ self.add_operating_system('CNL10', Cnl())
+
+ @classmethod
+ def detect(self):
+ return os.path.exists('/opt/cray/craype')
+
diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py
new file mode 100644
index 0000000000..d47dd640f9
--- /dev/null
+++ b/lib/spack/spack/platforms/darwin.py
@@ -0,0 +1,26 @@
+import subprocess
+from spack.architecture import Platform, Target
+from spack.operating_systems.mac_os import MacOs
+
+class Darwin(Platform):
+ priority = 89
+ front_end = 'x86_64'
+ back_end = 'x86_64'
+ default = 'x86_64'
+
+ def __init__(self):
+ super(Darwin, self).__init__('darwin')
+ self.add_target(self.default, Target(self.default))
+ mac_os = MacOs()
+
+ self.default_os = str(mac_os)
+ self.front_os = str(mac_os)
+ self.back_os = str(mac_os)
+
+ self.add_operating_system(str(mac_os), mac_os)
+
+ @classmethod
+ def detect(self):
+ platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE)
+ platform, _ = platform.communicate()
+ return 'darwin' in platform.strip().lower()
diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py
new file mode 100644
index 0000000000..4d8adac384
--- /dev/null
+++ b/lib/spack/spack/platforms/linux.py
@@ -0,0 +1,24 @@
+import subprocess
+from spack.architecture import Platform, Target
+from spack.operating_systems.linux_distro import LinuxDistro
+
+class Linux(Platform):
+ priority = 90
+ front_end = 'x86_64'
+ back_end = 'x86_64'
+ default = 'x86_64'
+
+ def __init__(self):
+ super(Linux, self).__init__('linux')
+ self.add_target(self.default, Target(self.default))
+ linux_dist = LinuxDistro()
+ self.default_os = str(linux_dist)
+ self.front_os = self.default_os
+ self.back_os = self.default_os
+ self.add_operating_system(str(linux_dist), linux_dist)
+
+ @classmethod
+ def detect(self):
+ platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE)
+ platform, _ = platform.communicate()
+ return 'linux' in platform.strip().lower()
diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py
new file mode 100644
index 0000000000..8fa2585a7a
--- /dev/null
+++ b/lib/spack/spack/platforms/test.py
@@ -0,0 +1,28 @@
+import subprocess
+from spack.architecture import Platform, Target
+from spack.operating_systems.linux_distro import LinuxDistro
+from spack.operating_systems.cnl import Cnl
+
+
+class Test(Platform):
+ priority = 1000000
+ front_end = 'x86_32'
+ back_end = 'x86_64'
+ default = 'x86_64'
+
+ back_os = 'CNL10'
+ default_os = 'CNL10'
+
+ def __init__(self):
+ super(Test, self).__init__('test')
+ self.add_target(self.default, Target(self.default))
+ self.add_target(self.front_end, Target(self.front_end))
+
+ self.add_operating_system(self.default_os, Cnl())
+ linux_dist = LinuxDistro()
+ self.front_os = linux_dist.name
+ self.add_operating_system(self.front_os, linux_dist)
+
+ @classmethod
+ def detect(self):
+ return True
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 89a023a750..54219ec1b4 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -18,9 +18,9 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
Spack allows very fine-grained control over how packages are installed and
@@ -72,7 +72,9 @@ Here is the EBNF grammar for a spec::
dep_list = { ^ spec }
spec = id [ options ]
options = { @version-list | +variant | -variant | ~variant |
- %compiler | =architecture }
+ %compiler | arch=architecture | [ flag ]=value}
+ flag = { cflags | cxxflags | fcflags | fflags | cppflags |
+ ldflags | ldlibs }
variant = id
architecture = id
compiler = id [ version-list ]
@@ -80,6 +82,9 @@ Here is the EBNF grammar for a spec::
version = id | id: | :id | id:id
id = [A-Za-z0-9_][A-Za-z0-9_.-]*
+Identifiers using the <name>=<value> command, such as architectures and
+compiler flags, require a space before the name.
+
There is one context-sensitive part: ids in versions may contain '.', while
other ids may not.
@@ -94,6 +99,7 @@ import sys
import itertools
import hashlib
import base64
+import imp
from StringIO import StringIO
from operator import attrgetter
import yaml
@@ -102,39 +108,45 @@ from yaml.error import MarkedYAMLError
import llnl.util.tty as tty
from llnl.util.lang import *
from llnl.util.tty.color import *
+from llnl.util.filesystem import join_path
import spack
+import spack.architecture
import spack.parse
import spack.error
import spack.compilers as compilers
+# TODO: move display_specs to some other location.
+from spack.cmd.find import display_specs
from spack.version import *
from spack.util.string import *
from spack.util.prefix import Prefix
+from spack.util.naming import mod_to_class
from spack.virtual import ProviderIndex
+from spack.build_environment import get_path_from_module, load_module
# Valid pattern for an identifier in Spack
identifier_re = r'\w[\w-]*'
# Convenient names for color formats so that other things can use them
-compiler_color = '@g'
-version_color = '@c'
-architecture_color = '@m'
-enabled_variant_color = '@B'
+compiler_color = '@g'
+version_color = '@c'
+architecture_color = '@m'
+enabled_variant_color = '@B'
disabled_variant_color = '@r'
-dependency_color = '@.'
-hash_color = '@K'
+dependency_color = '@.'
+hash_color = '@K'
"""This map determines the coloring of specs when using color output.
We make the fields different colors to enhance readability.
See spack.color for descriptions of the color codes. """
-color_formats = {'%' : compiler_color,
- '@' : version_color,
- '=' : architecture_color,
- '+' : enabled_variant_color,
- '~' : disabled_variant_color,
- '^' : dependency_color,
- '#' : hash_color }
+color_formats = {'%': compiler_color,
+ '@': version_color,
+ '=': architecture_color,
+ '+': enabled_variant_color,
+ '~': disabled_variant_color,
+ '^': dependency_color,
+ '#': hash_color}
"""Regex used for splitting by spec field separators."""
_separators = '[%s]' % ''.join(color_formats.keys())
@@ -151,7 +163,7 @@ def index_specs(specs):
"""
spec_dict = {}
for spec in specs:
- if not spec.name in spec_dict:
+ if spec.name not in spec_dict:
spec_dict[spec.name] = []
spec_dict[spec.name].append(spec)
return spec_dict
@@ -198,8 +210,8 @@ class CompilerSpec(object):
else:
raise TypeError(
- "Can only build CompilerSpec from string or CompilerSpec." +
- " Found %s" % type(arg))
+ "Can only build CompilerSpec from string or " +
+ "CompilerSpec. Found %s" % type(arg))
elif nargs == 2:
name, version = args
@@ -211,23 +223,19 @@ class CompilerSpec(object):
raise TypeError(
"__init__ takes 1 or 2 arguments. (%d given)" % nargs)
-
def _add_version(self, version):
self.versions.add(version)
-
def _autospec(self, compiler_spec_like):
if isinstance(compiler_spec_like, CompilerSpec):
return compiler_spec_like
return CompilerSpec(compiler_spec_like)
-
def satisfies(self, other, strict=False):
other = self._autospec(other)
return (self.name == other.name and
self.versions.satisfies(other.versions, strict=strict))
-
def constrain(self, other):
"""Intersect self's versions with other.
@@ -241,44 +249,37 @@ class CompilerSpec(object):
return self.versions.intersect(other.versions)
-
@property
def concrete(self):
"""A CompilerSpec is concrete if its versions are concrete and there
is an available compiler with the right version."""
return self.versions.concrete
-
@property
def version(self):
if not self.concrete:
raise SpecError("Spec is not concrete: " + str(self))
return self.versions[0]
-
def copy(self):
clone = CompilerSpec.__new__(CompilerSpec)
clone.name = self.name
clone.versions = self.versions.copy()
return clone
-
def _cmp_key(self):
return (self.name, self.versions)
-
def to_dict(self):
- d = {'name' : self.name}
+ d = {'name': self.name}
d.update(self.versions.to_dict())
- return { 'compiler' : d }
-
+ return {'compiler': d}
@staticmethod
def from_dict(d):
d = d['compiler']
return CompilerSpec(d['name'], VersionList.from_dict(d))
-
def __str__(self):
out = self.name
if self.versions and self.versions != _any_version:
@@ -292,43 +293,43 @@ class CompilerSpec(object):
@key_ordering
class VariantSpec(object):
+
"""Variants are named, build-time options for a package. Names depend
on the particular package being built, and each named variant can
be enabled or disabled.
"""
- def __init__(self, name, enabled):
+ def __init__(self, name, value):
self.name = name
- self.enabled = enabled
-
+ self.value = value
def _cmp_key(self):
- return (self.name, self.enabled)
-
+ return (self.name, self.value)
def copy(self):
- return VariantSpec(self.name, self.enabled)
-
+ return VariantSpec(self.name, self.value)
def __str__(self):
- out = '+' if self.enabled else '~'
- return out + self.name
+ if self.value in [True, False]:
+ out = '+' if self.value else '~'
+ return out + self.name
+ else:
+ return ' ' + self.name + "=" + self.value
class VariantMap(HashableMap):
+
def __init__(self, spec):
super(VariantMap, self).__init__()
self.spec = spec
-
def satisfies(self, other, strict=False):
if strict or self.spec._concrete:
- return all(k in self and self[k].enabled == other[k].enabled
+ return all(k in self and self[k].value == other[k].value
for k in other)
else:
- return all(self[k].enabled == other[k].enabled
+ return all(self[k].value == other[k].value
for k in other if k in self)
-
def constrain(self, other):
"""Add all variants in other that aren't in self to self.
@@ -343,11 +344,11 @@ class VariantMap(HashableMap):
changed = False
for k in other:
if k in self:
- if self[k].enabled != other[k].enabled:
+ if self[k].value != other[k].value:
raise UnsatisfiableVariantSpecError(self[k], other[k])
else:
self[k] = other[k].copy()
- changed =True
+ changed = True
return changed
@property
@@ -355,27 +356,92 @@ class VariantMap(HashableMap):
return self.spec._concrete or all(
v in self for v in self.spec.package_class.variants)
-
def copy(self):
clone = VariantMap(None)
for name, variant in self.items():
clone[name] = variant.copy()
return clone
-
def __str__(self):
sorted_keys = sorted(self.keys())
return ''.join(str(self[key]) for key in sorted_keys)
+_valid_compiler_flags = [
+ 'cflags', 'cxxflags', 'fflags', 'ldflags', 'ldlibs', 'cppflags']
+
+
+class FlagMap(HashableMap):
+
+ def __init__(self, spec):
+ super(FlagMap, self).__init__()
+ self.spec = spec
+
+ def satisfies(self, other, strict=False):
+ if strict or (self.spec and self.spec._concrete):
+ return all(f in self and set(self[f]) <= set(other[f])
+ for f in other)
+ else:
+ return all(set(self[f]) <= set(other[f])
+ for f in other if (other[f] != [] and f in self))
+
+ def constrain(self, other):
+ """Add all flags in other that aren't in self to self.
+
+ Return whether the spec changed.
+ """
+ if other.spec and other.spec._concrete:
+ for k in self:
+ if k not in other:
+ raise UnsatisfiableCompilerFlagSpecError(
+ self[k], '<absent>')
+
+ changed = False
+ for k in other:
+ if k in self and not set(self[k]) <= set(other[k]):
+ raise UnsatisfiableCompilerFlagSpecError(
+ ' '.join(f for f in self[k]),
+ ' '.join(f for f in other[k]))
+ elif k not in self:
+ self[k] = other[k]
+ changed = True
+ return changed
+
+ @staticmethod
+ def valid_compiler_flags():
+ return _valid_compiler_flags
+
+ @property
+ def concrete(self):
+ return all(flag in self for flag in _valid_compiler_flags)
+
+ def copy(self):
+ clone = FlagMap(None)
+ for name, value in self.items():
+ clone[name] = value
+ return clone
+
+ def _cmp_key(self):
+ return ''.join(str(key) + ' '.join(str(v) for v in value)
+ for key, value in sorted(self.items()))
+
+ def __str__(self):
+ sorted_keys = filter(
+ lambda flag: self[flag] != [], sorted(self.keys()))
+ cond_symbol = ' ' if len(sorted_keys) > 0 else ''
+ return cond_symbol + ' '.join(str(key) + '=\"' + ' '.join(str(f)
+ for f in self[key]) + '\"'
+ for key in sorted_keys)
+
+
class DependencyMap(HashableMap):
+
"""Each spec has a DependencyMap containing specs for its dependencies.
The DependencyMap is keyed by name. """
@property
def concrete(self):
return all(d.concrete for d in self.values())
-
def __str__(self):
return ''.join(
["^" + str(self[name]) for name in sorted(self.keys())])
@@ -383,6 +449,7 @@ class DependencyMap(HashableMap):
@key_ordering
class Spec(object):
+
def __init__(self, spec_like, *dep_like, **kwargs):
# Copy if spec_like is a Spec.
if isinstance(spec_like, Spec):
@@ -409,20 +476,24 @@ class Spec(object):
self.versions = other.versions
self.architecture = other.architecture
self.compiler = other.compiler
+ self.compiler_flags = other.compiler_flags
+ self.compiler_flags.spec = self
self.dependencies = other.dependencies
self.variants = other.variants
self.variants.spec = self
self.namespace = other.namespace
+ self._hash = other._hash
# Specs are by default not assumed to be normal, but in some
# cases we've read them from a file want to assume normal.
# This allows us to manipulate specs that Spack doesn't have
# package.py files for.
- self._normal = kwargs.get('normal', False)
+ self._normal = kwargs.get('normal', False)
self._concrete = kwargs.get('concrete', False)
# Allow a spec to be constructed with an external path.
self.external = kwargs.get('external', None)
+ self.external_module = kwargs.get('external_module', None)
# This allows users to construct a spec DAG with literals.
# Note that given two specs a and b, Spec(a) copies a, but
@@ -431,7 +502,6 @@ class Spec(object):
spec = dep if isinstance(dep, Spec) else Spec(dep)
self._add_dependency(spec)
-
#
# Private routines here are called by the parser when building a spec.
#
@@ -439,32 +509,111 @@ class Spec(object):
"""Called by the parser to add an allowable version."""
self.versions.add(version)
-
- def _add_variant(self, name, enabled):
+ def _add_variant(self, name, value):
"""Called by the parser to add a variant."""
- if name in self.variants: raise DuplicateVariantError(
+ if name in self.variants:
+ raise DuplicateVariantError(
"Cannot specify variant '%s' twice" % name)
- self.variants[name] = VariantSpec(name, enabled)
-
+ if isinstance(value, basestring) and value.upper() == 'TRUE':
+ value = True
+ elif isinstance(value, basestring) and value.upper() == 'FALSE':
+ value = False
+ self.variants[name] = VariantSpec(name, value)
+
+ def _add_flag(self, name, value):
+ """Called by the parser to add a known flag.
+ Known flags currently include "arch"
+ """
+ valid_flags = FlagMap.valid_compiler_flags()
+ if name == 'arch' or name == 'architecture':
+ parts = value.split('-')
+ if len(parts) == 3:
+ platform, op_sys, target = parts
+ else:
+ platform, op_sys, target = None, None, value
+
+ assert(self.architecture.platform is None)
+ assert(self.architecture.platform_os is None)
+ assert(self.architecture.target is None)
+ assert(self.architecture.os_string is None)
+ assert(self.architecture.target_string is None)
+ self._set_platform(platform)
+ self._set_os(op_sys)
+ self._set_target(target)
+ elif name == 'platform':
+ self._set_platform(value)
+ elif name == 'os' or name == 'operating_system':
+ if self.architecture.platform:
+ self._set_os(value)
+ else:
+ self.architecture.os_string = value
+ elif name == 'target':
+ if self.architecture.platform:
+ self._set_target(value)
+ else:
+ self.architecture.target_string = value
+ elif name in valid_flags:
+ assert(self.compiler_flags is not None)
+ self.compiler_flags[name] = value.split()
+ else:
+ self._add_variant(name, value)
def _set_compiler(self, compiler):
"""Called by the parser to set the compiler."""
- if self.compiler: raise DuplicateCompilerSpecError(
+ if self.compiler:
+ raise DuplicateCompilerSpecError(
"Spec for '%s' cannot have two compilers." % self.name)
self.compiler = compiler
+ def _set_platform(self, value):
+ """Called by the parser to set the architecture platform"""
+ if isinstance(value, basestring):
+ mod_path = spack.platform_path
+ mod_string = 'spack.platformss'
+ names = list_modules(mod_path)
+ if value in names:
+ # Create a platform object from the name
+ mod_name = mod_string + value
+ path = join_path(mod_path, value) + '.py'
+ mod = imp.load_source(mod_name, path)
+ class_name = mod_to_class(value)
+ if not hasattr(mod, class_name):
+ tty.die('No class %s defined in %s' % (class_name, mod_name))
+ cls = getattr(mod, class_name)
+ if not inspect.isclass(cls):
+ tty.die('%s.%s is not a class' % (mod_name, class_name))
+ platform = cls()
+ else:
+ tty.die("No platform class %s defined." % value)
+ else:
+ # The value is a platform
+ platform = value
+
+ self.architecture.platform = platform
+
+ # Set os and target if we previously got strings for them
+ if self.architecture.os_string:
+ self._set_os(self.architecture.os_string)
+ self.architecture.os_string = None
+ if self.architecture.target_string:
+ self._set_target(self.architecture.target_string)
+ self.architecture.target_string = None
- def _set_architecture(self, architecture):
- """Called by the parser to set the architecture."""
- if self.architecture: raise DuplicateArchitectureError(
- "Spec for '%s' cannot have two architectures." % self.name)
- self.architecture = architecture
+ def _set_os(self, value):
+ """Called by the parser to set the architecture operating system"""
+ if self.architecture.platform:
+ self.architecture.platform_os = self.architecture.platform.operating_system(value)
+ def _set_target(self, value):
+ """Called by the parser to set the architecture target"""
+ if self.architecture.platform:
+ self.architecture.target = self.architecture.platform.target(value)
def _add_dependency(self, spec):
"""Called by the parser to add another spec as a dependency."""
if spec.name in self.dependencies:
- raise DuplicateDependencyError("Cannot depend on '%s' twice" % spec)
+ raise DuplicateDependencyError(
+ "Cannot depend on '%s' twice" % spec)
self.dependencies[spec.name] = spec
spec.dependents[self.name] = self
@@ -473,8 +622,8 @@ class Spec(object):
#
@property
def fullname(self):
- return '%s.%s' % (self.namespace, self.name) if self.namespace else self.name
-
+ return (('%s.%s' % (self.namespace, self.name)) if self.namespace else
+ (self.name if self.name else ''))
@property
def root(self):
@@ -494,12 +643,10 @@ class Spec(object):
assert(all(first_root is d.root for d in depiter))
return first_root
-
@property
def package(self):
return spack.repo.get(self)
-
@property
def package_class(self):
"""Internal package call gets only the class object for a package.
@@ -507,7 +654,6 @@ class Spec(object):
"""
return spack.repo.get_pkg_class(self.name)
-
@property
def virtual(self):
"""Right now, a spec is virtual if no package exists with its name.
@@ -519,12 +665,10 @@ class Spec(object):
"""
return Spec.is_virtual(self.name)
-
@staticmethod
def is_virtual(name):
"""Test if a name is virtual without requiring a Spec."""
- return not spack.repo.exists(name)
-
+ return (name is not None) and (not spack.repo.exists(name))
@property
def concrete(self):
@@ -540,12 +684,12 @@ class Spec(object):
and self.versions.concrete
and self.variants.concrete
and self.architecture
+ and self.architecture.concrete
and self.compiler and self.compiler.concrete
+ and self.compiler_flags.concrete
and self.dependencies.concrete)
-
return self._concrete
-
def traverse(self, visited=None, d=0, **kwargs):
"""Generic traversal of the DAG represented by this spec.
This will yield each node in the spec. Options:
@@ -581,7 +725,7 @@ class Spec(object):
in the traversal.
root [=True]
- If false, this won't yield the root node, just its descendents.
+ If False, this won't yield the root node, just its descendents.
direction [=children|parents]
If 'children', does a traversal of this spec's children. If
@@ -589,14 +733,14 @@ class Spec(object):
"""
# get initial values for kwargs
- depth = kwargs.get('depth', False)
- key_fun = kwargs.get('key', id)
+ depth = kwargs.get('depth', False)
+ key_fun = kwargs.get('key', id)
if isinstance(key_fun, basestring):
key_fun = attrgetter(key_fun)
yield_root = kwargs.get('root', True)
- cover = kwargs.get('cover', 'nodes')
- direction = kwargs.get('direction', 'children')
- order = kwargs.get('order', 'pre')
+ cover = kwargs.get('cover', 'nodes')
+ direction = kwargs.get('direction', 'children')
+ order = kwargs.get('order', 'pre')
# Make sure kwargs have legal values; raise ValueError if not.
def validate(name, val, allowed_values):
@@ -633,47 +777,50 @@ class Spec(object):
visited.add(key)
for name in sorted(successors):
child = successors[name]
- for elt in child.traverse(visited, d+1, **kwargs):
+ for elt in child.traverse(visited, d + 1, **kwargs):
yield elt
# Postorder traversal yields after successors
if yield_me and order == 'post':
yield result
-
@property
def short_spec(self):
"""Returns a version of the spec with the dependencies hashed
instead of completely enumerated."""
return self.format('$_$@$%@$+$=$#')
-
@property
def cshort_spec(self):
"""Returns a version of the spec with the dependencies hashed
instead of completely enumerated."""
return self.format('$_$@$%@$+$=$#', color=True)
-
@property
def prefix(self):
return Prefix(spack.install_layout.path_for_spec(self))
-
def dag_hash(self, length=None):
"""
Return a hash of the entire spec DAG, including connectivity.
"""
- yaml_text = yaml.dump(
- self.to_node_dict(), default_flow_style=True, width=sys.maxint)
- sha = hashlib.sha1(yaml_text)
- return base64.b32encode(sha.digest()).lower()[:length]
-
+ if self._hash:
+ return self._hash[:length]
+ else:
+ yaml_text = yaml.dump(
+ self.to_node_dict(), default_flow_style=True, width=sys.maxint)
+ sha = hashlib.sha1(yaml_text)
+ b32_hash = base64.b32encode(sha.digest()).lower()[:length]
+ if self.concrete:
+ self._hash = b32_hash
+ return b32_hash
def to_node_dict(self):
+ params = dict((name, v.value) for name, v in self.variants.items())
+ params.update(dict((name, value)
+ for name, value in self.compiler_flags.items()))
d = {
- 'variants' : dict(
- (name,v.enabled) for name, v in self.variants.items()),
+ 'parameters' : params,
'arch' : self.architecture,
'dependencies' : dict((d, self.dependencies[d].dag_hash())
for d in sorted(self.dependencies))
@@ -684,13 +831,20 @@ class Spec(object):
if not self.concrete or self.namespace:
d['namespace'] = self.namespace
+ if self.architecture:
+ # TODO: Fix the target.to_dict to account for the tuple
+ # Want it to be a dict of dicts
+ d['arch'] = self.architecture.to_dict()
+ else:
+ d['arch'] = None
+
if self.compiler:
d.update(self.compiler.to_dict())
else:
d['compiler'] = None
d.update(self.versions.to_dict())
- return { self.name : d }
+ return {self.name: d}
def to_yaml(self, stream=None):
node_list = []
@@ -698,10 +852,9 @@ class Spec(object):
node = s.to_node_dict()
node[s.name]['hash'] = s.dag_hash()
node_list.append(node)
- return yaml.dump({ 'spec' : node_list },
+ return yaml.dump({'spec': node_list},
stream=stream, default_flow_style=False)
-
@staticmethod
def from_node_dict(node):
name = next(iter(node))
@@ -710,19 +863,34 @@ class Spec(object):
spec = Spec(name)
spec.namespace = node.get('namespace', None)
spec.versions = VersionList.from_dict(node)
- spec.architecture = node['arch']
+
+ if 'hash' in node:
+ spec._hash = node['hash']
+
+ spec.architecture = spack.architecture.arch_from_dict(node['arch'])
if node['compiler'] is None:
spec.compiler = None
else:
spec.compiler = CompilerSpec.from_dict(node)
- for name, enabled in node['variants'].items():
- spec.variants[name] = VariantSpec(name, enabled)
+ if 'parameters' in node:
+ for name, value in node['parameters'].items():
+ if name in _valid_compiler_flags:
+ spec.compiler_flags[name] = value
+ else:
+ spec.variants[name] = VariantSpec(name, value)
+ elif 'variants' in node:
+ for name, value in node['variants'].items():
+ spec.variants[name] = VariantSpec(name, value)
+ for name in FlagMap.valid_compiler_flags():
+ spec.compiler_flags[name] = []
+ else:
+ raise SpackRecordError(
+ "Did not find a valid format for variants in YAML file")
return spec
-
@staticmethod
def from_yaml(stream):
"""Construct a spec from YAML.
@@ -755,15 +923,16 @@ class Spec(object):
deps[name].dependencies[dep_name] = deps[dep_name]
return spec
-
def _concretize_helper(self, presets=None, visited=None):
"""Recursive helper function for concretize().
This concretizes everything bottom-up. As things are
concretized, they're added to the presets, and ancestors
will prefer the settings of their children.
"""
- if presets is None: presets = {}
- if visited is None: visited = set()
+ if presets is None:
+ presets = {}
+ if visited is None:
+ visited = set()
if self.name in visited:
return False
@@ -776,22 +945,23 @@ class Spec(object):
if self.name in presets:
changed |= self.constrain(presets[self.name])
-
else:
# Concretize virtual dependencies last. Because they're added
# to presets below, their constraints will all be merged, but we'll
# still need to select a concrete package later.
- changed |= any(
- (spack.concretizer.concretize_architecture(self),
- spack.concretizer.concretize_compiler(self),
- spack.concretizer.concretize_version(self),
- spack.concretizer.concretize_variants(self)))
+ if not self.virtual:
+ changed |= any(
+ (spack.concretizer.concretize_architecture(self),
+ spack.concretizer.concretize_compiler(self),
+ spack.concretizer.concretize_compiler_flags(
+ self), # has to be concretized after compiler
+ spack.concretizer.concretize_version(self),
+ spack.concretizer.concretize_variants(self)))
presets[self.name] = self
visited.add(self.name)
return changed
-
def _replace_with(self, concrete):
"""Replace this virtual spec with a concrete spec."""
assert(self.virtual)
@@ -803,7 +973,6 @@ class Spec(object):
if concrete.name not in dependent.dependencies:
dependent._add_dependency(concrete)
-
def _replace_node(self, replacement):
"""Replace this spec with another.
@@ -821,7 +990,6 @@ class Spec(object):
del dep.dependents[self.name]
del self.dependencies[dep.name]
-
def _expand_virtual_packages(self):
"""Find virtual packages in this spec, replace them with providers,
and normalize again to include the provider's (potentially virtual)
@@ -841,11 +1009,12 @@ class Spec(object):
"""
# Make an index of stuff this spec already provides
self_index = ProviderIndex(self.traverse(), restrict=True)
-
changed = False
done = False
+
while not done:
done = True
+
for spec in list(self.traverse()):
replacement = None
if spec.virtual:
@@ -854,12 +1023,14 @@ class Spec(object):
# TODO: may break if in-place on self but
# shouldn't happen if root is traversed first.
spec._replace_with(replacement)
- done=False
+ done = False
break
if not replacement:
- # Get a list of possible replacements in order of preference.
- candidates = spack.concretizer.choose_virtual_or_external(spec)
+ # Get a list of possible replacements in order of
+ # preference.
+ candidates = spack.concretizer.choose_virtual_or_external(
+ spec)
# Try the replacements in order, skipping any that cause
# satisfiability problems.
@@ -872,20 +1043,22 @@ class Spec(object):
copy[spec.name]._dup(replacement.copy(deps=False))
try:
- # If there are duplicate providers or duplicate provider
- # deps, consolidate them and merge constraints.
+ # If there are duplicate providers or duplicate
+ # provider deps, consolidate them and merge
+ # constraints.
copy.normalize(force=True)
break
- except SpecError as e:
+ except SpecError:
# On error, we'll try the next replacement.
continue
# If replacement is external then trim the dependencies
- if replacement.external:
+ if replacement.external or replacement.external_module:
if (spec.dependencies):
changed = True
spec.dependencies = DependencyMap()
replacement.dependencies = DependencyMap()
+ replacement.architecture = self.architecture
# TODO: could this and the stuff in _dup be cleaned up?
def feq(cfield, sfield):
@@ -897,9 +1070,9 @@ class Spec(object):
feq(replacement.architecture, spec.architecture) and
feq(replacement.dependencies, spec.dependencies) and
feq(replacement.variants, spec.variants) and
- feq(replacement.external, spec.external)):
+ feq(replacement.external, spec.external) and
+ feq(replacement.external_module, spec.external_module)):
continue
-
# Refine this spec to the candidate. This uses
# replace_with AND dup so that it can work in
# place. TODO: make this more efficient.
@@ -910,12 +1083,11 @@ class Spec(object):
changed = True
self_index.update(spec)
- done=False
+ done = False
break
return changed
-
def concretize(self):
"""A spec is concrete if it describes one build of a package uniquely.
This will ensure that this spec is concrete.
@@ -924,10 +1096,12 @@ class Spec(object):
of a package, this will add constraints to make it concrete.
Some rigorous validation and checks are also performed on the spec.
- Concretizing ensures that it is self-consistent and that it's consistent
- with requirements of its pacakges. See flatten() and normalize() for
- more details on this.
+ Concretizing ensures that it is self-consistent and that it's
+ consistent with requirements of its pacakges. See flatten() and
+ normalize() for more details on this.
"""
+ if not self.name:
+ raise SpecError("Attempting to concretize anonymous spec")
if self._concrete:
return
@@ -940,7 +1114,7 @@ class Spec(object):
self._expand_virtual_packages(),
self._concretize_helper())
changed = any(changes)
- force=True
+ force = True
for s in self.traverse():
# After concretizing, assign namespaces to anything left.
@@ -954,10 +1128,18 @@ class Spec(object):
if s.namespace is None:
s.namespace = spack.repo.repo_for_pkg(s.name).namespace
+
+ for s in self.traverse(root=False):
+ if s.external_module:
+ compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture)
+ for mod in compiler.modules:
+ load_module(mod)
+
+ s.external = get_path_from_module(s.external_module)
+
# Mark everything in the spec as concrete, as well.
self._mark_concrete()
-
def _mark_concrete(self):
"""Mark this spec and its dependencies as concrete.
@@ -968,7 +1150,6 @@ class Spec(object):
s._normal = True
s._concrete = True
-
def concretized(self):
"""This is a non-destructive version of concretize(). First clones,
then returns a concrete version of this package without modifying
@@ -977,7 +1158,6 @@ class Spec(object):
clone.concretize()
return clone
-
def flat_dependencies(self, **kwargs):
"""Return a DependencyMap containing all of this spec's
dependencies with their constraints merged.
@@ -1016,7 +1196,6 @@ class Spec(object):
# parser doesn't allow it. Spack must be broken!
raise InconsistentSpecError("Invalid Spec DAG: %s" % e.message)
-
def index(self):
"""Return DependencyMap that points to all the dependencies in this
spec."""
@@ -1025,7 +1204,6 @@ class Spec(object):
dm[spec.name] = spec
return dm
-
def flatten(self):
"""Pull all dependencies up to the root (this spec).
Merge constraints for dependencies with the same name, and if they
@@ -1033,7 +1211,6 @@ class Spec(object):
for dep in self.flat_dependencies(copy=False):
self._add_dependency(dep)
-
def _evaluate_dependency_conditions(self, name):
"""Evaluate all the conditions on a dependency with this name.
@@ -1054,12 +1231,11 @@ class Spec(object):
try:
dep.constrain(dep_spec)
except UnsatisfiableSpecError, e:
- e.message = ("Conflicting conditional dependencies on package "
- "%s for spec %s" % (self.name, self))
+ e.message = ("Conflicting conditional dependencies on"
+ "package %s for spec %s" % (self.name, self))
raise e
return dep
-
def _find_provider(self, vdep, provider_index):
"""Find provider for a virtual spec in the provider index.
Raise an exception if there is a conflicting virtual
@@ -1071,6 +1247,12 @@ class Spec(object):
# If there is a provider for the vpkg, then use that instead of
# the virtual package.
if providers:
+ # Remove duplicate providers that can concretize to the same
+ # result.
+ for provider in providers:
+ for spec in providers:
+ if spec is not provider and provider.satisfies(spec):
+ providers.remove(spec)
# Can't have multiple providers for the same thing in one spec.
if len(providers) > 1:
raise MultipleProviderError(vdep, providers)
@@ -1085,11 +1267,10 @@ class Spec(object):
elif required:
raise UnsatisfiableProviderSpecError(required[0], vdep)
-
def _merge_dependency(self, dep, visited, spec_deps, provider_index):
"""Merge the dependency into this spec.
- This is the core of the normalize() method. There are a few basic steps:
+ This is the core of normalize(). There are some basic steps:
* If dep is virtual, evaluate whether it corresponds to an
existing concrete dependency, and merge if so.
@@ -1123,19 +1304,17 @@ class Spec(object):
if required:
raise UnsatisfiableProviderSpecError(required[0], dep)
provider_index.update(dep)
-
# If the spec isn't already in the set of dependencies, clone
# it from the package description.
if dep.name not in spec_deps:
spec_deps[dep.name] = dep.copy()
changed = True
-
# Constrain package information with spec info
try:
changed |= spec_deps[dep.name].constrain(dep)
except UnsatisfiableSpecError, e:
- e.message = "Invalid spec: '%s'. "
+ e.message = "Invalid spec: '%s'. "
e.message += "Package %s requires %s %s, but spec asked for %s"
e.message %= (spec_deps[dep.name], dep.name, e.constraint_type,
e.required, e.provided)
@@ -1146,10 +1325,10 @@ class Spec(object):
if dep.name not in self.dependencies:
self._add_dependency(dependency)
- changed |= dependency._normalize_helper(visited, spec_deps, provider_index)
+ changed |= dependency._normalize_helper(
+ visited, spec_deps, provider_index)
return changed
-
def _normalize_helper(self, visited, spec_deps, provider_index):
"""Recursive helper function for _normalize."""
if self.name in visited:
@@ -1158,7 +1337,7 @@ class Spec(object):
# if we descend into a virtual spec, there's nothing more
# to normalize. Concretize will finish resolving it later.
- if self.virtual or self.external:
+ if self.virtual or self.external or self.external_module:
return False
# Combine constraints from package deps with constraints from
@@ -1172,7 +1351,6 @@ class Spec(object):
for dep_name in pkg.dependencies:
# Do we depend on dep_name? If so pkg_dep is not None.
pkg_dep = self._evaluate_dependency_conditions(dep_name)
-
# If pkg_dep is a dependency, merge it.
if pkg_dep:
changed |= self._merge_dependency(
@@ -1181,30 +1359,31 @@ class Spec(object):
return any_change
-
def normalize(self, force=False):
"""When specs are parsed, any dependencies specified are hanging off
the root, and ONLY the ones that were explicitly provided are there.
Normalization turns a partial flat spec into a DAG, where:
1. Known dependencies of the root package are in the DAG.
- 2. Each node's dependencies dict only contains its known direct deps.
+ 2. Each node's dependencies dict only contains its known direct
+ deps.
3. There is only ONE unique spec for each package in the DAG.
* This includes virtual packages. If there a non-virtual
package that provides a virtual package that is in the spec,
then we replace the virtual package with the non-virtual one.
- TODO: normalize should probably implement some form of cycle detection,
- to ensure that the spec is actually a DAG.
-
+ TODO: normalize should probably implement some form of cycle
+ detection, to ensure that the spec is actually a DAG.
"""
+ if not self.name:
+ raise SpecError("Attempting to normalize anonymous spec")
+
if self._normal and not force:
return False
# Ensure first that all packages & compilers in the DAG exist.
self.validate_names()
-
# Get all the dependencies into one DependencyMap
spec_deps = self.flat_dependencies(copy=False)
@@ -1228,14 +1407,14 @@ class Spec(object):
self._normal = True
return any_change
-
def normalized(self):
- """Return a normalized copy of this spec without modifying this spec."""
+ """
+ Return a normalized copy of this spec without modifying this spec.
+ """
clone = self.copy()
clone.normalize()
return clone
-
def validate_names(self):
"""This checks that names of packages and compilers in this spec are real.
If they're not, it will raise either UnknownPackageError or
@@ -1243,7 +1422,7 @@ class Spec(object):
"""
for spec in self.traverse():
# Don't get a package for a virtual name.
- if not spec.virtual:
+ if (not spec.virtual) and spec.name:
spack.repo.get(spec.fullname)
# validate compiler in addition to the package name.
@@ -1256,7 +1435,6 @@ class Spec(object):
if vname not in spec.package_class.variants:
raise UnknownVariantError(spec.name, vname)
-
def constrain(self, other, deps=True):
"""Merge the constraints of other with self.
@@ -1264,26 +1442,40 @@ class Spec(object):
"""
other = self._autospec(other)
- if not self.name == other.name:
+ if not (self.name == other.name or
+ (not self.name) or
+ (not other.name)):
raise UnsatisfiableSpecNameError(self.name, other.name)
- if other.namespace is not None:
- if self.namespace is not None and other.namespace != self.namespace:
- raise UnsatisfiableSpecNameError(self.fullname, other.fullname)
+ if (other.namespace is not None and
+ self.namespace is not None and
+ other.namespace != self.namespace):
+ raise UnsatisfiableSpecNameError(self.fullname, other.fullname)
if not self.versions.overlaps(other.versions):
raise UnsatisfiableVersionSpecError(self.versions, other.versions)
for v in other.variants:
if (v in self.variants and
- self.variants[v].enabled != other.variants[v].enabled):
+ self.variants[v].value != other.variants[v].value):
raise UnsatisfiableVariantSpecError(self.variants[v],
other.variants[v])
+ # TODO: Check out the logic here
if self.architecture is not None and other.architecture is not None:
- if self.architecture != other.architecture:
- raise UnsatisfiableArchitectureSpecError(self.architecture,
- other.architecture)
+ if self.architecture.platform is not None and other.architecture.platform is not None:
+ if self.architecture.platform != other.architecture.platform:
+ raise UnsatisfiableArchitectureSpecError(self.architecture,
+ other.architecture)
+ if self.architecture.platform_os is not None and other.architecture.platform_os is not None:
+ if self.architecture.platform_os != other.architecture.platform_os:
+ raise UnsatisfiableArchitectureSpecError(self.architecture,
+ other.architecture)
+ if self.architecture.target is not None and other.architecture.target is not None:
+ if self.architecture.target != other.architecture.target:
+ raise UnsatisfiableArchitectureSpecError(self.architecture,
+ other.architecture)
+
changed = False
if self.compiler is not None and other.compiler is not None:
@@ -1295,16 +1487,25 @@ class Spec(object):
changed |= self.versions.intersect(other.versions)
changed |= self.variants.constrain(other.variants)
- old = self.architecture
- self.architecture = self.architecture or other.architecture
- changed |= (self.architecture != old)
+ changed |= self.compiler_flags.constrain(other.compiler_flags)
+
+ old = str(self.architecture)
+ if self.architecture is None or other.architecture is None:
+ self.architecture = self.architecture or other.architecture
+ else:
+ if self.architecture.platform is None or other.architecture.platform is None:
+ self.architecture.platform = self.architecture.platform or other.architecture.platform
+ if self.architecture.platform_os is None or other.architecture.platform_os is None:
+ self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os
+ if self.architecture.target is None or other.architecture.target is None:
+ self.architecture.target = self.architecture.target or other.architecture.target
+ changed |= (str(self.architecture) != old)
if deps:
changed |= self._constrain_dependencies(other)
return changed
-
def _constrain_dependencies(self, other):
"""Apply constraints of other spec's dependencies to this spec."""
other = self._autospec(other)
@@ -1323,7 +1524,6 @@ class Spec(object):
for name in self.common_dependencies(other):
changed |= self[name].constrain(other[name], deps=False)
-
# Update with additional constraints from other spec
for name in other.dep_difference(self):
self._add_dependency(other[name].copy())
@@ -1331,7 +1531,6 @@ class Spec(object):
return changed
-
def common_dependencies(self, other):
"""Return names of dependencies that self an other have in common."""
common = set(
@@ -1340,14 +1539,12 @@ class Spec(object):
s.name for s in other.traverse(root=False))
return common
-
def constrained(self, other, deps=True):
"""Return a constrained copy without modifying this spec."""
clone = self.copy(deps=deps)
clone.constrain(other, deps)
return clone
-
def dep_difference(self, other):
"""Returns dependencies in self that are not in other."""
mine = set(s.name for s in self.traverse(root=False))
@@ -1355,21 +1552,24 @@ class Spec(object):
s.name for s in other.traverse(root=False))
return mine
-
def _autospec(self, spec_like):
- """Used to convert arguments to specs. If spec_like is a spec, returns it.
- If it's a string, tries to parse a string. If that fails, tries to parse
- a local spec from it (i.e. name is assumed to be self's name).
+ """
+ Used to convert arguments to specs. If spec_like is a spec, returns
+ it. If it's a string, tries to parse a string. If that fails, tries
+ to parse a local spec from it (i.e. name is assumed to be self's name).
"""
if isinstance(spec_like, spack.spec.Spec):
return spec_like
try:
- return spack.spec.Spec(spec_like)
+ spec = spack.spec.Spec(spec_like)
+ if not spec.name:
+ raise SpecError(
+ "anonymous package -- this will always be handled")
+ return spec
except SpecError:
return parse_anonymous_spec(spec_like, self.name)
-
def satisfies(self, other, deps=True, strict=False):
"""Determine if this spec satisfies all constraints of another.
@@ -1396,14 +1596,14 @@ class Spec(object):
return False
# Otherwise, first thing we care about is whether the name matches
- if self.name != other.name:
+ if self.name != other.name and self.name and other.name:
return False
# namespaces either match, or other doesn't require one.
- if other.namespace is not None:
- if self.namespace is not None and self.namespace != other.namespace:
- return False
-
+ if (other.namespace is not None and
+ self.namespace is not None and
+ self.namespace != other.namespace):
+ return False
if self.versions and other.versions:
if not self.versions.satisfies(other.versions, strict=strict):
return False
@@ -1417,26 +1617,43 @@ class Spec(object):
elif strict and (other.compiler and not self.compiler):
return False
- if not self.variants.satisfies(other.variants, strict=strict):
+ var_strict = strict
+ if (not self.name) or (not other.name):
+ var_strict = True
+ if not self.variants.satisfies(other.variants, strict=var_strict):
return False
# Architecture satisfaction is currently just string equality.
# If not strict, None means unconstrained.
if self.architecture and other.architecture:
- if self.architecture != other.architecture:
+ if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or
+ (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or
+ (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)):
return False
- elif strict and (other.architecture and not self.architecture):
+ elif strict and ((other.architecture and not self.architecture) or
+ (other.architecture.platform and not self.architecture.platform) or
+ (other.architecture.platform_os and not self.architecture.platform_os) or
+ (other.architecture.target and not self.architecture.target)):
+ return False
+
+ if not self.compiler_flags.satisfies(
+ other.compiler_flags,
+ strict=strict):
return False
# If we need to descend into dependencies, do it, otherwise we're done.
if deps:
- return self.satisfies_dependencies(other, strict=strict)
+ deps_strict = strict
+ if not (self.name and other.name):
+ deps_strict = True
+ return self.satisfies_dependencies(other, strict=deps_strict)
else:
return True
-
def satisfies_dependencies(self, other, strict=False):
- """This checks constraints on common dependencies against each other."""
+ """
+ This checks constraints on common dependencies against each other.
+ """
other = self._autospec(other)
if strict:
@@ -1447,7 +1664,8 @@ class Spec(object):
return False
elif not self.dependencies or not other.dependencies:
- # if either spec doesn't restrict dependencies then both are compatible.
+ # if either spec doesn't restrict dependencies then both are
+ # compatible.
return True
# Handle first-order constraints directly
@@ -1463,11 +1681,12 @@ class Spec(object):
if not self_index.satisfies(other_index):
return False
- # These two loops handle cases where there is an overly restrictive vpkg
- # in one spec for a provider in the other (e.g., mpi@3: is not compatible
- # with mpich2)
+ # These two loops handle cases where there is an overly restrictive
+ # vpkg in one spec for a provider in the other (e.g., mpi@3: is not
+ # compatible with mpich2)
for spec in self.virtual_dependencies():
- if spec.name in other_index and not other_index.providers_for(spec):
+ if (spec.name in other_index and
+ not other_index.providers_for(spec)):
return False
for spec in other.virtual_dependencies():
@@ -1476,12 +1695,10 @@ class Spec(object):
return True
-
def virtual_dependencies(self):
"""Return list of any virtual deps in this spec."""
return [spec for spec in self.traverse() if spec.virtual]
-
def _dup(self, other, **kwargs):
"""Copy the spec other into self. This is an overwriting
copy. It does not copy any dependents (parents), but by default
@@ -1491,16 +1708,17 @@ class Spec(object):
Options:
dependencies[=True]
- Whether deps should be copied too. Set to false to copy a
+ Whether deps should be copied too. Set to False to copy a
spec but not its dependencies.
"""
# We don't count dependencies as changes here
changed = True
if hasattr(self, 'name'):
- changed = (self.name != other.name and self.versions != other.versions and
- self.architecture != other.architecture and self.compiler != other.compiler and
- self.variants != other.variants and self._normal != other._normal and
- self.concrete != other.concrete and self.external != other.external)
+ changed = (self.name != other.name and self.versions != other.versions and \
+ self.architecture != other.architecture and self.compiler != other.compiler and \
+ self.variants != other.variants and self._normal != other._normal and \
+ self.concrete != other.concrete and self.external != other.external and \
+ self.external_module != other.external_module and self.compiler_flags != other.compiler_flags)
# Local node attributes get copied first.
self.name = other.name
@@ -1510,10 +1728,13 @@ class Spec(object):
if kwargs.get('cleardeps', True):
self.dependents = DependencyMap()
self.dependencies = DependencyMap()
+ self.compiler_flags = other.compiler_flags.copy()
self.variants = other.variants.copy()
self.variants.spec = self
self.external = other.external
+ self.external_module = other.external_module
self.namespace = other.namespace
+ self._hash = other._hash
# If we copy dependencies, preserve DAG structure in the new spec
if kwargs.get('deps', True):
@@ -1532,9 +1753,9 @@ class Spec(object):
self._normal = other._normal
self._concrete = other._concrete
self.external = other.external
+ self.external_module = other.external_module
return changed
-
def copy(self, **kwargs):
"""Return a copy of this spec.
By default, returns a deep copy. Supply dependencies=False
@@ -1544,14 +1765,12 @@ class Spec(object):
clone._dup(self, **kwargs)
return clone
-
@property
def version(self):
if not self.versions.concrete:
raise SpecError("Spec version is not concrete: " + str(self))
return self.versions[0]
-
def __getitem__(self, name):
"""Get a dependency from the spec by its name."""
for spec in self.traverse():
@@ -1570,7 +1789,6 @@ class Spec(object):
raise KeyError("No spec with name %s in %s" % (name, self))
-
def __contains__(self, spec):
"""True if this spec satisfis the provided spec, or if any dependency
does. If the spec has no name, then we parse this one first.
@@ -1582,13 +1800,11 @@ class Spec(object):
return False
-
def sorted_deps(self):
"""Return a list of all dependencies sorted by name."""
deps = self.flat_dependencies()
return tuple(deps[name] for name in sorted(deps))
-
def _eq_dag(self, other, vs, vo):
"""Recursive helper for eq_dag and ne_dag. Does the actual DAG
traversal."""
@@ -1601,18 +1817,22 @@ class Spec(object):
if len(self.dependencies) != len(other.dependencies):
return False
- ssorted = [self.dependencies[name] for name in sorted(self.dependencies)]
- osorted = [other.dependencies[name] for name in sorted(other.dependencies)]
+ ssorted = [self.dependencies[name]
+ for name in sorted(self.dependencies)]
+ osorted = [other.dependencies[name]
+ for name in sorted(other.dependencies)]
for s, o in zip(ssorted, osorted):
visited_s = id(s) in vs
visited_o = id(o) in vo
# Check for duplicate or non-equal dependencies
- if visited_s != visited_o: return False
+ if visited_s != visited_o:
+ return False
# Skip visited nodes
- if visited_s or visited_o: continue
+ if visited_s or visited_o:
+ continue
# Recursive check for equality
if not s._eq_dag(o, vs, vo):
@@ -1620,17 +1840,14 @@ class Spec(object):
return True
-
def eq_dag(self, other):
"""True if the full dependency DAGs of specs are equal"""
return self._eq_dag(other, set(), set())
-
def ne_dag(self, other):
"""True if the full dependency DAGs of specs are not equal"""
return not self.eq_dag(other)
-
def _cmp_node(self):
"""Comparison key for just *this node* and not its deps."""
return (self.name,
@@ -1638,19 +1855,18 @@ class Spec(object):
self.versions,
self.variants,
self.architecture,
- self.compiler)
+ self.compiler,
+ self.compiler_flags)
def eq_node(self, other):
"""Equality with another spec, not including dependencies."""
return self._cmp_node() == other._cmp_node()
-
def ne_node(self, other):
"""Inequality with another spec, not including dependencies."""
return self._cmp_node() != other._cmp_node()
-
def _cmp_key(self):
"""This returns a key for the spec *including* DAG structure.
@@ -1662,52 +1878,56 @@ class Spec(object):
tuple(hash(self.dependencies[name])
for name in sorted(self.dependencies)),)
-
def colorized(self):
return colorize_spec(self)
-
- def format(self, format_string='$_$@$%@$+$=', **kwargs):
- """Prints out particular pieces of a spec, depending on what is
- in the format string. The format strings you can provide are::
-
- $_ Package name
- $. Full package name (with namespace)
- $@ Version with '@' prefix
- $% Compiler with '%' prefix
- $%@ Compiler with '%' prefix & compiler version with '@' prefix
- $+ Options
- $= Architecture with '=' prefix
- $# 7-char prefix of DAG hash with '-' prefix
- $$ $
-
- You can also use full-string versions, which leave off the prefixes:
-
- ${PACKAGE} Package name
- ${VERSION} Version
- ${COMPILER} Full compiler string
- ${COMPILERNAME} Compiler name
- ${COMPILERVER} Compiler version
- ${OPTIONS} Options
- ${ARCHITECTURE} Architecture
- ${SHA1} Dependencies 8-char sha1 prefix
-
- ${SPACK_ROOT} The spack root directory
- ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt
-
- Optionally you can provide a width, e.g. $20_ for a 20-wide name.
- Like printf, you can provide '-' for left justification, e.g.
- $-20_ for a left-justified name.
-
- Anything else is copied verbatim into the output stream.
-
- *Example:* ``$_$@$+`` translates to the name, version, and options
- of the package, but no dependencies, arch, or compiler.
-
- TODO: allow, e.g., $6# to customize short hash length
- TODO: allow, e.g., $## for full hash.
- """
- color = kwargs.get('color', False)
+ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
+ """
+ Prints out particular pieces of a spec, depending on what is
+ in the format string. The format strings you can provide are::
+
+ $_ Package name
+ $. Full package name (with namespace)
+ $@ Version with '@' prefix
+ $% Compiler with '%' prefix
+ $%@ Compiler with '%' prefix & compiler version with '@' prefix
+ $%+ Compiler with '%' prefix & compiler flags prefixed by name
+ $%@+ Compiler, compiler version, and compiler flags with same
+ prefixes as above
+ $+ Options
+ $= Architecture prefixed by 'arch='
+ $# 7-char prefix of DAG hash with '-' prefix
+ $$ $
+
+ You can also use full-string versions, which elide the prefixes:
+
+ ${PACKAGE} Package name
+ ${VERSION} Version
+ ${COMPILER} Full compiler string
+ ${COMPILERNAME} Compiler name
+ ${COMPILERVER} Compiler version
+ ${COMPILERFLAGS} Compiler flags
+ ${OPTIONS} Options
+ ${ARCHITECTURE} Architecture
+ ${SHA1} Dependencies 8-char sha1 prefix
+
+ ${SPACK_ROOT} The spack root directory
+ ${SPACK_INSTALL} The default spack install directory,
+ ${SPACK_PREFIX}/opt
+
+ Optionally you can provide a width, e.g. $20_ for a 20-wide name.
+ Like printf, you can provide '-' for left justification, e.g.
+ $-20_ for a left-justified name.
+
+ Anything else is copied verbatim into the output stream.
+
+ *Example:* ``$_$@$+`` translates to the name, version, and options
+ of the package, but no dependencies, arch, or compiler.
+
+ TODO: allow, e.g., $6# to customize short hash length
+ TODO: allow, e.g., $## for full hash.
+ """
+ color = kwargs.get('color', False)
length = len(format_string)
out = StringIO()
named = escape = compiler = False
@@ -1734,7 +1954,8 @@ class Spec(object):
fmt += 's'
if c == '_':
- out.write(fmt % self.name)
+ name = self.name if self.name else ''
+ out.write(fmt % name)
elif c == '.':
out.write(fmt % self.fullname)
elif c == '@':
@@ -1748,8 +1969,8 @@ class Spec(object):
if self.variants:
write(fmt % str(self.variants), c)
elif c == '=':
- if self.architecture:
- write(fmt % (c + str(self.architecture)), c)
+ if self.architecture and str(self.architecture):
+ write(fmt % (' arch' + c + str(self.architecture)), c)
elif c == '#':
out.write('-' + fmt % (self.dag_hash(7)))
elif c == '$':
@@ -1764,22 +1985,28 @@ class Spec(object):
elif compiler:
if c == '@':
if (self.compiler and self.compiler.versions and
- self.compiler.versions != _any_version):
+ self.compiler.versions != _any_version):
write(c + str(self.compiler.versions), '%')
+ elif c == '+':
+ if self.compiler_flags:
+ write(fmt % str(self.compiler_flags), '%')
+ compiler = False
elif c == '$':
escape = True
+ compiler = False
else:
out.write(c)
- compiler = False
+ compiler = False
elif named:
if not c == '}':
if i == length - 1:
- raise ValueError("Error: unterminated ${ in format: '%s'"
- % format_string)
+ raise ValueError("Error: unterminated ${ in format:"
+ "'%s'" % format_string)
named_str += c
- continue;
+ continue
if named_str == 'PACKAGE':
+ name = self.name if self.name else ''
write(fmt % self.name, '@')
if named_str == 'VERSION':
if self.versions and self.versions != _any_version:
@@ -1793,12 +2020,15 @@ class Spec(object):
elif named_str == 'COMPILERVER':
if self.compiler:
write(fmt % self.compiler.versions, '%')
+ elif named_str == 'COMPILERFLAGS':
+ if self.compiler:
+ write(fmt % str(self.compiler_flags), '%')
elif named_str == 'OPTIONS':
if self.variants:
write(fmt % str(self.variants), '+')
elif named_str == 'ARCHITECTURE':
- if self.architecture:
- write(fmt % str(self.architecture), '=')
+ if self.architecture and str(self.architecture):
+ write(fmt % str(self.architecture), ' arch=')
elif named_str == 'SHA1':
if self.dependencies:
out.write(fmt % str(self.dag_hash(7)))
@@ -1820,24 +2050,56 @@ class Spec(object):
result = out.getvalue()
return result
-
def dep_string(self):
return ''.join("^" + dep.format() for dep in self.sorted_deps())
+ def __cmp__(self, other):
+ #Package name sort order is not configurable, always goes alphabetical
+ if self.name != other.name:
+ return cmp(self.name, other.name)
+
+ #Package version is second in compare order
+ pkgname = self.name
+ if self.versions != other.versions:
+ return spack.pkgsort.version_compare(pkgname,
+ self.versions, other.versions)
+
+ #Compiler is third
+ if self.compiler != other.compiler:
+ return spack.pkgsort.compiler_compare(pkgname,
+ self.compiler, other.compiler)
+
+ #Variants
+ if self.variants != other.variants:
+ return spack.pkgsort.variant_compare(pkgname,
+ self.variants, other.variants)
+
+ #Target
+ if self.architecture != other.architecture:
+ return spack.pkgsort.architecture_compare(pkgname,
+ self.architecture, other.architecture)
+
+ #Dependency is not configurable
+ if self.dependencies != other.dependencies:
+ return -1 if self.dependencies < other.dependencies else 1
+
+ #Equal specs
+ return 0
+
+
def __str__(self):
return self.format() + self.dep_string()
-
def tree(self, **kwargs):
"""Prints out this spec and its dependencies, tree-formatted
with indentation."""
- color = kwargs.pop('color', False)
- depth = kwargs.pop('depth', False)
+ color = kwargs.pop('color', False)
+ depth = kwargs.pop('depth', False)
showid = kwargs.pop('ids', False)
- cover = kwargs.pop('cover', 'nodes')
+ cover = kwargs.pop('cover', 'nodes')
indent = kwargs.pop('indent', 0)
- fmt = kwargs.pop('format', '$_$@$%@$+$=')
+ fmt = kwargs.pop('format', '$_$@$%@+$+$=')
prefix = kwargs.pop('prefix', None)
check_kwargs(kwargs, self.tree)
@@ -1861,7 +2123,6 @@ class Spec(object):
out += node.format(fmt, color=color) + "\n"
return out
-
def __repr__(self):
return str(self)
@@ -1869,80 +2130,143 @@ class Spec(object):
#
# These are possible token types in the spec grammar.
#
-DEP, AT, COLON, COMMA, ON, OFF, PCT, EQ, ID = range(9)
+HASH, DEP, AT, COLON, COMMA, ON, OFF, PCT, EQ, QT, ID = range(11)
+
class SpecLexer(spack.parse.Lexer):
+
"""Parses tokens that make up spack specs."""
+
def __init__(self):
super(SpecLexer, self).__init__([
- (r'\^', lambda scanner, val: self.token(DEP, val)),
- (r'\@', lambda scanner, val: self.token(AT, val)),
- (r'\:', lambda scanner, val: self.token(COLON, val)),
- (r'\,', lambda scanner, val: self.token(COMMA, val)),
- (r'\+', lambda scanner, val: self.token(ON, val)),
- (r'\-', lambda scanner, val: self.token(OFF, val)),
- (r'\~', lambda scanner, val: self.token(OFF, val)),
- (r'\%', lambda scanner, val: self.token(PCT, val)),
- (r'\=', lambda scanner, val: self.token(EQ, val)),
+ (r'/', lambda scanner, val: self.token(HASH, val)),
+ (r'\^', lambda scanner, val: self.token(DEP, val)),
+ (r'\@', lambda scanner, val: self.token(AT, val)),
+ (r'\:', lambda scanner, val: self.token(COLON, val)),
+ (r'\,', lambda scanner, val: self.token(COMMA, val)),
+ (r'\+', lambda scanner, val: self.token(ON, val)),
+ (r'\-', lambda scanner, val: self.token(OFF, val)),
+ (r'\~', lambda scanner, val: self.token(OFF, val)),
+ (r'\%', lambda scanner, val: self.token(PCT, val)),
+ (r'\=', lambda scanner, val: self.token(EQ, val)),
# This is more liberal than identifier_re (see above).
# Checked by check_identifier() for better error messages.
+ (r'([\"\'])(?:(?=(\\?))\2.)*?\1',
+ lambda scanner, val: self.token(QT, val)),
(r'\w[\w.-]*', lambda scanner, val: self.token(ID, val)),
- (r'\s+', lambda scanner, val: None)])
+ (r'\s+', lambda scanner, val: None)])
+
+# Lexer is always the same for every parser.
+_lexer = SpecLexer()
class SpecParser(spack.parse.Parser):
- def __init__(self):
- super(SpecParser, self).__init__(SpecLexer())
+ def __init__(self):
+ super(SpecParser, self).__init__(_lexer)
+ self.previous = None
def do_parse(self):
specs = []
-
try:
while self.next:
+ # TODO: clean this parsing up a bit
+ if self.previous:
+ specs.append(self.spec(self.previous.value))
if self.accept(ID):
- specs.append(self.spec())
+ self.previous = self.token
+ if self.accept(EQ):
+ if not specs:
+ specs.append(self.spec(None))
+ if self.accept(QT):
+ self.token.value = self.token.value[1:-1]
+ else:
+ self.expect(ID)
+ specs[-1]._add_flag(
+ self.previous.value, self.token.value)
+ else:
+ specs.append(self.spec(self.previous.value))
+ self.previous = None
+ elif self.accept(HASH):
+ specs.append(self.spec_by_hash())
elif self.accept(DEP):
if not specs:
- self.last_token_error("Dependency has no package")
- self.expect(ID)
- specs[-1]._add_dependency(self.spec())
+ self.previous = self.token
+ specs.append(self.spec(None))
+ self.previous = None
+ if self.accept(HASH):
+ specs[-1]._add_dependency(self.spec_by_hash())
+ else:
+ self.expect(ID)
+ specs[-1]._add_dependency(self.spec(self.token.value))
else:
- self.unexpected_token()
+ # Attempt to construct an anonymous spec, but check that
+ # the first token is valid
+ # TODO: Is this check even necessary, or will it all be Lex
+ # errors now?
+ specs.append(self.spec(None, True))
+
except spack.parse.ParseError, e:
raise SpecParseError(e)
- return specs
+ # If the spec has an os or a target and no platform, give it the default platform
+ for spec in specs:
+ for s in spec.traverse():
+ if s.architecture.os_string or s.architecture.target_string:
+ s._set_platform(spack.architecture.sys_type())
+ return specs
def parse_compiler(self, text):
self.setup(text)
return self.compiler()
+ def spec_by_hash(self):
+ self.expect(ID)
+
+ specs = spack.installed_db.query()
+ matches = [spec for spec in specs if
+ spec.dag_hash()[:len(self.token.value)] == self.token.value]
- def spec(self):
+ if not matches:
+ tty.die("%s does not match any installed packages." %
+ self.token.value)
+
+ if len(matches) != 1:
+ raise AmbiguousHashError(
+ "Multiple packages specify hash %s." % self.token.value,
+ *matches)
+
+ return matches[0]
+
+ def spec(self, name, check_valid_token=False):
"""Parse a spec out of the input. If a spec is supplied, then initialize
and return it instead of creating a new one."""
-
- spec_namespace, dot, spec_name = self.token.value.rpartition('.')
- if not spec_namespace:
+ if name:
+ spec_namespace, dot, spec_name = name.rpartition('.')
+ if not spec_namespace:
+ spec_namespace = None
+ self.check_identifier(spec_name)
+ else:
spec_namespace = None
-
- self.check_identifier(spec_name)
+ spec_name = None
# This will init the spec without calling __init__.
spec = Spec.__new__(Spec)
spec.name = spec_name
spec.versions = VersionList()
spec.variants = VariantMap(spec)
- spec.architecture = None
+ spec.architecture = spack.architecture.Arch()
spec.compiler = None
spec.external = None
- spec.dependents = DependencyMap()
+ spec.external_module = None
+ spec.compiler_flags = FlagMap(spec)
+ spec.dependents = DependencyMap()
spec.dependencies = DependencyMap()
spec.namespace = spec_namespace
+ spec._hash = None
spec._normal = False
spec._concrete = False
@@ -1951,26 +2275,51 @@ class SpecParser(spack.parse.Parser):
# unspecified or not.
added_version = False
+ if self.previous and self.previous.value == DEP:
+ if self.accept(HASH):
+ spec.add_dependency(self.spec_by_hash())
+ else:
+ self.expect(ID)
+ if self.accept(EQ):
+ raise SpecParseError(spack.parse.ParseError(
+ "", "", "Expected dependency received anonymous spec"))
+ spec.add_dependency(self.spec(self.token.value))
+
while self.next:
if self.accept(AT):
vlist = self.version_list()
for version in vlist:
spec._add_version(version)
added_version = True
+ check_valid_token = False
elif self.accept(ON):
spec._add_variant(self.variant(), True)
+ check_valid_token = False
elif self.accept(OFF):
spec._add_variant(self.variant(), False)
+ check_valid_token = False
elif self.accept(PCT):
spec._set_compiler(self.compiler())
+ check_valid_token = False
- elif self.accept(EQ):
- spec._set_architecture(self.architecture())
+ elif self.accept(ID):
+ self.previous = self.token
+ if self.accept(EQ):
+ if self.accept(QT):
+ self.token.value = self.token.value[1:-1]
+ else:
+ self.expect(ID)
+ spec._add_flag(self.previous.value, self.token.value)
+ self.previous = None
+ else:
+ return spec
else:
+ if check_valid_token:
+ self.unexpected_token()
break
# If there was no version in the spec, consier it an open range
@@ -1979,17 +2328,14 @@ class SpecParser(spack.parse.Parser):
return spec
-
- def variant(self):
- self.expect(ID)
- self.check_identifier()
- return self.token.value
-
-
- def architecture(self):
- self.expect(ID)
- return self.token.value
-
+ def variant(self, name=None):
+ # TODO: Make generalized variants possible
+ if name:
+ return name
+ else:
+ self.expect(ID)
+ self.check_identifier()
+ return self.token.value
def version(self):
start = None
@@ -2007,11 +2353,12 @@ class SpecParser(spack.parse.Parser):
# No colon and no id: invalid version.
self.next_token_error("Invalid version specifier")
- if start: start = Version(start)
- if end: end = Version(end)
+ if start:
+ start = Version(start)
+ if end:
+ end = Version(end)
return VersionRange(start, end)
-
def version_list(self):
vlist = []
vlist.append(self.version())
@@ -2019,7 +2366,6 @@ class SpecParser(spack.parse.Parser):
vlist.append(self.version())
return vlist
-
def compiler(self):
self.expect(ID)
self.check_identifier()
@@ -2035,7 +2381,6 @@ class SpecParser(spack.parse.Parser):
compiler.versions = VersionList(':')
return compiler
-
def check_identifier(self, id=None):
"""The only identifiers that can contain '.' are versions, but version
ids are context-sensitive so we have to check on a case-by-case
@@ -2068,9 +2413,16 @@ def parse_anonymous_spec(spec_like, pkg_name):
if isinstance(spec_like, str):
try:
anon_spec = Spec(spec_like)
+ if anon_spec.name != pkg_name:
+ raise SpecParseError(spack.parse.ParseError(
+ "",
+ "",
+ "Expected anonymous spec for package %s but found spec for"
+ "package %s" % (pkg_name, anon_spec.name)))
except SpecParseError:
- anon_spec = Spec(pkg_name + spec_like)
- if anon_spec.name != pkg_name: raise ValueError(
+ anon_spec = Spec(pkg_name + ' ' + spec_like)
+ if anon_spec.name != pkg_name:
+ raise ValueError(
"Invalid spec for package %s: %s" % (pkg_name, spec_like))
else:
anon_spec = spec_like.copy()
@@ -2083,13 +2435,17 @@ def parse_anonymous_spec(spec_like, pkg_name):
class SpecError(spack.error.SpackError):
+
"""Superclass for all errors that occur while constructing specs."""
+
def __init__(self, message):
super(SpecError, self).__init__(message)
class SpecParseError(SpecError):
+
"""Wrapper for ParseError for when we're parsing specs."""
+
def __init__(self, parse_error):
super(SpecParseError, self).__init__(parse_error.message)
self.string = parse_error.string
@@ -2097,61 +2453,79 @@ class SpecParseError(SpecError):
class DuplicateDependencyError(SpecError):
+
"""Raised when the same dependency occurs in a spec twice."""
+
def __init__(self, message):
super(DuplicateDependencyError, self).__init__(message)
class DuplicateVariantError(SpecError):
+
"""Raised when the same variant occurs in a spec twice."""
+
def __init__(self, message):
super(DuplicateVariantError, self).__init__(message)
class DuplicateCompilerSpecError(SpecError):
+
"""Raised when the same compiler occurs in a spec twice."""
+
def __init__(self, message):
super(DuplicateCompilerSpecError, self).__init__(message)
class UnsupportedCompilerError(SpecError):
+
"""Raised when the user asks for a compiler spack doesn't know about."""
+
def __init__(self, compiler_name):
super(UnsupportedCompilerError, self).__init__(
"The '%s' compiler is not yet supported." % compiler_name)
class UnknownVariantError(SpecError):
+
"""Raised when the same variant occurs in a spec twice."""
+
def __init__(self, pkg, variant):
super(UnknownVariantError, self).__init__(
"Package %s has no variant %s!" % (pkg, variant))
class DuplicateArchitectureError(SpecError):
+
"""Raised when the same architecture occurs in a spec twice."""
+
def __init__(self, message):
super(DuplicateArchitectureError, self).__init__(message)
class InconsistentSpecError(SpecError):
+
"""Raised when two nodes in the same spec DAG have inconsistent
constraints."""
+
def __init__(self, message):
super(InconsistentSpecError, self).__init__(message)
class InvalidDependencyException(SpecError):
+
"""Raised when a dependency in a spec is not actually a dependency
of the package."""
+
def __init__(self, message):
super(InvalidDependencyException, self).__init__(message)
class NoProviderError(SpecError):
+
"""Raised when there is no package that provides a particular
virtual dependency.
"""
+
def __init__(self, vpkg):
super(NoProviderError, self).__init__(
"No providers found for virtual package: '%s'" % vpkg)
@@ -2159,9 +2533,11 @@ class NoProviderError(SpecError):
class MultipleProviderError(SpecError):
+
"""Raised when there is no package that provides a particular
virtual dependency.
"""
+
def __init__(self, vpkg, providers):
"""Takes the name of the vpkg"""
super(MultipleProviderError, self).__init__(
@@ -2172,8 +2548,10 @@ class MultipleProviderError(SpecError):
class UnsatisfiableSpecError(SpecError):
+
"""Raised when a spec conflicts with package constraints.
Provide the requirement that was violated when raising."""
+
def __init__(self, provided, required, constraint_type):
super(UnsatisfiableSpecError, self).__init__(
"%s does not satisfy %s" % (provided, required))
@@ -2183,55 +2561,96 @@ class UnsatisfiableSpecError(SpecError):
class UnsatisfiableSpecNameError(UnsatisfiableSpecError):
+
"""Raised when two specs aren't even for the same package."""
+
def __init__(self, provided, required):
super(UnsatisfiableSpecNameError, self).__init__(
provided, required, "name")
class UnsatisfiableVersionSpecError(UnsatisfiableSpecError):
+
"""Raised when a spec version conflicts with package constraints."""
+
def __init__(self, provided, required):
super(UnsatisfiableVersionSpecError, self).__init__(
provided, required, "version")
class UnsatisfiableCompilerSpecError(UnsatisfiableSpecError):
+
"""Raised when a spec comiler conflicts with package constraints."""
+
def __init__(self, provided, required):
super(UnsatisfiableCompilerSpecError, self).__init__(
provided, required, "compiler")
class UnsatisfiableVariantSpecError(UnsatisfiableSpecError):
+
"""Raised when a spec variant conflicts with package constraints."""
+
def __init__(self, provided, required):
super(UnsatisfiableVariantSpecError, self).__init__(
provided, required, "variant")
+class UnsatisfiableCompilerFlagSpecError(UnsatisfiableSpecError):
+
+ """Raised when a spec variant conflicts with package constraints."""
+
+ def __init__(self, provided, required):
+ super(UnsatisfiableCompilerFlagSpecError, self).__init__(
+ provided, required, "compiler_flags")
+
+
class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError):
+
"""Raised when a spec architecture conflicts with package constraints."""
+
def __init__(self, provided, required):
super(UnsatisfiableArchitectureSpecError, self).__init__(
provided, required, "architecture")
class UnsatisfiableProviderSpecError(UnsatisfiableSpecError):
+
"""Raised when a provider is supplied but constraints don't match
a vpkg requirement"""
+
def __init__(self, provided, required):
super(UnsatisfiableProviderSpecError, self).__init__(
provided, required, "provider")
# TODO: get rid of this and be more specific about particular incompatible
# dep constraints
+
+
class UnsatisfiableDependencySpecError(UnsatisfiableSpecError):
+
"""Raised when some dependency of constrained specs are incompatible"""
+
def __init__(self, provided, required):
super(UnsatisfiableDependencySpecError, self).__init__(
provided, required, "dependency")
+
class SpackYAMLError(spack.error.SpackError):
+
def __init__(self, msg, yaml_error):
super(SpackYAMLError, self).__init__(msg, str(yaml_error))
+
+
+class SpackRecordError(spack.error.SpackError):
+
+ def __init__(self, msg):
+ super(SpackRecordError, self).__init__(msg)
+
+
+class AmbiguousHashError(SpecError):
+
+ def __init__(self, msg, *specs):
+ super(AmbiguousHashError, self).__init__(msg)
+ for spec in specs:
+ print ' ', spec.format('$.$@$%@+$+$=$#')
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index 1668e271fa..fb91f24721 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -31,15 +31,16 @@ from llnl.util.filesystem import join_path
from llnl.util.tty.colify import colify
from spack.test.tally_plugin import Tally
"""Names of tests to be included in Spack's test suite"""
-test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage',
+
+test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage',
'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize',
'multimethod', 'install', 'package_sanity', 'config',
'directory_layout', 'pattern', 'python_version', 'git_fetch',
'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate',
'cc', 'link_tree', 'spec_yaml', 'optional_deps',
'make_executable', 'configure_guess', 'lock', 'database',
- 'namespace_trie', 'yaml', 'sbang', 'environment',
- 'cmd.uninstall', 'cmd.test_install']
+ 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find',
+ 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd']
def list_tests():
diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py
new file mode 100644
index 0000000000..a6847c5744
--- /dev/null
+++ b/lib/spack/spack/test/architecture.py
@@ -0,0 +1,112 @@
+""" Test checks if the architecture class is created correctly and also that
+ the functions are looking for the correct architecture name
+"""
+import unittest
+import os
+import platform as py_platform
+import spack
+from spack.architecture import *
+from spack.spec import *
+from spack.platforms.cray_xc import CrayXc
+from spack.platforms.linux import Linux
+from spack.platforms.bgq import Bgq
+from spack.platforms.darwin import Darwin
+
+from spack.test.mock_packages_test import *
+
+#class ArchitectureTest(unittest.TestCase):
+class ArchitectureTest(MockPackagesTest):
+
+ def setUp(self):
+ super(ArchitectureTest, self).setUp()
+ self.platform = sys_type()
+
+ def tearDown(self):
+ super(ArchitectureTest, self).tearDown()
+
+ def test_dict_functions_for_architecture(self):
+ arch = Arch()
+ arch.platform = spack.architecture.sys_type()
+ arch.platform_os = arch.platform.operating_system('default_os')
+ arch.target = arch.platform.target('default_target')
+
+ d = arch.to_dict()
+
+ new_arch = spack.architecture.arch_from_dict(d)
+
+ self.assertEqual(arch, new_arch)
+
+ self.assertTrue( isinstance(arch, Arch) )
+ self.assertTrue( isinstance(arch.platform, Platform) )
+ self.assertTrue( isinstance(arch.platform_os, OperatingSystem) )
+ self.assertTrue( isinstance(arch.target, Target) )
+ self.assertTrue( isinstance(new_arch, Arch) )
+ self.assertTrue( isinstance(new_arch.platform, Platform) )
+ self.assertTrue( isinstance(new_arch.platform_os, OperatingSystem) )
+ self.assertTrue( isinstance(new_arch.target, Target) )
+
+
+ def test_sys_type(self):
+ output_platform_class = sys_type()
+ my_arch_class = None
+ if os.path.exists('/opt/cray/craype'):
+ my_platform_class = CrayXc()
+ elif os.path.exists('/bgsys'):
+ my_platform_class = Bgq()
+ elif 'Linux' in py_platform.system():
+ my_platform_class = Linux()
+ elif 'Darwin' in py_platform.system():
+ my_platform_class = Darwin()
+
+ self.assertEqual(str(output_platform_class), str(my_platform_class))
+
+ def test_user_front_end_input(self):
+ """Test when user inputs just frontend that both the frontend target
+ and frontend operating system match
+ """
+ frontend_os = self.platform.operating_system("frontend")
+ frontend_target = self.platform.target("frontend")
+ frontend_spec = Spec("libelf os=frontend target=frontend")
+ frontend_spec.concretize()
+ self.assertEqual(frontend_os, frontend_spec.architecture.platform_os)
+ self.assertEqual(frontend_target, frontend_spec.architecture.target)
+
+ def test_user_back_end_input(self):
+ """Test when user inputs backend that both the backend target and
+ backend operating system match
+ """
+ backend_os = self.platform.operating_system("backend")
+ backend_target = self.platform.target("backend")
+ backend_spec = Spec("libelf os=backend target=backend")
+ backend_spec.concretize()
+ self.assertEqual(backend_os, backend_spec.architecture.platform_os)
+ self.assertEqual(backend_target, backend_spec.architecture.target)
+
+ def test_user_defaults(self):
+ default_os = self.platform.operating_system("default_os")
+ default_target = self.platform.target("default_target")
+
+ default_spec = Spec("libelf") # default is no args
+ default_spec.concretize()
+ self.assertEqual(default_os, default_spec.architecture.platform_os)
+ self.assertEqual(default_target, default_spec.architecture.target)
+
+ def test_user_input_combination(self):
+ os_list = self.platform.operating_sys.keys()
+ target_list = self.platform.targets.keys()
+ additional = ["fe", "be", "frontend", "backend"]
+
+ os_list.extend(additional)
+ target_list.extend(additional)
+
+ combinations = itertools.product(os_list, target_list)
+ results = []
+ for arch in combinations:
+ o,t = arch
+ spec = Spec("libelf os=%s target=%s" % (o, t))
+ spec.concretize()
+ results.append(spec.architecture.platform_os == self.platform.operating_system(o))
+ results.append(spec.architecture.target == self.platform.target(t))
+ res = all(results)
+
+ self.assertTrue(res)
diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py
index a630866143..ea2b164462 100644
--- a/lib/spack/spack/test/cc.py
+++ b/lib/spack/spack/test/cc.py
@@ -56,11 +56,16 @@ class CompilerTest(unittest.TestCase):
self.cc = Executable(join_path(spack.build_env_path, "cc"))
self.ld = Executable(join_path(spack.build_env_path, "ld"))
self.cpp = Executable(join_path(spack.build_env_path, "cpp"))
+ self.cxx = Executable(join_path(spack.build_env_path, "c++"))
+ self.fc = Executable(join_path(spack.build_env_path, "fc"))
self.realcc = "/bin/mycc"
self.prefix = "/spack-test-prefix"
os.environ['SPACK_CC'] = self.realcc
+ os.environ['SPACK_CXX'] = self.realcc
+ os.environ['SPACK_FC'] = self.realcc
+
os.environ['SPACK_PREFIX'] = self.prefix
os.environ['SPACK_ENV_PATH']="test"
os.environ['SPACK_DEBUG_LOG_DIR'] = "."
@@ -102,6 +107,15 @@ class CompilerTest(unittest.TestCase):
self.assertEqual(self.cc(*args, output=str).strip(), expected)
+ def check_cxx(self, command, args, expected):
+ os.environ['SPACK_TEST_COMMAND'] = command
+ self.assertEqual(self.cxx(*args, output=str).strip(), expected)
+
+ def check_fc(self, command, args, expected):
+ os.environ['SPACK_TEST_COMMAND'] = command
+ self.assertEqual(self.fc(*args, output=str).strip(), expected)
+
+
def check_ld(self, command, args, expected):
os.environ['SPACK_TEST_COMMAND'] = command
self.assertEqual(self.ld(*args, output=str).strip(), expected)
@@ -142,6 +156,64 @@ class CompilerTest(unittest.TestCase):
self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld")
+ def test_flags(self):
+ os.environ['SPACK_LDFLAGS'] = '-L foo'
+ os.environ['SPACK_LDLIBS'] = '-lfoo'
+ os.environ['SPACK_CPPFLAGS'] = '-g -O1'
+ os.environ['SPACK_CFLAGS'] = '-Wall'
+ os.environ['SPACK_CXXFLAGS'] = '-Werror'
+ os.environ['SPACK_FFLAGS'] = '-w'
+
+ # Test ldflags added properly in ld mode
+ self.check_ld('dump-args', test_command,
+ "ld " +
+ '-rpath ' + self.prefix + '/lib ' +
+ '-rpath ' + self.prefix + '/lib64 ' +
+ '-L foo ' +
+ ' '.join(test_command) + ' ' +
+ '-lfoo')
+
+ # Test cppflags added properly in cpp mode
+ self.check_cpp('dump-args', test_command,
+ "cpp " +
+ '-g -O1 ' +
+ ' '.join(test_command))
+
+ # Test ldflags, cppflags, and language specific flags are added in proper order
+ self.check_cc('dump-args', test_command,
+ self.realcc + ' ' +
+ '-Wl,-rpath,' + self.prefix + '/lib ' +
+ '-Wl,-rpath,' + self.prefix + '/lib64 ' +
+ '-g -O1 ' +
+ '-Wall ' +
+ '-L foo ' +
+ ' '.join(test_command) + ' ' +
+ '-lfoo')
+
+ self.check_cxx('dump-args', test_command,
+ self.realcc + ' ' +
+ '-Wl,-rpath,' + self.prefix + '/lib ' +
+ '-Wl,-rpath,' + self.prefix + '/lib64 ' +
+ '-g -O1 ' +
+ '-Werror ' +
+ '-L foo ' +
+ ' '.join(test_command) + ' ' +
+ '-lfoo')
+
+ self.check_fc('dump-args', test_command,
+ self.realcc + ' ' +
+ '-Wl,-rpath,' + self.prefix + '/lib ' +
+ '-Wl,-rpath,' + self.prefix + '/lib64 ' +
+ '-w ' +
+ '-g -O1 ' +
+ '-L foo ' +
+ ' '.join(test_command) + ' ' +
+ '-lfoo')
+
+ os.environ['SPACK_LDFLAGS']=''
+ os.environ['SPACK_LDLIBS']=''
+
+
def test_dep_rpath(self):
"""Ensure RPATHs for root package are added."""
self.check_cc('dump-args', test_command,
diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py
new file mode 100644
index 0000000000..371e9650e0
--- /dev/null
+++ b/lib/spack/spack/test/cmd/find.py
@@ -0,0 +1,60 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+
+import spack.cmd.find
+import unittest
+
+
+class Bunch(object):
+
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+
+
+class FindTest(unittest.TestCase):
+
+ def test_query_arguments(self):
+ query_arguments = spack.cmd.find.query_arguments
+ # Default arguments
+ args = Bunch(only_missing=False, missing=False,
+ unknown=False, explicit=False, implicit=False)
+ q_args = query_arguments(args)
+ self.assertTrue('installed' in q_args)
+ self.assertTrue('known' in q_args)
+ self.assertTrue('explicit' in q_args)
+ self.assertEqual(q_args['installed'], True)
+ self.assertEqual(q_args['known'], any)
+ self.assertEqual(q_args['explicit'], any)
+ # Check that explicit works correctly
+ args.explicit = True
+ q_args = query_arguments(args)
+ self.assertEqual(q_args['explicit'], True)
+ args.explicit = False
+ args.implicit = True
+ q_args = query_arguments(args)
+ self.assertEqual(q_args['explicit'], False)
+ args.explicit = True
+ self.assertRaises(SystemExit, query_arguments, args)
diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py
new file mode 100644
index 0000000000..d89814154b
--- /dev/null
+++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py
@@ -0,0 +1,81 @@
+import os
+import shutil
+from tempfile import mkdtemp
+
+from llnl.util.filesystem import set_executable, mkdirp
+
+import spack.spec
+import spack.cmd.compiler
+import spack.compilers
+from spack.version import Version
+from spack.test.mock_packages_test import *
+
+test_version = '4.5-spacktest'
+
+class MockArgs(object):
+ def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None):
+ self.add_paths = add_paths
+ self.scope = scope
+ self.compiler_spec = compiler_spec
+ self.all = all
+
+
+def make_mock_compiler():
+ """Make a directory containing a fake, but detectable compiler."""
+ mock_compiler_dir = mkdtemp()
+ bin_dir = os.path.join(mock_compiler_dir, 'bin')
+ mkdirp(bin_dir)
+
+ gcc_path = os.path.join(bin_dir, 'gcc')
+ gxx_path = os.path.join(bin_dir, 'g++')
+ gfortran_path = os.path.join(bin_dir, 'gfortran')
+
+ with open(gcc_path, 'w') as f:
+ f.write("""\
+#!/bin/sh
+
+for arg in "$@"; do
+ if [ "$arg" = -dumpversion ]; then
+ echo '%s'
+ fi
+done
+""" % test_version)
+
+ # Create some mock compilers in the temporary directory
+ set_executable(gcc_path)
+ shutil.copy(gcc_path, gxx_path)
+ shutil.copy(gcc_path, gfortran_path)
+
+ return mock_compiler_dir
+
+
+class CompilerCmdTest(MockPackagesTest):
+ """ Test compiler commands for add and remove """
+
+
+ def test_compiler_remove(self):
+ args = MockArgs(all=True, compiler_spec='gcc@4.5.0')
+ spack.cmd.compiler.compiler_remove(args)
+ compilers = spack.compilers.all_compilers()
+ self.assertTrue(spack.spec.CompilerSpec("gcc@4.5.0") not in compilers)
+
+
+ def test_compiler_add(self):
+ # compilers available by default.
+ old_compilers = set(spack.compilers.all_compilers())
+
+ # add our new compiler and find again.
+ compiler_dir = make_mock_compiler()
+
+ try:
+ args = MockArgs(add_paths=[compiler_dir])
+ spack.cmd.compiler.compiler_find(args)
+
+ # ensure new compiler is in there
+ new_compilers = set(spack.compilers.all_compilers())
+ new_compiler = new_compilers - old_compilers
+ self.assertTrue(new_compiler)
+ self.assertTrue(new_compiler.pop().version == Version(test_version))
+
+ finally:
+ shutil.rmtree(compiler_dir, ignore_errors=True)
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 799fdae3a9..ab201f406a 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -23,6 +23,7 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import spack
+import spack.architecture
from spack.spec import Spec, CompilerSpec
from spack.version import ver
from spack.concretize import find_spec
@@ -38,11 +39,20 @@ class ConcretizeTest(MockPackagesTest):
for name in abstract.variants:
avariant = abstract.variants[name]
cvariant = concrete.variants[name]
- self.assertEqual(avariant.enabled, cvariant.enabled)
+ self.assertEqual(avariant.value, cvariant.value)
+
+ if abstract.compiler_flags:
+ for flag in abstract.compiler_flags:
+ aflag = abstract.compiler_flags[flag]
+ cflag = concrete.compiler_flags[flag]
+ self.assertTrue(set(aflag) <= set(cflag))
for name in abstract.package.variants:
self.assertTrue(name in concrete.variants)
+ for flag in concrete.compiler_flags.valid_compiler_flags():
+ self.assertTrue(flag in concrete.compiler_flags)
+
if abstract.compiler and abstract.compiler.concrete:
self.assertEqual(abstract.compiler, concrete.compiler)
@@ -75,9 +85,14 @@ class ConcretizeTest(MockPackagesTest):
def test_concretize_variant(self):
self.check_concretize('mpich+debug')
self.check_concretize('mpich~debug')
+ self.check_concretize('mpich debug=2')
self.check_concretize('mpich')
+ def test_conretize_compiler_flags(self):
+ self.check_concretize('mpich cppflags="-O3"')
+
+
def test_concretize_preferred_version(self):
spec = self.check_concretize('python')
self.assertEqual(spec.versions, ver('2.7.11'))
@@ -231,7 +246,7 @@ class ConcretizeTest(MockPackagesTest):
def test_external_package(self):
- spec = Spec('externaltool')
+ spec = Spec('externaltool%gcc')
spec.concretize()
self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
@@ -239,6 +254,19 @@ class ConcretizeTest(MockPackagesTest):
self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+ def test_external_package_module(self):
+ # No tcl modules on darwin/linux machines
+ # TODO: improved way to check for this.
+ if (spack.architecture.sys_type().name == 'darwin' or
+ spack.architecture.sys_type().name == 'linux'):
+ return
+
+ spec = Spec('externalmodule')
+ spec.concretize()
+ self.assertEqual(spec['externalmodule'].external_module, 'external-module')
+ self.assertFalse('externalprereq' in spec)
+ self.assertTrue(spec['externalmodule'].compiler.satisfies('gcc'))
+
def test_nobuild_package(self):
got_error = False
spec = Spec('externaltool%clang')
diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py
index eff482f4c6..252d77e66b 100644
--- a/lib/spack/spack/test/config.py
+++ b/lib/spack/spack/test/config.py
@@ -32,45 +32,75 @@ from ordereddict_backport import OrderedDict
from spack.test.mock_packages_test import *
# Some sample compiler config data
-a_comps = {
- "x86_64_E5v2_IntelIB": {
- "gcc@4.7.3" : {
+a_comps = [
+ {'compiler': {
+ 'paths': {
"cc" : "/gcc473",
"cxx": "/g++473",
"f77": None,
- "fc" : None },
- "gcc@4.5.0" : {
+ "fc" : None
+ },
+ 'modules': None,
+ 'spec': 'gcc@4.7.3',
+ 'operating_system': 'CNL10'
+ }},
+ {'compiler': {
+ 'paths': {
"cc" : "/gcc450",
"cxx": "/g++450",
- "f77": "/gfortran",
- "fc" : "/gfortran" },
- "clang@3.3" : {
+ "f77": 'gfortran',
+ "fc" : 'gfortran'
+ },
+ 'modules': None,
+ 'spec': 'gcc@4.5.0',
+ 'operating_system': 'CNL10'
+ }},
+ {'compiler': {
+ 'paths': {
"cc" : "<overwritten>",
"cxx": "<overwritten>",
- "f77": "<overwritten>",
- "fc" : "<overwritten>" }
- }
-}
-
-b_comps = {
- "x86_64_E5v3": {
- "icc@10.0" : {
+ "f77": '<overwritten>',
+ "fc" : '<overwritten>' },
+ 'modules': None,
+ 'spec': 'clang@3.3',
+ 'operating_system': 'CNL10'
+ }}
+]
+
+b_comps = [
+ {'compiler': {
+ 'paths': {
"cc" : "/icc100",
- "cxx": "/icc100",
+ "cxx": "/icp100",
"f77": None,
- "fc" : None },
- "icc@11.1" : {
+ "fc" : None
+ },
+ 'modules': None,
+ 'spec': 'icc@10.0',
+ 'operating_system': 'CNL10'
+ }},
+ {'compiler': {
+ 'paths': {
"cc" : "/icc111",
"cxx": "/icp111",
- "f77": "/ifort",
- "fc" : "/ifort" },
- "clang@3.3" : {
- "cc" : "/clang",
- "cxx": "/clang++",
- "f77": None,
- "fc" : None}
- }
-}
+ "f77": 'ifort',
+ "fc" : 'ifort'
+ },
+ 'modules': None,
+ 'spec': 'icc@11.1',
+ 'operating_system': 'CNL10'
+ }},
+ {'compiler': {
+ 'paths': {
+ "cc" : "<overwritten>",
+ "cxx": "<overwritten>",
+ "f77": '<overwritten>',
+ "fc" : '<overwritten>' },
+ 'modules': None,
+ 'spec': 'clang@3.3',
+ 'operating_system': 'CNL10'
+ }}
+]
# Some Sample repo data
repos_low = [ "/some/path" ]
@@ -89,15 +119,28 @@ class ConfigTest(MockPackagesTest):
super(ConfigTest, self).tearDown()
shutil.rmtree(self.tmp_dir, True)
- def check_config(self, comps, arch, *compiler_names):
+
+ def check_config(self, comps, *compiler_names):
"""Check that named compilers in comps match Spack's config."""
config = spack.config.get_config('compilers')
compiler_list = ['cc', 'cxx', 'f77', 'fc']
- for key in compiler_names:
- for c in compiler_list:
- expected = comps[arch][key][c]
- actual = config[arch][key][c]
- self.assertEqual(expected, actual)
+ param_list = ['modules', 'paths', 'spec', 'operating_system']
+ for compiler in config:
+ conf = compiler['compiler']
+ if conf['spec'] in compiler_names:
+ comp = None
+ for c in comps:
+ if c['compiler']['spec'] == conf['spec']:
+ comp = c['compiler']
+ break
+ if not comp:
+ self.fail('Bad config spec')
+ for p in param_list:
+ self.assertEqual(conf[p], comp[p])
+ for c in compiler_list:
+ expected = comp['paths'][c]
+ actual = conf['paths'][c]
+ self.assertEqual(expected, actual)
def test_write_list_in_memory(self):
spack.config.update_config('repos', repos_low, 'test_low_priority')
@@ -111,8 +154,9 @@ class ConfigTest(MockPackagesTest):
spack.config.update_config('compilers', b_comps, 'test_high_priority')
# Make sure the config looks how we expect.
- self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0')
- self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3')
+ self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0')
+ self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3')
+
def test_write_key_to_disk(self):
# Write b_comps "on top of" a_comps.
@@ -123,8 +167,8 @@ class ConfigTest(MockPackagesTest):
spack.config.clear_config_caches()
# Same check again, to ensure consistency.
- self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0')
- self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3')
+ self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0')
+ self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3')
def test_write_to_same_priority_file(self):
# Write b_comps in the same file as a_comps.
@@ -135,5 +179,5 @@ class ConfigTest(MockPackagesTest):
spack.config.clear_config_caches()
# Same check again, to ensure consistency.
- self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0')
- self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3')
+ self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0')
+ self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3')
diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh
new file mode 100644
index 0000000000..800f639ac8
--- /dev/null
+++ b/lib/spack/spack/test/data/sourceme_first.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export NEW_VAR='new'
+export UNSET_ME='overridden'
diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh
new file mode 100644
index 0000000000..9955a0e6d6
--- /dev/null
+++ b/lib/spack/spack/test/data/sourceme_second.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export PATH_LIST='/path/first:/path/second:/path/fourth'
+unset EMPTY_PATH_LIST \ No newline at end of file
diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py
index ded1539e18..f3644cb0b7 100644
--- a/lib/spack/spack/test/environment.py
+++ b/lib/spack/spack/test/environment.py
@@ -24,16 +24,24 @@
##############################################################################
import unittest
import os
+
+from spack import spack_root
+from llnl.util.filesystem import join_path
from spack.environment import EnvironmentModifications
+from spack.environment import SetEnv, UnsetEnv
+from spack.environment import RemovePath, PrependPath, AppendPath
class EnvironmentTest(unittest.TestCase):
+
def setUp(self):
- os.environ.clear()
os.environ['UNSET_ME'] = 'foo'
os.environ['EMPTY_PATH_LIST'] = ''
os.environ['PATH_LIST'] = '/path/second:/path/third'
- os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'
+ os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' # NOQA: ignore=E501
+
+ def tearDown(self):
+ pass
def test_set(self):
env = EnvironmentModifications()
@@ -74,9 +82,18 @@ class EnvironmentTest(unittest.TestCase):
env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
env.apply_modifications()
- self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST'])
- self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST'])
- self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST'])
+ self.assertEqual(
+ '/path/first:/path/second:/path/third:/path/last',
+ os.environ['PATH_LIST']
+ )
+ self.assertEqual(
+ '/path/first:/path/middle:/path/last',
+ os.environ['EMPTY_PATH_LIST']
+ )
+ self.assertEqual(
+ '/path/first:/path/middle:/path/last',
+ os.environ['NEWLY_CREATED_PATH_LIST']
+ )
self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
def test_extra_arguments(self):
@@ -95,3 +112,41 @@ class EnvironmentTest(unittest.TestCase):
self.assertEqual(len(copy_construct), 2)
for x, y in zip(env, copy_construct):
assert x is y
+
+ def test_source_files(self):
+ datadir = join_path(spack_root, 'lib', 'spack',
+ 'spack', 'test', 'data')
+ files = [
+ join_path(datadir, 'sourceme_first.sh'),
+ join_path(datadir, 'sourceme_second.sh')
+ ]
+ env = EnvironmentModifications.from_sourcing_files(*files)
+ modifications = env.group_by_name()
+
+ self.assertEqual(len(modifications), 4)
+ # Set new variables
+ self.assertEqual(len(modifications['NEW_VAR']), 1)
+ self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv))
+ self.assertEqual(modifications['NEW_VAR'][0].value, 'new')
+ # Unset variables
+ self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1)
+ self.assertTrue(isinstance(
+ modifications['EMPTY_PATH_LIST'][0], UnsetEnv))
+ # Modified variables
+ self.assertEqual(len(modifications['UNSET_ME']), 1)
+ self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv))
+ self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden')
+
+ self.assertEqual(len(modifications['PATH_LIST']), 3)
+ self.assertTrue(
+ isinstance(modifications['PATH_LIST'][0], RemovePath)
+ )
+ self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third')
+ self.assertTrue(
+ isinstance(modifications['PATH_LIST'][1], AppendPath)
+ )
+ self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth')
+ self.assertTrue(
+ isinstance(modifications['PATH_LIST'][2], PrependPath)
+ )
+ self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first')
diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py
index 595667bf35..a56bd8ebdc 100644
--- a/lib/spack/spack/test/mock_packages_test.py
+++ b/lib/spack/spack/test/mock_packages_test.py
@@ -34,20 +34,127 @@ from ordereddict_backport import OrderedDict
from spack.repository import RepoPath
from spack.spec import Spec
+platform = spack.architecture.sys_type()
+
+linux_os_name = 'debian'
+linux_os_version = '6'
+
+if platform.name == 'linux':
+ linux_os = platform.operating_system("default_os")
+ linux_os_name = linux_os.name
+ linux_os_version = linux_os.version
+
mock_compiler_config = """\
compilers:
- all:
- clang@3.3:
+- compiler:
+ spec: clang@3.3
+ operating_system: {0}{1}
+ paths:
cc: /path/to/clang
cxx: /path/to/clang++
f77: None
fc: None
- gcc@4.5.0:
+ modules: 'None'
+- compiler:
+ spec: gcc@4.5.0
+ operating_system: {0}{1}
+ paths:
+ cc: /path/to/gcc
+ cxx: /path/to/g++
+ f77: None
+ fc: None
+ modules: 'None'
+- compiler:
+ spec: clang@3.3
+ operating_system: CNL10
+ paths:
+ cc: /path/to/clang
+ cxx: /path/to/clang++
+ f77: None
+ fc: None
+ modules: 'None'
+- compiler:
+ spec: clang@3.3
+ operating_system: SuSE11
+ paths:
+ cc: /path/to/clang
+ cxx: /path/to/clang++
+ f77: None
+ fc: None
+ modules: 'None'
+- compiler:
+ spec: clang@3.3
+ operating_system: redhat6
+ paths:
+ cc: /path/to/clang
+ cxx: /path/to/clang++
+ f77: None
+ fc: None
+ modules: 'None'
+- compiler:
+ spec: clang@3.3
+ operating_system: yosemite
+ paths:
+ cc: /path/to/clang
+ cxx: /path/to/clang++
+ f77: None
+ fc: None
+ modules: 'None'
+- compiler:
+ paths:
cc: /path/to/gcc
cxx: /path/to/g++
f77: /path/to/gfortran
fc: /path/to/gfortran
-"""
+ operating_system: CNL10
+ spec: gcc@4.5.0
+ modules: 'None'
+- compiler:
+ paths:
+ cc: /path/to/gcc
+ cxx: /path/to/g++
+ f77: /path/to/gfortran
+ fc: /path/to/gfortran
+ operating_system: SuSE11
+ spec: gcc@4.5.0
+ modules: 'None'
+- compiler:
+ paths:
+ cc: /path/to/gcc
+ cxx: /path/to/g++
+ f77: /path/to/gfortran
+ fc: /path/to/gfortran
+ operating_system: redhat6
+ spec: gcc@4.5.0
+ modules: 'None'
+- compiler:
+ paths:
+ cc: /path/to/gcc
+ cxx: /path/to/g++
+ f77: /path/to/gfortran
+ fc: /path/to/gfortran
+ operating_system: yosemite
+ spec: gcc@4.5.0
+ modules: 'None'
+- compiler:
+ paths:
+ cc: /path/to/gcc
+ cxx: /path/to/g++
+ f77: /path/to/gfortran
+ fc: /path/to/gfortran
+ operating_system: elcapitan
+ spec: gcc@4.5.0
+ modules: 'None'
+- compiler:
+ spec: clang@3.3
+ operating_system: elcapitan
+ paths:
+ cc: /path/to/clang
+ cxx: /path/to/clang++
+ f77: None
+ fc: None
+ modules: 'None'
+""".format(linux_os_name, linux_os_version)
mock_packages_config = """\
packages:
@@ -60,6 +167,10 @@ packages:
paths:
externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang
externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc
+ externalmodule:
+ buildable: False
+ modules:
+ externalmodule@1.0%gcc@4.5.0: external-module
"""
class MockPackagesTest(unittest.TestCase):
diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py
index 56e294de26..582e067860 100644
--- a/lib/spack/spack/test/modules.py
+++ b/lib/spack/spack/test/modules.py
@@ -73,7 +73,7 @@ configuration_alter_environment = {
'all': {
'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
},
- '=x86-linux': {
+ 'platform=test target=x86_64': {
'environment': {'set': {'FOO': 'foo'},
'unset': ['BAR']}
}
@@ -116,6 +116,7 @@ class TclTests(MockPackagesTest):
def get_modulefile_content(self, spec):
spec.concretize()
+ print spec, '&&&&&'
generator = spack.modules.TclModule(spec)
generator.write()
content = FILE_REGISTRY[generator.file_name].split('\n')
@@ -123,27 +124,28 @@ class TclTests(MockPackagesTest):
def test_simple_case(self):
spack.modules.CONFIGURATION = configuration_autoload_direct
- spec = spack.spec.Spec('mpich@3.0.4=x86-linux')
+ spec = spack.spec.Spec('mpich@3.0.4')
content = self.get_modulefile_content(spec)
self.assertTrue('module-whatis "mpich @3.0.4"' in content)
def test_autoload(self):
spack.modules.CONFIGURATION = configuration_autoload_direct
- spec = spack.spec.Spec('mpileaks=x86-linux')
+ spec = spack.spec.Spec('mpileaks')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
self.assertEqual(len([x for x in content if 'module load ' in x]), 2)
spack.modules.CONFIGURATION = configuration_autoload_all
- spec = spack.spec.Spec('mpileaks=x86-linux')
+ spec = spack.spec.Spec('mpileaks')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
self.assertEqual(len([x for x in content if 'module load ' in x]), 5)
def test_alter_environment(self):
spack.modules.CONFIGURATION = configuration_alter_environment
- spec = spack.spec.Spec('mpileaks=x86-linux')
+ spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
content = self.get_modulefile_content(spec)
+ print content
self.assertEqual(
len([x
for x in content
@@ -152,8 +154,9 @@ class TclTests(MockPackagesTest):
len([x for x in content if 'setenv FOO "foo"' in x]), 1)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1)
- spec = spack.spec.Spec('libdwarf=x64-linux')
+ spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32')
content = self.get_modulefile_content(spec)
+ print content
self.assertEqual(
len([x
for x in content
@@ -164,14 +167,14 @@ class TclTests(MockPackagesTest):
def test_blacklist(self):
spack.modules.CONFIGURATION = configuration_blacklist
- spec = spack.spec.Spec('mpileaks=x86-linux')
+ spec = spack.spec.Spec('mpileaks')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
def test_conflicts(self):
spack.modules.CONFIGURATION = configuration_conflicts
- spec = spack.spec.Spec('mpileaks=x86-linux')
+ spec = spack.spec.Spec('mpileaks')
content = self.get_modulefile_content(spec)
self.assertEqual(
len([x for x in content if x.startswith('conflict')]), 2)
diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py
index f653ca3477..034e6b3923 100644
--- a/lib/spack/spack/test/multimethod.py
+++ b/lib/spack/spack/test/multimethod.py
@@ -25,9 +25,13 @@
"""
Test for multi_method dispatch.
"""
+import unittest
import spack
from spack.multimethod import *
+from spack.version import *
+from spack.spec import Spec
+from spack.multimethod import when
from spack.test.mock_packages_test import *
from spack.version import *
@@ -88,21 +92,18 @@ class MultiMethodTest(MockPackagesTest):
self.assertEqual(pkg.has_a_default(), 'default')
- def test_architecture_match(self):
- pkg = spack.repo.get('multimethod=x86_64')
- self.assertEqual(pkg.different_by_architecture(), 'x86_64')
-
- pkg = spack.repo.get('multimethod=ppc64')
- self.assertEqual(pkg.different_by_architecture(), 'ppc64')
-
- pkg = spack.repo.get('multimethod=ppc32')
- self.assertEqual(pkg.different_by_architecture(), 'ppc32')
-
- pkg = spack.repo.get('multimethod=arm64')
- self.assertEqual(pkg.different_by_architecture(), 'arm64')
+ def test_target_match(self):
+ platform = spack.architecture.sys_type()
+ targets = platform.targets.values()
+ for target in targets[:-1]:
+ pkg = spack.repo.get('multimethod target='+target.name)
+ self.assertEqual(pkg.different_by_target(), target.name)
- pkg = spack.repo.get('multimethod=macos')
- self.assertRaises(NoSuchMethodError, pkg.different_by_architecture)
+ pkg = spack.repo.get('multimethod target='+targets[-1].name)
+ if len(targets) == 1:
+ self.assertEqual(pkg.different_by_target(), targets[-1].name)
+ else:
+ self.assertRaises(NoSuchMethodError, pkg.different_by_target)
def test_dependency_match(self):
diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py
new file mode 100644
index 0000000000..ed5f6ff8ad
--- /dev/null
+++ b/lib/spack/spack/test/operating_system.py
@@ -0,0 +1,55 @@
+""" Test checks if the operating_system class is created correctly and that
+the functions are using the correct operating_system. Also checks whether
+the operating_system correctly uses the compiler_strategy
+"""
+
+import unittest
+import os
+import platform
+from spack.platforms.cray_xc import CrayXc
+from spack.platforms.linux import Linux
+from spack.platforms.darwin import Darwin
+from spack.operating_system.linux_distro import LinuxDistro
+from spack.operating_system.mac_os import MacOs
+from spack.operating_system.cnl import ComputeNodeLinux
+
+class TestOperatingSystem(unittest.TestCase):
+
+ def setUp(self):
+ cray_xc = CrayXc()
+ linux = Linux()
+ darwin = Darwin()
+ self.cray_operating_sys = cray_xc.operating_system('front_os')
+ self.cray_default_os = cray_xc.operating_system('default_os')
+ self.cray_back_os = cray_xc.operating_system('back_os')
+ self.darwin_operating_sys = darwin.operating_system('default_os')
+ self.linux_operating_sys = linux.operating_system('default_os')
+
+ def test_cray_front_end_operating_system(self):
+ self.assertIsInstance(self.cray_operating_sys, LinuxDistro)
+
+ def test_cray_front_end_compiler_strategy(self):
+ self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH")
+
+ def test_cray_back_end_operating_system(self):
+ self.assertIsInstance(self.cray_back_os,ComputeNodeLinux)
+
+ def test_cray_back_end_compiler_strategy(self):
+ self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES")
+
+ def test_linux_operating_system(self):
+ self.assertIsInstance(self.linux_operating_sys, LinuxDistro)
+
+ def test_linux_compiler_strategy(self):
+ self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH")
+
+
+ def test_cray_front_end_compiler_list(self):
+ """ Operating systems will now be in charge of finding compilers.
+ So, depending on which operating system you want to build for
+ or which operating system you are on, then you could detect
+ compilers in a certain way. Cray linux environment on the front
+ end is just a regular linux distro whereas the Cray linux compute
+ node is a stripped down version which modules are important
+ """
+ self.assertEquals(True, False)
diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py
index 90382dfc4a..b5ba0ecf35 100644
--- a/lib/spack/spack/test/optional_deps.py
+++ b/lib/spack/spack/test/optional_deps.py
@@ -42,6 +42,13 @@ class ConcretizeTest(MockPackagesTest):
self.check_normalize('optional-dep-test+a',
Spec('optional-dep-test+a', Spec('a')))
+ self.check_normalize('optional-dep-test a=true',
+ Spec('optional-dep-test a=true', Spec('a')))
+
+
+ self.check_normalize('optional-dep-test a=true',
+ Spec('optional-dep-test+a', Spec('a')))
+
self.check_normalize('optional-dep-test@1.1',
Spec('optional-dep-test@1.1', Spec('b')))
diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py
index 4645f98565..712f07ac4d 100644
--- a/lib/spack/spack/test/spec_dag.py
+++ b/lib/spack/spack/test/spec_dag.py
@@ -29,8 +29,11 @@ You can find the dummy packages here::
spack/lib/spack/spack/test/mock_packages
"""
import spack
+import spack.architecture
import spack.package
+from llnl.util.lang import list_modules
+
from spack.spec import Spec
from spack.test.mock_packages_test import *
@@ -239,8 +242,10 @@ class SpecDagTest(MockPackagesTest):
def test_unsatisfiable_architecture(self):
- self.set_pkg_dep('mpileaks', 'mpich=bgqos_0')
- spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf')
+ platform = spack.architecture.sys_type()
+
+ self.set_pkg_dep('mpileaks', 'mpich platform=test target=be')
+ spec = Spec('mpileaks ^mpich platform=test target=fe ^callpath ^dyninst ^libelf ^libdwarf')
self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize)
diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py
index 60eb86d652..9876bfd5a8 100644
--- a/lib/spack/spack/test/spec_semantics.py
+++ b/lib/spack/spack/test/spec_semantics.py
@@ -22,6 +22,8 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+import unittest
+import spack.architecture
from spack.spec import *
from spack.test.mock_packages_test import *
@@ -106,7 +108,8 @@ class SpecSematicsTest(MockPackagesTest):
def test_satisfies_namespaced_dep(self):
- """Ensure spec from same or unspecified namespace satisfies namespace constraint."""
+ """Ensure spec from same or unspecified namespace satisfies namespace
+ constraint."""
self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich')
self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi')
@@ -138,11 +141,16 @@ class SpecSematicsTest(MockPackagesTest):
def test_satisfies_architecture(self):
- self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib')
- self.check_satisfies('foo=bgqos_0', '=bgqos_0')
-
- self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib')
- self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0')
+ platform = spack.architecture.sys_type()
+ self.check_satisfies(
+ 'foo platform=test target=frontend os=frontend',
+ 'platform=test target=frontend os=frontend')
+ self.check_satisfies(
+ 'foo platform=test target=backend os=backend',
+ 'platform=test target=backend', 'platform=test os=backend')
+ self.check_satisfies(
+ 'foo platform=test target=default_target os=default_os',
+ 'platform=test target=default_target os=default_os')
def test_satisfies_dependencies(self):
@@ -157,10 +165,14 @@ class SpecSematicsTest(MockPackagesTest):
self.check_satisfies('mpileaks^mpich@2.0', '^mpich@1:3')
self.check_unsatisfiable('mpileaks^mpich@1.2', '^mpich@2.0')
- self.check_satisfies('mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6')
- self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6')
- self.check_unsatisfiable('mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6')
- self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6')
+ self.check_satisfies(
+ 'mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6')
+ self.check_unsatisfiable(
+ 'mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6')
+ self.check_unsatisfiable(
+ 'mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6')
+ self.check_unsatisfiable(
+ 'mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6')
def test_satisfies_virtual_dependencies(self):
@@ -190,12 +202,20 @@ class SpecSematicsTest(MockPackagesTest):
def test_satisfies_matching_variant(self):
self.check_satisfies('mpich+foo', 'mpich+foo')
self.check_satisfies('mpich~foo', 'mpich~foo')
+ self.check_satisfies('mpich foo=1', 'mpich foo=1')
+
+ #confirm that synonymous syntax works correctly
+ self.check_satisfies('mpich+foo', 'mpich foo=True')
+ self.check_satisfies('mpich foo=true', 'mpich+foo')
+ self.check_satisfies('mpich~foo', 'mpich foo=FALSE')
+ self.check_satisfies('mpich foo=False', 'mpich~foo')
def test_satisfies_unconstrained_variant(self):
# only asked for mpich, no constraints. Either will do.
self.check_satisfies('mpich+foo', 'mpich')
self.check_satisfies('mpich~foo', 'mpich')
+ self.check_satisfies('mpich foo=1', 'mpich')
def test_unsatisfiable_variants(self):
@@ -204,16 +224,44 @@ class SpecSematicsTest(MockPackagesTest):
# 'mpich' is not concrete:
self.check_satisfies('mpich', 'mpich+foo', False)
self.check_satisfies('mpich', 'mpich~foo', False)
+ self.check_satisfies('mpich', 'mpich foo=1', False)
# 'mpich' is concrete:
self.check_unsatisfiable('mpich', 'mpich+foo', True)
self.check_unsatisfiable('mpich', 'mpich~foo', True)
+ self.check_unsatisfiable('mpich', 'mpich foo=1', True)
def test_unsatisfiable_variant_mismatch(self):
# No matchi in specs
self.check_unsatisfiable('mpich~foo', 'mpich+foo')
self.check_unsatisfiable('mpich+foo', 'mpich~foo')
+ self.check_unsatisfiable('mpich foo=1', 'mpich foo=2')
+
+
+ def test_satisfies_matching_compiler_flag(self):
+ self.check_satisfies('mpich cppflags="-O3"', 'mpich cppflags="-O3"')
+ self.check_satisfies('mpich cppflags="-O3 -Wall"', 'mpich cppflags="-O3 -Wall"')
+
+
+ def test_satisfies_unconstrained_compiler_flag(self):
+ # only asked for mpich, no constraints. Any will do.
+ self.check_satisfies('mpich cppflags="-O3"', 'mpich')
+
+
+ def test_unsatisfiable_compiler_flag(self):
+ # This case is different depending on whether the specs are concrete.
+
+ # 'mpich' is not concrete:
+ self.check_satisfies('mpich', 'mpich cppflags="-O3"', False)
+
+ # 'mpich' is concrete:
+ self.check_unsatisfiable('mpich', 'mpich cppflags="-O3"', True)
+
+
+ def test_unsatisfiable_compiler_flag_mismatch(self):
+ # No matchi in specs
+ self.check_unsatisfiable('mpich cppflags="-O3"', 'mpich cppflags="-O2"')
def test_satisfies_virtual(self):
@@ -301,18 +349,29 @@ class SpecSematicsTest(MockPackagesTest):
self.check_constrain('libelf+debug+foo', 'libelf+debug', 'libelf+foo')
self.check_constrain('libelf+debug+foo', 'libelf+debug', 'libelf+debug+foo')
+ self.check_constrain('libelf debug=2 foo=1', 'libelf debug=2', 'libelf foo=1')
+ self.check_constrain('libelf debug=2 foo=1', 'libelf debug=2', 'libelf debug=2 foo=1')
+
self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf~foo')
self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo')
- def test_constrain_arch(self):
- self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0')
- self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0')
+ def test_constrain_compiler_flags(self):
+ self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cppflags="-Wall"')
+ self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cflags="-O3" cppflags="-Wall"')
+
+ def test_constrain_architecture(self):
+ self.check_constrain('libelf target=default_target os=default_os',
+ 'libelf target=default_target os=default_os',
+ 'libelf target=default_target os=default_os')
+ self.check_constrain('libelf target=default_target os=default_os',
+ 'libelf',
+ 'libelf target=default_target os=default_os')
def test_constrain_compiler(self):
- self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0')
- self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0')
+ self.check_constrain('libelf %gcc@4.4.7', 'libelf %gcc@4.4.7', 'libelf %gcc@4.4.7')
+ self.check_constrain('libelf %gcc@4.4.7', 'libelf', 'libelf %gcc@4.4.7')
def test_invalid_constraint(self):
@@ -321,9 +380,11 @@ class SpecSematicsTest(MockPackagesTest):
self.check_invalid_constraint('libelf+debug', 'libelf~debug')
self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo')
+ self.check_invalid_constraint('libelf debug=2', 'libelf debug=1')
- self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54')
-
+ self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"')
+ self.check_invalid_constraint('libelf platform=test target=be os=be',
+ 'libelf target=fe os=fe')
def test_constrain_changed(self):
self.check_constrain_changed('libelf', '@1.0')
@@ -332,7 +393,12 @@ class SpecSematicsTest(MockPackagesTest):
self.check_constrain_changed('libelf%gcc', '%gcc@4.5')
self.check_constrain_changed('libelf', '+debug')
self.check_constrain_changed('libelf', '~debug')
- self.check_constrain_changed('libelf', '=bgqos_0')
+ self.check_constrain_changed('libelf', 'debug=2')
+ self.check_constrain_changed('libelf', 'cppflags="-O3"')
+
+ platform = spack.architecture.sys_type()
+ self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name)
+ self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name)
def test_constrain_not_changed(self):
@@ -343,9 +409,12 @@ class SpecSematicsTest(MockPackagesTest):
self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5')
self.check_constrain_not_changed('libelf+debug', '+debug')
self.check_constrain_not_changed('libelf~debug', '~debug')
- self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0')
- self.check_constrain_not_changed('libelf^foo', 'libelf^foo')
- self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar')
+ self.check_constrain_not_changed('libelf debug=2', 'debug=2')
+ self.check_constrain_not_changed('libelf cppflags="-O3"', 'cppflags="-O3"')
+
+ platform = spack.architecture.sys_type()
+ default_target = platform.target('default_target').name
+ self.check_constrain_not_changed('libelf target='+default_target, 'target='+default_target)
def test_constrain_dependency_changed(self):
@@ -355,7 +424,10 @@ class SpecSematicsTest(MockPackagesTest):
self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5')
self.check_constrain_changed('libelf^foo', 'libelf^foo+debug')
self.check_constrain_changed('libelf^foo', 'libelf^foo~debug')
- self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0')
+
+ platform = spack.architecture.sys_type()
+ default_target = platform.target('default_target').name
+ self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target)
def test_constrain_dependency_not_changed(self):
@@ -365,4 +437,8 @@ class SpecSematicsTest(MockPackagesTest):
self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5')
self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug')
self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug')
- self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0')
+ self.check_constrain_not_changed('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"')
+
+ platform = spack.architecture.sys_type()
+ default_target = platform.target('default_target').name
+ self.check_constrain_not_changed('libelf^foo target='+default_target, 'libelf^foo target='+default_target)
diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py
index 928d111ea9..4a534d7b5c 100644
--- a/lib/spack/spack/test/spec_syntax.py
+++ b/lib/spack/spack/test/spec_syntax.py
@@ -58,7 +58,7 @@ class SpecSyntaxTest(unittest.TestCase):
# ================================================================================
# Parse checks
# ================================================================================
- def check_parse(self, expected, spec=None):
+ def check_parse(self, expected, spec=None, remove_arch=True):
"""Assert that the provided spec is able to be parsed.
If this is called with one argument, it assumes that the string is
canonical (i.e., no spaces and ~ instead of - for variants) and that it
@@ -70,6 +70,7 @@ class SpecSyntaxTest(unittest.TestCase):
if spec is None:
spec = expected
output = spack.spec.parse(spec)
+
parsed = (" ".join(str(spec) for spec in output))
self.assertEqual(expected, parsed)
@@ -104,6 +105,8 @@ class SpecSyntaxTest(unittest.TestCase):
def test_full_specs(self):
self.check_parse("mvapich_foo^_openmpi@1.2:1.4,1.6%intel@12.1+debug~qt_4^stackwalker@8.1_1e")
+ self.check_parse("mvapich_foo^_openmpi@1.2:1.4,1.6%intel@12.1 debug=2~qt_4^stackwalker@8.1_1e")
+ self.check_parse('mvapich_foo^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags="-O3"+debug~qt_4^stackwalker@8.1_1e')
def test_canonicalize(self):
self.check_parse(
@@ -128,7 +131,10 @@ class SpecSyntaxTest(unittest.TestCase):
def test_duplicate_variant(self):
self.assertRaises(DuplicateVariantError, self.check_parse, "x@1.2+debug+debug")
- self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2+debug+debug")
+ self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2+debug debug=true")
+ self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2 debug=false debug=true")
+ self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2 debug=false~debug")
+
def test_duplicate_depdendence(self):
self.assertRaises(DuplicateDependencyError, self.check_parse, "x ^y ^y")
diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py
index a026403e2e..4624f901c8 100644
--- a/lib/spack/spack/test/versions.py
+++ b/lib/spack/spack/test/versions.py
@@ -43,7 +43,6 @@ class VersionsTest(unittest.TestCase):
self.assertFalse(a > b)
self.assertFalse(a >= b)
-
def assert_ver_gt(self, a, b):
a, b = ver(a), ver(b)
self.assertTrue(a > b)
@@ -53,7 +52,6 @@ class VersionsTest(unittest.TestCase):
self.assertFalse(a < b)
self.assertFalse(a <= b)
-
def assert_ver_eq(self, a, b):
a, b = ver(a), ver(b)
self.assertFalse(a > b)
@@ -63,55 +61,43 @@ class VersionsTest(unittest.TestCase):
self.assertFalse(a < b)
self.assertTrue(a <= b)
-
def assert_in(self, needle, haystack):
self.assertTrue(ver(needle) in ver(haystack))
-
def assert_not_in(self, needle, haystack):
self.assertFalse(ver(needle) in ver(haystack))
-
def assert_canonical(self, canonical_list, version_list):
self.assertEqual(ver(canonical_list), ver(version_list))
-
def assert_overlaps(self, v1, v2):
self.assertTrue(ver(v1).overlaps(ver(v2)))
-
def assert_no_overlap(self, v1, v2):
self.assertFalse(ver(v1).overlaps(ver(v2)))
-
def assert_satisfies(self, v1, v2):
self.assertTrue(ver(v1).satisfies(ver(v2)))
-
def assert_does_not_satisfy(self, v1, v2):
self.assertFalse(ver(v1).satisfies(ver(v2)))
-
def check_intersection(self, expected, a, b):
self.assertEqual(ver(expected), ver(a).intersection(ver(b)))
-
def check_union(self, expected, a, b):
self.assertEqual(ver(expected), ver(a).union(ver(b)))
-
def test_two_segments(self):
self.assert_ver_eq('1.0', '1.0')
self.assert_ver_lt('1.0', '2.0')
self.assert_ver_gt('2.0', '1.0')
-
def test_three_segments(self):
self.assert_ver_eq('2.0.1', '2.0.1')
self.assert_ver_lt('2.0', '2.0.1')
self.assert_ver_gt('2.0.1', '2.0')
-
def test_alpha(self):
# TODO: not sure whether I like this. 2.0.1a is *usually*
# TODO: less than 2.0.1, but special-casing it makes version
@@ -120,7 +106,6 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_gt('2.0.1a', '2.0.1')
self.assert_ver_lt('2.0.1', '2.0.1a')
-
def test_patch(self):
self.assert_ver_eq('5.5p1', '5.5p1')
self.assert_ver_lt('5.5p1', '5.5p2')
@@ -129,7 +114,6 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_lt('5.5p1', '5.5p10')
self.assert_ver_gt('5.5p10', '5.5p1')
-
def test_num_alpha_with_no_separator(self):
self.assert_ver_lt('10xyz', '10.1xyz')
self.assert_ver_gt('10.1xyz', '10xyz')
@@ -137,7 +121,6 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_lt('xyz10', 'xyz10.1')
self.assert_ver_gt('xyz10.1', 'xyz10')
-
def test_alpha_with_dots(self):
self.assert_ver_eq('xyz.4', 'xyz.4')
self.assert_ver_lt('xyz.4', '8')
@@ -145,30 +128,25 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_lt('xyz.4', '2')
self.assert_ver_gt('2', 'xyz.4')
-
def test_nums_and_patch(self):
self.assert_ver_lt('5.5p2', '5.6p1')
self.assert_ver_gt('5.6p1', '5.5p2')
self.assert_ver_lt('5.6p1', '6.5p1')
self.assert_ver_gt('6.5p1', '5.6p1')
-
def test_rc_versions(self):
self.assert_ver_gt('6.0.rc1', '6.0')
self.assert_ver_lt('6.0', '6.0.rc1')
-
def test_alpha_beta(self):
self.assert_ver_gt('10b2', '10a1')
self.assert_ver_lt('10a2', '10b2')
-
def test_double_alpha(self):
self.assert_ver_eq('1.0aa', '1.0aa')
self.assert_ver_lt('1.0a', '1.0aa')
self.assert_ver_gt('1.0aa', '1.0a')
-
def test_padded_numbers(self):
self.assert_ver_eq('10.0001', '10.0001')
self.assert_ver_eq('10.0001', '10.1')
@@ -176,24 +154,20 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_lt('10.0001', '10.0039')
self.assert_ver_gt('10.0039', '10.0001')
-
def test_close_numbers(self):
self.assert_ver_lt('4.999.9', '5.0')
self.assert_ver_gt('5.0', '4.999.9')
-
def test_date_stamps(self):
self.assert_ver_eq('20101121', '20101121')
self.assert_ver_lt('20101121', '20101122')
self.assert_ver_gt('20101122', '20101121')
-
def test_underscores(self):
self.assert_ver_eq('2_0', '2_0')
self.assert_ver_eq('2.0', '2_0')
self.assert_ver_eq('2_0', '2.0')
-
def test_rpm_oddities(self):
self.assert_ver_eq('1b.fc17', '1b.fc17')
self.assert_ver_lt('1b.fc17', '1.fc17')
@@ -202,7 +176,6 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_gt('1g.fc17', '1.fc17')
self.assert_ver_lt('1.fc17', '1g.fc17')
-
# Stuff below here is not taken from RPM's tests and is
# unique to spack
def test_version_ranges(self):
@@ -214,7 +187,6 @@ class VersionsTest(unittest.TestCase):
self.assert_ver_lt('1.2:1.4', '1.5:1.6')
self.assert_ver_gt('1.5:1.6', '1.2:1.4')
-
def test_contains(self):
self.assert_in('1.3', '1.2:1.4')
self.assert_in('1.2.5', '1.2:1.4')
@@ -233,7 +205,6 @@ class VersionsTest(unittest.TestCase):
self.assert_in('1.4.1', '1.2.7:1.4')
self.assert_not_in('1.4.1', '1.2.7:1.4.0')
-
def test_in_list(self):
self.assert_in('1.2', ['1.5', '1.2', '1.3'])
self.assert_in('1.2.5', ['1.5', '1.2:1.3'])
@@ -245,7 +216,6 @@ class VersionsTest(unittest.TestCase):
self.assert_not_in('1.2.5:1.5', ['1.5', '1.2:1.3'])
self.assert_not_in('1.1:1.2.5', ['1.5', '1.2:1.3'])
-
def test_ranges_overlap(self):
self.assert_overlaps('1.2', '1.2')
self.assert_overlaps('1.2.1', '1.2.1')
@@ -262,7 +232,6 @@ class VersionsTest(unittest.TestCase):
self.assert_overlaps(':', '1.6:1.9')
self.assert_overlaps('1.6:1.9', ':')
-
def test_overlap_with_containment(self):
self.assert_in('1.6.5', '1.6')
self.assert_in('1.6.5', ':1.6')
@@ -273,7 +242,6 @@ class VersionsTest(unittest.TestCase):
self.assert_not_in(':1.6', '1.6.5')
self.assert_in('1.6.5', ':1.6')
-
def test_lists_overlap(self):
self.assert_overlaps('1.2b:1.7,5', '1.6:1.9,1')
self.assert_overlaps('1,2,3,4,5', '3,4,5,6,7')
@@ -287,7 +255,6 @@ class VersionsTest(unittest.TestCase):
self.assert_no_overlap('1,2,3,4,5', '6,7')
self.assert_no_overlap('1,2,3,4,5', '6:7')
-
def test_canonicalize_list(self):
self.assert_canonical(['1.2', '1.3', '1.4'],
['1.2', '1.3', '1.3', '1.4'])
@@ -316,7 +283,6 @@ class VersionsTest(unittest.TestCase):
self.assert_canonical([':'],
[':,1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4'])
-
def test_intersection(self):
self.check_intersection('2.5',
'1.0:2.5', '2.5:3.0')
@@ -325,12 +291,11 @@ class VersionsTest(unittest.TestCase):
self.check_intersection('0:1', ':', '0:1')
self.check_intersection(['1.0', '2.5:2.7'],
- ['1.0:2.7'], ['2.5:3.0','1.0'])
+ ['1.0:2.7'], ['2.5:3.0', '1.0'])
self.check_intersection(['2.5:2.7'],
- ['1.1:2.7'], ['2.5:3.0','1.0'])
+ ['1.1:2.7'], ['2.5:3.0', '1.0'])
self.check_intersection(['0:1'], [':'], ['0:1'])
-
def test_intersect_with_containment(self):
self.check_intersection('1.6.5', '1.6.5', ':1.6')
self.check_intersection('1.6.5', ':1.6', '1.6.5')
@@ -338,7 +303,6 @@ class VersionsTest(unittest.TestCase):
self.check_intersection('1.6:1.6.5', ':1.6.5', '1.6')
self.check_intersection('1.6:1.6.5', '1.6', ':1.6.5')
-
def test_union_with_containment(self):
self.check_union(':1.6', '1.6.5', ':1.6')
self.check_union(':1.6', ':1.6', '1.6.5')
@@ -346,8 +310,6 @@ class VersionsTest(unittest.TestCase):
self.check_union(':1.6', ':1.6.5', '1.6')
self.check_union(':1.6', '1.6', ':1.6.5')
-
- def test_union_with_containment(self):
self.check_union(':', '1.0:', ':2.0')
self.check_union('1:4', '1:3', '2:4')
@@ -356,7 +318,6 @@ class VersionsTest(unittest.TestCase):
# Tests successor/predecessor case.
self.check_union('1:4', '1:2', '3:4')
-
def test_basic_version_satisfaction(self):
self.assert_satisfies('4.7.3', '4.7.3')
@@ -372,7 +333,6 @@ class VersionsTest(unittest.TestCase):
self.assert_does_not_satisfy('4.8', '4.9')
self.assert_does_not_satisfy('4', '4.9')
-
def test_basic_version_satisfaction_in_lists(self):
self.assert_satisfies(['4.7.3'], ['4.7.3'])
@@ -388,7 +348,6 @@ class VersionsTest(unittest.TestCase):
self.assert_does_not_satisfy(['4.8'], ['4.9'])
self.assert_does_not_satisfy(['4'], ['4.9'])
-
def test_version_range_satisfaction(self):
self.assert_satisfies('4.7b6', '4.3:4.7')
self.assert_satisfies('4.3.0', '4.3:4.7')
@@ -400,7 +359,6 @@ class VersionsTest(unittest.TestCase):
self.assert_satisfies('4.7b6', '4.3:4.7')
self.assert_does_not_satisfy('4.8.0', '4.3:4.7')
-
def test_version_range_satisfaction_in_lists(self):
self.assert_satisfies(['4.7b6'], ['4.3:4.7'])
self.assert_satisfies(['4.3.0'], ['4.3:4.7'])
@@ -423,3 +381,11 @@ class VersionsTest(unittest.TestCase):
self.assert_satisfies('4.8.0', '4.2, 4.3:4.8')
self.assert_satisfies('4.8.2', '4.2, 4.3:4.8')
+
+ def test_formatted_strings(self):
+ versions = '1.2.3', '1_2_3', '1-2-3'
+ for item in versions:
+ v = Version(item)
+ self.assertEqual(v.dotted, '1.2.3')
+ self.assertEqual(v.dashed, '1-2-3')
+ self.assertEqual(v.underscored, '1_2_3')
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index d2ccfde69b..14b56e8d6c 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -22,10 +22,8 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-__all__ = ['Executable', 'which', 'ProcessError']
import os
-import sys
import re
import subprocess
import inspect
@@ -34,9 +32,12 @@ import llnl.util.tty as tty
import spack
import spack.error
+__all__ = ['Executable', 'which', 'ProcessError']
+
class Executable(object):
"""Class representing a program that can be run on the command line."""
+
def __init__(self, name):
self.exe = name.split(' ')
self.returncode = None
@@ -44,16 +45,13 @@ class Executable(object):
if not self.exe:
raise ProcessError("Cannot construct executable for '%s'" % name)
-
def add_default_arg(self, arg):
self.exe.append(arg)
-
@property
def command(self):
return ' '.join(self.exe)
-
def __call__(self, *args, **kwargs):
"""Run this executable in a subprocess.
@@ -105,6 +103,8 @@ class Executable(object):
fail_on_error = kwargs.pop("fail_on_error", True)
ignore_errors = kwargs.pop("ignore_errors", ())
+ env = kwargs.get('env', None)
+
# TODO: This is deprecated. Remove in a future version.
return_output = kwargs.pop("return_output", False)
@@ -114,8 +114,8 @@ class Executable(object):
else:
output = kwargs.pop("output", None)
- error = kwargs.pop("error", None)
- input = kwargs.pop("input", None)
+ error = kwargs.pop("error", None)
+ input = kwargs.pop("input", None)
if input is str:
raise ValueError("Cannot use `str` as input stream.")
@@ -126,85 +126,91 @@ class Executable(object):
return subprocess.PIPE, False
else:
return arg, False
+
ostream, close_ostream = streamify(output, 'w')
- estream, close_estream = streamify(error, 'w')
- istream, close_istream = streamify(input, 'r')
+ estream, close_estream = streamify(error, 'w')
+ istream, close_istream = streamify(input, 'r')
# if they just want to ignore one error code, make it a tuple.
if isinstance(ignore_errors, int):
- ignore_errors = (ignore_errors,)
+ ignore_errors = (ignore_errors, )
quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
if quoted_args:
- tty.warn("Quotes in command arguments can confuse scripts like configure.",
- "The following arguments may cause problems when executed:",
- str("\n".join([" "+arg for arg in quoted_args])),
- "Quotes aren't needed because spack doesn't use a shell.",
- "Consider removing them")
+ tty.warn(
+ "Quotes in command arguments can confuse scripts like"
+ " configure.",
+ "The following arguments may cause problems when executed:",
+ str("\n".join([" " + arg for arg in quoted_args])),
+ "Quotes aren't needed because spack doesn't use a shell.",
+ "Consider removing them")
cmd = self.exe + list(args)
- cmd_line = "'%s'" % "' '".join(map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
+ cmd_line = "'%s'" % "' '".join(
+ map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
tty.debug(cmd_line)
try:
proc = subprocess.Popen(
- cmd, stdin=istream, stderr=estream, stdout=ostream)
+ cmd,
+ stdin=istream,
+ stderr=estream,
+ stdout=ostream,
+ env=env)
out, err = proc.communicate()
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):
- raise ProcessError("Command exited with status %d:"
- % proc.returncode, cmd_line)
+ raise ProcessError("Command exited with status %d:" %
+ proc.returncode, cmd_line)
+
if output is str or error is str:
result = ''
- if output is str: result += out
- if error is str: result += err
+ if output is str:
+ result += out
+ if error is str:
+ result += err
return result
except OSError, e:
raise ProcessError(
- "%s: %s" % (self.exe[0], e.strerror),
- "Command: " + cmd_line)
+ "%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line)
except subprocess.CalledProcessError, e:
if fail_on_error:
raise ProcessError(
- str(e),
- "\nExit status %d when invoking command: %s"
- % (proc.returncode, cmd_line))
+ str(e), "\nExit status %d when invoking command: %s" %
+ (proc.returncode, cmd_line))
finally:
- if close_ostream: output.close()
- if close_estream: error.close()
- if close_istream: input.close()
-
+ if close_ostream:
+ output.close()
+ if close_estream:
+ error.close()
+ if close_istream:
+ input.close()
def __eq__(self, other):
return self.exe == other.exe
-
def __neq__(self, other):
return not (self == other)
-
def __hash__(self):
- return hash((type(self),) + tuple(self.exe))
-
+ return hash((type(self), ) + tuple(self.exe))
def __repr__(self):
return "<exe: %s>" % self.exe
-
def __str__(self):
return ' '.join(self.exe)
-
def which(name, **kwargs):
"""Finds an executable in the path like command-line which."""
- path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
+ path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
required = kwargs.get('required', False)
if not path:
@@ -233,14 +239,16 @@ class ProcessError(spack.error.SpackError):
@property
def long_message(self):
msg = self._long_message
- if msg: msg += "\n\n"
+ if msg:
+ msg += "\n\n"
if self.build_log:
msg += "See build log for details:\n"
msg += " %s" % self.build_log
if self.package_context:
- if msg: msg += "\n\n"
+ if msg:
+ msg += "\n\n"
msg += '\n'.join(self.package_context)
return msg
@@ -267,7 +275,7 @@ def _get_package_context():
frame = f[0]
# Find a frame with 'self' in the local variables.
- if not 'self' in frame.f_locals:
+ if 'self' not in frame.f_locals:
continue
# Look only at a frame in a subclass of spack.Package
@@ -280,9 +288,8 @@ def _get_package_context():
# Build a message showing where in install we failed.
lines.append("%s:%d, in %s:" % (
- inspect.getfile(frame.f_code),
- frame.f_lineno,
- frame.f_code.co_name))
+ inspect.getfile(frame.f_code), frame.f_lineno, frame.f_code.co_name
+ ))
sourcelines, start = inspect.getsourcelines(frame)
for i, line in enumerate(sourcelines):
diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py
index 20686d44b2..ad875f5ef5 100644
--- a/lib/spack/spack/variant.py
+++ b/lib/spack/spack/variant.py
@@ -32,5 +32,5 @@ currently variants are just flags.
class Variant(object):
"""Represents a variant on a build. Can be either on or off."""
def __init__(self, default, description):
- self.default = bool(default)
+ self.default = default
self.description = str(description)
diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py
index 247f6d2362..858d581472 100644
--- a/lib/spack/spack/version.py
+++ b/lib/spack/spack/version.py
@@ -43,16 +43,16 @@ be called on any of the types::
intersection
concrete
"""
-import os
-import sys
import re
from bisect import bisect_left
from functools import wraps
+
from functools_backport import total_ordering
# Valid version characters
VALID_VERSION = r'[A-Za-z0-9_.-]'
+
def int_if_int(string):
"""Convert a string to int if possible. Otherwise, return a string."""
try:
@@ -62,10 +62,11 @@ def int_if_int(string):
def coerce_versions(a, b):
- """Convert both a and b to the 'greatest' type between them, in this order:
+ """
+ Convert both a and b to the 'greatest' type between them, in this order:
Version < VersionRange < VersionList
- This is used to simplify comparison operations below so that we're always
- comparing things that are of the same type.
+ This is used to simplify comparison operations below so that we're always
+ comparing things that are of the same type.
"""
order = (Version, VersionRange, VersionList)
ta, tb = type(a), type(b)
@@ -105,6 +106,7 @@ def coerced(method):
@total_ordering
class Version(object):
"""Class to represent versions"""
+
def __init__(self, string):
string = str(string)
@@ -124,6 +126,17 @@ class Version(object):
# last element of separators is ''
self.separators = tuple(re.split(segment_regex, string)[1:-1])
+ @property
+ def dotted(self):
+ return '.'.join(str(x) for x in self.version)
+
+ @property
+ def underscored(self):
+ return '_'.join(str(x) for x in self.version)
+
+ @property
+ def dashed(self):
+ return '-'.join(str(x) for x in self.version)
def up_to(self, index):
"""Return a version string up to the specified component, exclusive.
@@ -131,15 +144,12 @@ class Version(object):
"""
return '.'.join(str(x) for x in self[:index])
-
def lowest(self):
return self
-
def highest(self):
return self
-
@coerced
def satisfies(self, other):
"""A Version 'satisfies' another if it is at least as specific and has a
@@ -147,11 +157,10 @@ class Version(object):
gcc@4.7 so that when a user asks to build with gcc@4.7, we can find
a suitable compiler.
"""
- nself = len(self.version)
+ nself = len(self.version)
nother = len(other.version)
return nother <= nself and self.version[:nother] == other.version
-
def wildcard(self):
"""Create a regex that will match variants of this version string."""
def a_or_n(seg):
@@ -181,28 +190,22 @@ class Version(object):
wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1)
return wc
-
def __iter__(self):
return iter(self.version)
-
def __getitem__(self, idx):
return tuple(self.version[idx])
-
def __repr__(self):
return self.string
-
def __str__(self):
return self.string
-
@property
def concrete(self):
return self
-
@coerced
def __lt__(self, other):
"""Version comparison is designed for consistency with the way RPM
@@ -235,28 +238,23 @@ class Version(object):
# If the common prefix is equal, the one with more segments is bigger.
return len(self.version) < len(other.version)
-
@coerced
def __eq__(self, other):
return (other is not None and
type(other) == Version and self.version == other.version)
-
def __ne__(self, other):
return not (self == other)
-
def __hash__(self):
return hash(self.version)
-
@coerced
def __contains__(self, other):
if other is None:
return False
return other.version[:len(self.version)] == self.version
-
def is_predecessor(self, other):
"""True if the other version is the immediate predecessor of this one.
That is, NO versions v exist such that:
@@ -269,16 +267,13 @@ class Version(object):
ol = other.version[-1]
return type(sl) == int and type(ol) == int and (ol - sl == 1)
-
def is_successor(self, other):
return other.is_predecessor(self)
-
@coerced
def overlaps(self, other):
return self in other or other in self
-
@coerced
def union(self, other):
if self == other or other in self:
@@ -288,7 +283,6 @@ class Version(object):
else:
return VersionList([self, other])
-
@coerced
def intersection(self, other):
if self == other:
@@ -299,6 +293,7 @@ class Version(object):
@total_ordering
class VersionRange(object):
+
def __init__(self, start, end):
if isinstance(start, basestring):
start = Version(start)
@@ -310,15 +305,12 @@ class VersionRange(object):
if start and end and end < start:
raise ValueError("Invalid Version range: %s" % self)
-
def lowest(self):
return self.start
-
def highest(self):
return self.end
-
@coerced
def __lt__(self, other):
"""Sort VersionRanges lexicographically so that they are ordered first
@@ -331,28 +323,24 @@ class VersionRange(object):
s, o = self, other
if s.start != o.start:
- return s.start is None or (o.start is not None and s.start < o.start)
+ return s.start is None or (o.start is not None and s.start < o.start) # NOQA: ignore=E501
return (s.end != o.end and
o.end is None or (s.end is not None and s.end < o.end))
-
@coerced
def __eq__(self, other):
return (other is not None and
type(other) == VersionRange and
self.start == other.start and self.end == other.end)
-
def __ne__(self, other):
return not (self == other)
-
@property
def concrete(self):
return self.start if self.start == self.end else None
-
@coerced
def __contains__(self, other):
if other is None:
@@ -373,57 +361,55 @@ class VersionRange(object):
other.end in self.end)))
return in_upper
-
@coerced
def satisfies(self, other):
- """A VersionRange satisfies another if some version in this range
- would satisfy some version in the other range. To do this it must
- either:
- a) Overlap with the other range
- b) The start of this range satisfies the end of the other range.
-
- This is essentially the same as overlaps(), but overlaps assumes
- that its arguments are specific. That is, 4.7 is interpreted as
- 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied
- by 4.7.3.5, etc.
-
- Rationale:
- If a user asks for gcc@4.5:4.7, and a package is only compatible with
- gcc@4.7.3:4.8, then that package should be able to build under the
- constraints. Just using overlaps() would not work here.
-
- Note that we don't need to check whether the end of this range
- would satisfy the start of the other range, because overlaps()
- already covers that case.
-
- Note further that overlaps() is a symmetric operation, while
- satisfies() is not.
+ """
+ A VersionRange satisfies another if some version in this range
+ would satisfy some version in the other range. To do this it must
+ either:
+ a) Overlap with the other range
+ b) The start of this range satisfies the end of the other range.
+
+ This is essentially the same as overlaps(), but overlaps assumes
+ that its arguments are specific. That is, 4.7 is interpreted as
+ 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied
+ by 4.7.3.5, etc.
+
+ Rationale:
+ If a user asks for gcc@4.5:4.7, and a package is only compatible with
+ gcc@4.7.3:4.8, then that package should be able to build under the
+ constraints. Just using overlaps() would not work here.
+
+ Note that we don't need to check whether the end of this range
+ would satisfy the start of the other range, because overlaps()
+ already covers that case.
+
+ Note further that overlaps() is a symmetric operation, while
+ satisfies() is not.
"""
return (self.overlaps(other) or
# if either self.start or other.end are None, then this can't
# satisfy, or overlaps() would've taken care of it.
self.start and other.end and self.start.satisfies(other.end))
-
@coerced
def overlaps(self, other):
- return ((self.start == None or other.end is None or
+ return ((self.start is None or other.end is None or
self.start <= other.end or
other.end in self.start or self.start in other.end) and
- (other.start is None or self.end == None or
+ (other.start is None or self.end is None or
other.start <= self.end or
other.start in self.end or self.end in other.start))
-
@coerced
def union(self, other):
if not self.overlaps(other):
if (self.end is not None and other.start is not None and
- self.end.is_predecessor(other.start)):
+ self.end.is_predecessor(other.start)):
return VersionRange(self.start, other.end)
if (other.end is not None and self.start is not None and
- other.end.is_predecessor(self.start)):
+ other.end.is_predecessor(self.start)):
return VersionRange(other.start, self.end)
return VersionList([self, other])
@@ -442,13 +428,12 @@ class VersionRange(object):
else:
end = self.end
# TODO: See note in intersection() about < and in discrepancy.
- if not other.end in self.end:
+ if other.end not in self.end:
if end in other.end or other.end > self.end:
end = other.end
return VersionRange(start, end)
-
@coerced
def intersection(self, other):
if self.overlaps(other):
@@ -470,7 +455,7 @@ class VersionRange(object):
# 1.6 < 1.6.5 = True (lexicographic)
# Should 1.6 NOT be less than 1.6.5? Hm.
# Here we test (not end in other.end) first to avoid paradox.
- if other.end is not None and not end in other.end:
+ if other.end is not None and end not in other.end:
if other.end < end or other.end in end:
end = other.end
@@ -479,15 +464,12 @@ class VersionRange(object):
else:
return VersionList()
-
def __hash__(self):
return hash((self.start, self.end))
-
def __repr__(self):
return self.__str__()
-
def __str__(self):
out = ""
if self.start:
@@ -501,6 +483,7 @@ class VersionRange(object):
@total_ordering
class VersionList(object):
"""Sorted, non-redundant list of Versions and VersionRanges."""
+
def __init__(self, vlist=None):
self.versions = []
if vlist is not None:
@@ -515,7 +498,6 @@ class VersionList(object):
for v in vlist:
self.add(ver(v))
-
def add(self, version):
if type(version) in (Version, VersionRange):
# This normalizes single-value version ranges.
@@ -524,9 +506,9 @@ class VersionList(object):
i = bisect_left(self, version)
- while i-1 >= 0 and version.overlaps(self[i-1]):
- version = version.union(self[i-1])
- del self.versions[i-1]
+ while i - 1 >= 0 and version.overlaps(self[i - 1]):
+ version = version.union(self[i - 1])
+ del self.versions[i - 1]
i -= 1
while i < len(self) and version.overlaps(self[i]):
@@ -542,7 +524,6 @@ class VersionList(object):
else:
raise TypeError("Can't add %s to VersionList" % type(version))
-
@property
def concrete(self):
if len(self) == 1:
@@ -550,11 +531,9 @@ class VersionList(object):
else:
return None
-
def copy(self):
return VersionList(self)
-
def lowest(self):
"""Get the lowest version in the list."""
if not self:
@@ -562,7 +541,6 @@ class VersionList(object):
else:
return self[0].lowest()
-
def highest(self):
"""Get the highest version in the list."""
if not self:
@@ -570,7 +548,6 @@ class VersionList(object):
else:
return self[-1].highest()
-
@coerced
def overlaps(self, other):
if not other or not self:
@@ -586,14 +563,12 @@ class VersionList(object):
o += 1
return False
-
def to_dict(self):
"""Generate human-readable dict for YAML."""
if self.concrete:
- return { 'version' : str(self[0]) }
+ return {'version': str(self[0])}
else:
- return { 'versions' : [str(v) for v in self] }
-
+ return {'versions': [str(v) for v in self]}
@staticmethod
def from_dict(dictionary):
@@ -605,7 +580,6 @@ class VersionList(object):
else:
raise ValueError("Dict must have 'version' or 'versions' in it.")
-
@coerced
def satisfies(self, other, strict=False):
"""A VersionList satisfies another if some version in the list
@@ -633,20 +607,17 @@ class VersionList(object):
o += 1
return False
-
@coerced
def update(self, other):
for v in other.versions:
self.add(v)
-
@coerced
def union(self, other):
result = self.copy()
result.update(other)
return result
-
@coerced
def intersection(self, other):
# TODO: make this faster. This is O(n^2).
@@ -656,7 +627,6 @@ class VersionList(object):
result.add(s.intersection(o))
return result
-
@coerced
def intersect(self, other):
"""Intersect this spec's list with other.
@@ -678,50 +648,40 @@ class VersionList(object):
if i == 0:
if version not in self[0]:
return False
- elif all(version not in v for v in self[i-1:]):
+ elif all(version not in v for v in self[i - 1:]):
return False
return True
-
def __getitem__(self, index):
return self.versions[index]
-
def __iter__(self):
return iter(self.versions)
-
def __reversed__(self):
return reversed(self.versions)
-
def __len__(self):
return len(self.versions)
-
@coerced
def __eq__(self, other):
return other is not None and self.versions == other.versions
-
def __ne__(self, other):
return not (self == other)
-
@coerced
def __lt__(self, other):
return other is not None and self.versions < other.versions
-
def __hash__(self):
return hash(tuple(self.versions))
-
def __str__(self):
return ",".join(str(v) for v in self.versions)
-
def __repr__(self):
return str(self.versions)
@@ -730,7 +690,7 @@ def _string_to_version(string):
"""Converts a string to a Version, VersionList, or VersionRange.
This is private. Client code should use ver().
"""
- string = string.replace(' ','')
+ string = string.replace(' ', '')
if ',' in string:
return VersionList(string.split(','))
@@ -738,7 +698,7 @@ def _string_to_version(string):
elif ':' in string:
s, e = string.split(':')
start = Version(s) if s else None
- end = Version(e) if e else None
+ end = Version(e) if e else None
return VersionRange(start, end)
else:
diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py
index 91ad77c8fd..bb8333f023 100644
--- a/lib/spack/spack/virtual.py
+++ b/lib/spack/spack/virtual.py
@@ -67,10 +67,15 @@ class ProviderIndex(object):
if type(spec) != spack.spec.Spec:
spec = spack.spec.Spec(spec)
+ if not spec.name:
+ # Empty specs do not have a package
+ return
+
assert(not spec.virtual)
pkg = spec.package
for provided_spec, provider_spec in pkg.provided.iteritems():
+ provider_spec.compiler_flags = spec.compiler_flags.copy()#We want satisfaction other than flags
if provider_spec.satisfies(spec, deps=False):
provided_name = provided_spec.name
diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py
new file mode 100644
index 0000000000..c2d084d6c3
--- /dev/null
+++ b/lib/spack/spack/yaml_version_check.py
@@ -0,0 +1,55 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Yaml Version Check is a module for ensuring that config file
+formats are compatible with the current version of Spack."""
+import os.path
+import os
+import llnl.util.tty as tty
+import spack.util.spack_yaml as syaml
+import spack.config
+
+
+def check_yaml_versions():
+ check_compiler_yaml_version()
+
+def check_compiler_yaml_version():
+ config_scopes = spack.config.config_scopes
+ for scope in config_scopes.values():
+ file_name = os.path.join(scope.path, 'compilers.yaml')
+ data = None
+ if os.path.isfile(file_name):
+ with open(file_name) as f:
+ data = syaml.load(f)
+
+ if data:
+ compilers = data['compilers']
+ if len(compilers) > 0:
+ if (not isinstance(compilers, list)) or 'operating_system' not in compilers[0]['compiler']:
+ new_file = os.path.join(scope.path, '_old_compilers.yaml')
+ tty.warn('%s in out of date compilers format. '
+ 'Moved to %s. Spack automatically generate '
+ 'a compilers config file '
+ % (file_name, new_file))
+ os.rename(file_name, new_file)