diff options
188 files changed, 9240 insertions, 2462 deletions
@@ -8,6 +8,9 @@ # - E221: multiple spaces before operator # - E241: multiple spaces after ‘,’ # +# Let people use terse Python features: +# - E731 : lambda expressions +# # Spack allows wildcard imports: # - F403: disable wildcard import # @@ -16,5 +19,5 @@ # - F999: name name be undefined or undefined from star imports. # [flake8] -ignore = E221,E241,F403,F821,F999 +ignore = E221,E241,E731,F403,F821,F999,F405 max-line-length = 79 @@ -138,6 +138,9 @@ def main(): import spack.util.debug as debug debug.register_interrupt_handler() + from spack.yaml_version_check import check_yaml_versions + check_yaml_versions() + spack.spack_working_dir = working_dir if args.mock: from spack.repository import RepoPath diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index 99be5e7b6d..5f31cdd5c0 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -22,8 +22,8 @@ modules: include: - CPATH lib/pkgconfig: - - PKGCONFIG + - PKG_CONFIG_PATH lib64/pkgconfig: - - PKGCONFIG + - PKG_CONFIG_PATH '': - CMAKE_PREFIX_PATH diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 15db2f7a16..50c48b802b 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -102,8 +102,8 @@ that the packages is installed: ==> adept-utils is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da. ==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz ######################################################################## 100.0% - ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz - ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23. + ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz + ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23. ==> No patches needed for mpileaks. ==> Building mpileaks. @@ -132,10 +132,10 @@ sites, as installing a version that one user needs will not disrupt existing installations for other users. In addition to different versions, Spack can customize the compiler, -compile-time options (variants), and platform (for cross compiles) of -an installation. Spack is unique in that it can also configure the -*dependencies* a package is built with. For example, two -configurations of the same version of a package, one built with boost +compile-time options (variants), compiler flags, and platform (for +cross compiles) of an installation. Spack is unique in that it can +also configure the *dependencies* a package is built with. For example, +two configurations of the same version of a package, one built with boost 1.39.0, and the other version built with version 1.43.0, can coexist. This can all be done on the command line using the *spec* syntax. @@ -246,6 +246,12 @@ Packages are divided into groups according to their architecture and compiler. Within each group, Spack tries to keep the view simple, and only shows the version of installed packages. +``spack find`` can filter the package list based on the package name, spec, or +a number of properties of their installation status. For example, missing +dependencies of a spec can be shown with ``-m``, packages which were +explicitly installed with ``spack install <package>`` can be singled out with +``-e`` and those which have been pulled in only as dependencies with ``-E``. + In some cases, there may be different configurations of the *same* version of a package installed. For example, there are two installations of of ``libdwarf@20130729`` above. We can look at them @@ -328,9 +334,15 @@ of libelf would look like this: -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 +We can also search for packages that have a certain attribute. For example, +``spack find -l libdwarf +debug`` will show only installations of libdwarf +with the 'debug' compile-time option enabled, while ``spack find -l +debug`` +will find every installed package with a 'debug' compile-time option enabled. + The full spec syntax is discussed in detail in :ref:`sec-specs`. + Compiler configuration ----------------------------------- @@ -457,6 +469,26 @@ For compilers, like ``clang``, that do not support Fortran, put Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. +You can also add compiler flags to manually configured compilers. The +valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, +``ldflags``, and ``ldlibs``. For example,:: + + ... + chaos_5_x86_64_ib: + ... + intel@15.0.0: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + cppflags: -O3 -fPIC + ... + +These flags will be treated by spack as if they were enterred from +the command line each time this compiler is used. The compiler wrappers +then inject those flags into the compiler command. Compiler flags +enterred from the command line will be discussed in more detail in the +following section. .. _sec-specs: @@ -474,7 +506,7 @@ the full syntax of specs. Here is an example of a much longer spec than we've seen thus far:: - mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt =bgqos_0 ^callpath @1.1 %gcc@4.7.2 + mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt arch=bgq_os ^callpath @1.1 %gcc@4.7.2 If provided to ``spack install``, this will install the ``mpileaks`` library at some version between ``1.2`` and ``1.4`` (inclusive), @@ -492,8 +524,12 @@ More formally, a spec consists of the following pieces: * ``%`` Optional compiler specifier, with an optional compiler version (``gcc`` or ``gcc@4.7.3``) * ``+`` or ``-`` or ``~`` Optional variant specifiers (``+debug``, - ``-qt``, or ``~qt``) -* ``=`` Optional architecture specifier (``bgqos_0``) + ``-qt``, or ``~qt``) for boolean variants +* ``name=<value>`` Optional variant specifiers that are not restricted to +boolean variants +* ``name=<value>`` Optional compiler flag specifiers. Valid flag names are +``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``. +* ``arch=<value>`` Optional architecture specifier (``arch=bgq_os``) * ``^`` Dependency specs (``^callpath@1.1``) There are two things to notice here. The first is that specs are @@ -573,7 +609,7 @@ compilers, variants, and architectures just like any other spec. Specifiers are associated with the nearest package name to their left. For example, above, ``@1.1`` and ``%gcc@4.7.2`` associates with the ``callpath`` package, while ``@1.2:1.4``, ``%gcc@4.7.5``, ``+debug``, -``-qt``, and ``=bgqos_0`` all associate with the ``mpileaks`` package. +``-qt``, and ``arch=bgq_os`` all associate with the ``mpileaks`` package. In the diagram above, ``mpileaks`` depends on ``mpich`` with an unspecified version, but packages can depend on other packages with @@ -629,22 +665,25 @@ based on site policies. Variants ~~~~~~~~~~~~~~~~~~~~~~~ -.. Note:: - - Variants are not yet supported, but will be in the next Spack - release (0.9), due in Q2 2015. - -Variants are named options associated with a particular package, and -they can be turned on or off. For example, above, supplying -``+debug`` causes ``mpileaks`` to be built with debug flags. The -names of particular variants available for a package depend on what -was provided by the package author. ``spack info <package>`` will +Variants are named options associated with a particular package. They are +optional, as each package must provide default values for each variant it +makes available. Variants can be specified using +a flexible parameter syntax ``name=<value>``. For example, +``spack install libelf debug=True`` will install libelf build with debug +flags. The names of particular variants available for a package depend on +what was provided by the package author. ``spack into <package>`` will provide information on what build variants are available. -Depending on the package a variant may be on or off by default. For -``mpileaks`` here, ``debug`` is off by default, and we turned it on -with ``+debug``. If a package is on by default you can turn it off by -either adding ``-name`` or ``~name`` to the spec. +For compatibility with earlier versions, variants which happen to be +boolean in nature can be specified by a syntax that represents turning +options on and off. For example, in the previous spec we could have +supplied ``libelf +debug`` with the same effect of enabling the debug +compile time option for the libelf package. + +Depending on the package a variant may have any default value. For +``libelf`` here, ``debug`` is ``False`` by default, and we turned it on +with ``debug=True`` or ``+debug``. If a package is ``True`` by default +you can turn it off by either adding ``-name`` or ``~name`` to the spec. There are two syntaxes here because, depending on context, ``~`` and ``-`` may mean different things. In most shells, the following will @@ -656,7 +695,7 @@ result in the shell performing home directory substitution: mpileaks~debug # use this instead If there is a user called ``debug``, the ``~`` will be incorrectly -expanded. In this situation, you would want to write ``mpileaks +expanded. In this situation, you would want to write ``libelf -debug``. However, ``-`` can be ambiguous when included after a package name without spaces: @@ -671,12 +710,35 @@ package, not a request for ``mpileaks`` built without ``debug`` options. In this scenario, you should write ``mpileaks~debug`` to avoid ambiguity. -When spack normalizes specs, it prints them out with no spaces and -uses only ``~`` for disabled variants. We allow ``-`` and spaces on -the command line is provided for convenience and legibility. +When spack normalizes specs, it prints them out with no spaces boolean +variants using the backwards compatibility syntax and uses only ``~`` +for disabled boolean variants. We allow ``-`` and spaces on the command +line is provided for convenience and legibility. + +Compiler Flags +~~~~~~~~~~~~~~~~~~~~~~~ + +Compiler flags are specified using the same syntax as non-boolean variants, +but fulfill a different purpose. While the function of a variant is set by +the package, compiler flags are used by the compiler wrappers to inject +flags into the compile line of the build. Additionally, compiler flags are +inherited by dependencies. ``spack install libdwarf cppflags=\"-g\"`` will +install both libdwarf and libelf with the ``-g`` flag injected into their +compile line. + +Notice that the value of the compiler flags must be escape quoted on the +command line. From within python files, the same spec would be specified +``libdwarf cppflags="-g"``. This is necessary because of how the shell +handles the quote symbols. -Architecture specifier +The six compiler flags are injected in the order of implicit make commands +in gnu autotools. If all flags are set, the order is +``$cppflags $cflags|$cxxflags $ldflags command $ldlibs`` for C and C++ and +``$fflags $cppflags $ldflags command $ldlibs`` for fortran. + + +Architecture specifiers ~~~~~~~~~~~~~~~~~~~~~~~ .. Note:: @@ -684,12 +746,9 @@ Architecture specifier Architecture specifiers are part of specs but are not yet functional. They will be in Spack version 1.0, due in Q3 2015. -The architecture specifier starts with a ``=`` and also comes after -some package name within a spec. It allows a user to specify a -particular architecture for the package to be built. This is mostly -used for architectures that need cross-compilation, and in most cases, -users will not need to specify the architecture when they install a -package. +The architecture specifier looks identical to a variant specifier for a +non-boolean variant. The architecture can be specified only using the +reserved name ``arch`` (``arch=bgq_os``). .. _sec-virtual-dependencies: @@ -767,6 +826,23 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +Specifying Specs by Hash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Complicated specs can become cumbersome to enter on the command line, +especially when many of the qualifications are necessary to +distinguish between similar installs, for example when using the +``uninstall`` command. To avoid this, when referencing an existing spec, +Spack allows you to reference specs by their hash. We previously +discussed the spec hash that Spack computes. In place of a spec in any +command, substitute ``/<hash>`` where ``<hash>`` is any amount from +the beginning of a spec hash. If the given spec hash is sufficient +to be unique, Spack will replace the reference with the spec to which +it refers. Otherwise, it will prompt for a more qualified hash. + +Note that this will not work to reinstall a depencency uninstalled by +``spack uninstall -f``. + .. _spack-providers: ``spack providers`` @@ -996,8 +1072,8 @@ than one installed package matches it), then Spack will warn you: $ spack load libelf ==> Error: Multiple matches for spec libelf. Choose one: - libelf@0.8.13%gcc@4.4.7=chaos_5_x86_64_ib - libelf@0.8.13%intel@15.0.0=chaos_5_x86_64_ib + libelf@0.8.13%gcc@4.4.7 arch=chaos_5_x86_64_ib + libelf@0.8.13%intel@15.0.0 arch=chaos_5_x86_64_ib You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to @@ -1245,6 +1321,120 @@ regenerate all module and dotkit files from scratch: .. _extensions: +Filesystem Views +------------------------------- + +.. Maybe this is not the right location for this documentation. + +The Spack installation area allows for many package installation trees +to coexist and gives the user choices as to what versions and variants +of packages to use. To use them, the user must rely on a way to +aggregate a subset of those packages. The section on Environment +Modules gives one good way to do that which relies on setting various +environment variables. An alternative way to aggregate is through +**filesystem views**. + +A filesystem view is a single directory tree which is the union of the +directory hierarchies of the individual package installation trees +that have been included. The files of the view's installed packages +are brought into the view by symbolic or hard links back to their +location in the original Spack installation area. As the view is +formed, any clashes due to a file having the exact same path in its +package installation tree are handled in a first-come-first-served +basis and a warning is printed. Packages and their dependencies can +be both added and removed. During removal, empty directories will be +purged. These operations can be limited to pertain to just the +packages listed by the user or to exclude specific dependencies and +they allow for software installed outside of Spack to coexist inside +the filesystem view tree. + +By its nature, a filesystem view represents a particular choice of one +set of packages among all the versions and variants that are available +in the Spack installation area. It is thus equivalent to the +directory hiearchy that might exist under ``/usr/local``. While this +limits a view to including only one version/variant of any package, it +provides the benefits of having a simpler and traditional layout which +may be used without any particular knowledge that its packages were +built by Spack. + +Views can be used for a variety of purposes including: + +- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +- Producing ephemeral testing or developing environments. + +Using Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~ + +A filesystem view is created and packages are linked in by the ``spack +view`` command's ``symlink`` and ``hardlink`` sub-commands. The +``spack view remove`` command can be used to unlink some or all of the +filesystem view. + +The following example creates a filesystem view based +on an installed ``cmake`` package and then removes from the view the +files in the ``cmake`` package while retaining its dependencies. + +.. code-block:: sh + + + $ spack view -v symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" + + $ ls myview/ + bin doc etc include lib share + + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + + $ spack view -v -d false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput + + +Limitations of Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes some limitations that should be considered in +using filesystems views. + +Filesystem views are merely organizational. The binary executable +programs, shared libraries and other build products found in a view +are mere links into the "real" Spack installation area. If a view is +built with symbolic links it requires the Spack-installed package to +be kept in place. Building a view with hardlinks removes this +requirement but any internal paths (eg, rpath or ``#!`` interpreter +specifications) will still require the Spack-installed package files +to be in place. + +.. FIXME: reference the relocation work of Hegner and Gartung. + +As described above, when a view is built only a single instance of a +file may exist in the unified filesystem tree. If more than one +package provides a file at the same path (relative to its own root) +then it is the first package added to the view that "wins". A warning +is printed and it is up to the user to determine if the conflict +matters. + +It is up to the user to assure a consistent view is produced. In +particular if the user excludes packages, limits the following of +dependencies or removes packages the view may become inconsistent. In +particular, if two packages require the same sub-tree of dependencies, +removing one package (recursively) will remove its dependencies and +leave the other package broken. + + + + + Extensions & Python support ------------------------------------ @@ -1276,7 +1466,7 @@ You can find extensions for your Python installation like this: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1366,9 +1556,9 @@ installation: .. code-block:: sh $ spack activate py-numpy - ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-nose@1.3.4%gcc@4.4.7=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. Several things have happened here. The user requested that ``py-numpy`` be activated in the ``python`` installation it was built @@ -1383,7 +1573,7 @@ packages listed as activated: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1431,7 +1621,7 @@ dependencies, you can use ``spack activate -f``: .. code-block:: sh $ spack activate -f py-numpy - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. .. _spack-deactivate: diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/configuration.rst index 3abfa21a9d..c613071c65 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -1,6 +1,6 @@ -.. _site-configuration: +.. _configuration: -Site configuration +Configuration =================================== .. _temp-space: @@ -55,7 +55,7 @@ directory is. External Packages -~~~~~~~~~~~~~~~~~~~~~ +---------------------------- Spack can be configured to use externally-installed packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI @@ -70,15 +70,15 @@ directory. Here's an example of an external configuration: packages: openmpi: paths: - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in -the given directory. +the given directory. Each ``packages.yaml`` begins with a ``packages:`` token, followed by a list of package names. To specify externals, add a ``paths`` @@ -108,9 +108,9 @@ be: packages: openmpi: paths: - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel buildable: False The addition of the ``buildable`` flag tells Spack that it should never build @@ -118,13 +118,73 @@ its own version of OpenMPI, and it will instead always rely on a pre-built OpenMPI. Similar to ``paths``, ``buildable`` is specified as a property under a package name. -The ``buildable`` does not need to be paired with external packages. -It could also be used alone to forbid packages that may be +The ``buildable`` does not need to be paired with external packages. +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. +Concretization Preferences +-------------------------------- + +Spack can be configured to prefer certain compilers, package +versions, depends_on, and variants during concretization. +The preferred configuration can be controlled via the +``~/.spack/packages.yaml`` file for user configuations, or the +``etc/spack/packages.yaml`` site configuration. + + +Here's an example packages.yaml file that sets preferred packages: + +.. code-block:: sh + + packages: + dyninst: + compiler: [gcc@4.9] + variants: +debug + gperftools: + version: [2.2, 2.4, 2.3] + all: + compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] + providers: + mpi: [mvapich, mpich, openmpi] + + +At a high level, this example is specifying how packages should be +concretized. The dyninst package should prefer using gcc 4.9 and +be built with debug options. The gperftools package should prefer version +2.2 over 2.4. Every package on the system should prefer mvapich for +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten +on the command line if explicitly requested. + +Each packages.yaml file begins with the string ``packages:`` and +package names are specified on the next level. The special string ``all`` +applies settings to each package. Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec +``constraints``, with earlier entries in the list being preferred over +later entries. + +Sometimes a package installation may have constraints that forbid +the first concretization rule, in which case Spack will use the first +legal concretization rule. Going back to the example, if a user +requests gperftools 2.3 or later, then Spack will install version 2.4 +as the 2.4 version of gperftools is preferred over 2.3. + +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, +xlc isn't listed in the compiler list. Every listed compiler from +gcc to pgi will thus be preferred over the xlc compiler. + +The syntax for the ``provider`` section differs slightly from other +concretization rules. A provider lists a value that packages may +``depend_on`` (e.g, mpi) and a list of rules for fulfilling that +dependency. + + + Profiling -~~~~~~~~~~~~~~~~~~~~~ +------------------ Spack has some limited built-in support for profiling, and can report statistics using standard Python timing tools. To use this feature, @@ -133,7 +193,7 @@ supply ``-p`` to Spack on the command line, before any subcommands. .. _spack-p: ``spack -p`` -^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~ ``spack -p`` output looks like this: diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index 0998ba8da4..27a3b4b435 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -31,14 +31,21 @@ platform, all on the command line. # Specify a compiler (and its version), with % $ spack install mpileaks@1.1.2 %gcc@4.7.3 - # Add special compile-time options with + + # Add special compile-time options by name + $ spack install mpileaks@1.1.2 %gcc@4.7.3 debug=True + + # Add special boolean compile-time options with + $ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug - # Cross-compile for a different architecture with = - $ spack install mpileaks@1.1.2 =bgqos_0 + # Add compiler flags using the conventional names + $ spack install mpileaks@1.1.2 %gcc@4.7.3 cppflags=\"-O3 -floop-block\" + + # Cross-compile for a different architecture with arch= + $ spack install mpileaks@1.1.2 arch=bgqos_0 -Users can specify as many or few options as they care about. Spack -will fill in the unspecified values with sensible defaults. +Users can specify as many or few options as they care about. Spack +will fill in the unspecified values with sensible defaults. The two listed +syntaxes for variants are identical when the value is boolean. Customize dependencies diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index d6ce52b747..98ed9ff0fe 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -47,7 +47,7 @@ Table of Contents basic_usage packaging_guide mirrors - site_configuration + configuration developer_guide command_index package_list diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index f7f39d0d7b..54b886310a 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1221,11 +1221,13 @@ just as easily provide a version range: depends_on("libelf@0.8.2:0.8.4:") -Or a requirement for a particular variant: +Or a requirement for a particular variant or compiler flags: .. code-block:: python depends_on("libelf@0.8+debug") + depends_on('libelf debug=True') + depends_on('libelf cppflags="-fPIC") Both users *and* package authors can use the same spec syntax to refer to different package configurations. Users use the spec syntax on the @@ -1623,21 +1625,21 @@ the user runs ``spack install`` and the time the ``install()`` method is called. The concretized version of the spec above might look like this:: - mpileaks@2.3%gcc@4.7.3=linux-ppc64 - ^callpath@1.0%gcc@4.7.3+debug=linux-ppc64 - ^dyninst@8.1.2%gcc@4.7.3=linux-ppc64 - ^libdwarf@20130729%gcc@4.7.3=linux-ppc64 - ^libelf@0.8.11%gcc@4.7.3=linux-ppc64 - ^mpich@3.0.4%gcc@4.7.3=linux-ppc64 + mpileaks@2.3%gcc@4.7.3 arch=linux-ppc64 + ^callpath@1.0%gcc@4.7.3+debug arch=linux-ppc64 + ^dyninst@8.1.2%gcc@4.7.3 arch=linux-ppc64 + ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64 + ^libelf@0.8.11%gcc@4.7.3 arch=linux-ppc64 + ^mpich@3.0.4%gcc@4.7.3 arch=linux-ppc64 .. graphviz:: digraph { - "mpileaks@2.3\n%gcc@4.7.3\n=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n=linux-ppc64" - "mpileaks@2.3\n%gcc@4.7.3\n=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n=linux-ppc64" - "callpath@1.0\n%gcc@4.7.3+debug\n=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" - "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n=linux-ppc64" - "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n=linux-ppc64" + "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64" + "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64" + "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" + "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64" + "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64" } Here, all versions, compilers, and platforms are filled in, and there @@ -1648,8 +1650,8 @@ point will Spack call the ``install()`` method for your package. Concretization in Spack is based on certain selection policies that tell Spack how to select, e.g., a version, when one is not specified explicitly. Concretization policies are discussed in more detail in -:ref:`site-configuration`. Sites using Spack can customize them to -match the preferences of their own users. +:ref:`configuration`. Sites using Spack can customize them to match +the preferences of their own users. .. _spack-spec: @@ -1666,9 +1668,9 @@ running ``spack spec``. For example: ^libdwarf ^libelf - dyninst@8.0.1%gcc@4.7.3=linux-ppc64 - ^libdwarf@20130729%gcc@4.7.3=linux-ppc64 - ^libelf@0.8.13%gcc@4.7.3=linux-ppc64 + dyninst@8.0.1%gcc@4.7.3 arch=linux-ppc64 + ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64 + ^libelf@0.8.13%gcc@4.7.3 arch=linux-ppc64 This is useful when you want to know exactly what Spack will do when you ask for a particular spec. @@ -1682,60 +1684,8 @@ be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -Spack can be configured to prefer certain compilers, package -versions, depends_on, and variants during concretization. -The preferred configuration can be controlled via the -``~/.spack/packages.yaml`` file for user configuations, or the -``etc/spack/packages.yaml`` site configuration. - - -Here's an example packages.yaml file that sets preferred packages: - -.. code-block:: sh - - packages: - dyninst: - compiler: [gcc@4.9] - variants: +debug - gperftools: - version: [2.2, 2.4, 2.3] - all: - compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] - providers: - mpi: [mvapich, mpich, openmpi] - - -At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9 and -be built with debug options. The gperftools package should prefer version -2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). -These options are used to fill in implicit defaults. Any of them can be overwritten -on the command line if explicitly requested. - -Each packages.yaml file begins with the string ``packages:`` and -package names are specified on the next level. The special string ``all`` -applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``variants``, ``version``, -or ``providers``. Each component has an ordered list of spec -``constraints``, with earlier entries in the list being preferred over -later entries. - -Sometimes a package installation may have constraints that forbid -the first concretization rule, in which case Spack will use the first -legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or later, then Spack will install version 2.4 -as the 2.4 version of gperftools is preferred over 2.3. - -An explicit concretization rule in the preferred section will always -take preference over unlisted concretizations. In the above example, -xlc isn't listed in the compiler list. Every listed compiler from -gcc to pgi will thus be preferred over the xlc compiler. - -The syntax for the ``provider`` section differs slightly from other -concretization rules. A provider lists a value that packages may -``depend_on`` (e.g, mpi) and a list of rules for fulfilling that -dependency. +See the `documentation in the config section <concretization-preferences_>`_ +for more details. .. _install-method: @@ -1960,6 +1910,12 @@ the command line. ``$rpath_flag`` can be overriden on a compiler specific basis in ``lib/spack/spack/compilers/$compiler.py``. +The compiler wrappers also pass the compiler flags specified by the user from +the command line (``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, +and/or ``ldlibs``). They do not override the canonical autotools flags with the +same names (but in ALL-CAPS) that may be passed into the build by particularly +challenging package scripts. + Compiler flags ~~~~~~~~~~~~~~ In rare circumstances such as compiling and running small unit tests, a package @@ -2219,12 +2175,12 @@ example: def install(self, prefix): # Do default install - @when('=chaos_5_x86_64_ib') + @when('arch=chaos_5_x86_64_ib') def install(self, prefix): # This will be executed instead of the default install if # the package's sys_type() is chaos_5_x86_64_ib. - @when('=bgqos_0") + @when('arch=bgqos_0") def install(self, prefix): # This will be executed if the package's sys_type is bgqos_0 @@ -2814,11 +2770,11 @@ build it: $ spack stage libelf ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz ######################################################################## 100.0% - ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13.tar.gz - ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64. + ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13.tar.gz + ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64. $ spack cd libelf $ pwd - /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13 + /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13 ``spack cd`` here changed he current working directory to the directory containing the expanded ``libelf`` source code. There are a diff --git a/lib/spack/env/cc b/lib/spack/env/cc index b9b79f83a3..bf98b4c354 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -55,7 +55,10 @@ parameters=( # The compiler input variables are checked for sanity later: # SPACK_CC, SPACK_CXX, SPACK_F77, SPACK_FC -# Debug flag is optional; set to "TRUE" for debug logging: +# The default compiler flags are passed from these variables: +# SPACK_CFLAGS, SPACK_CXXFLAGS, SPACK_FCFLAGS, SPACK_FFLAGS, +# SPACK_LDFLAGS, SPACK_LDLIBS +# Debug env var is optional; set to true for debug logging: # SPACK_DEBUG # Test command is used to unit test the compiler script. # SPACK_TEST_COMMAND @@ -99,21 +102,25 @@ case "$command" in command="$SPACK_CC" language="C" comp="CC" + lang_flags=C ;; c++|CC|g++|clang++|icpc|pgc++|xlc++) command="$SPACK_CXX" language="C++" comp="CXX" + lang_flags=CXX ;; f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" comp="FC" + lang_flags=F ;; f77|gfortran|ifort|pgfortran|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" comp="F77" + lang_flags=F ;; ld) mode=ld @@ -131,7 +138,7 @@ if [[ -z $mode ]]; then if [[ $arg == -v || $arg == -V || $arg == --version || $arg == -dumpversion ]]; then mode=vcheck break - fi + fi done fi @@ -167,6 +174,28 @@ if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=("" ".") +PATH="" +for dir in "${env_path[@]}"; do + addpath=true + for env_dir in "${spack_env_dirs[@]}"; do + if [[ $dir == $env_dir ]]; then + addpath=false + break + fi + done + if $addpath; then + PATH="${PATH:+$PATH:}$dir" + fi +done +export PATH + if [[ $mode == vcheck ]]; then exec ${command} "$@" fi @@ -188,6 +217,42 @@ fi input_command="$@" args=("$@") +# Prepend cppflags, cflags, cxxflags, fcflags, fflags, and ldflags + +# Add ldflags +case "$mode" in + ld|ccld) + args=(${SPACK_LDFLAGS[@]} "${args[@]}") ;; +esac + +# Add compiler flags. +case "$mode" in + cc|ccld) + # Add c, cxx, fc, and f flags + case $lang_flags in + C) + args=(${SPACK_CFLAGS[@]} "${args[@]}") ;; + CXX) + args=(${SPACK_CXXFLAGS[@]} "${args[@]}") ;; + esac + ;; +esac + +# Add cppflags +case "$mode" in + cpp|as|cc|ccld) + args=(${SPACK_CPPFLAGS[@]} "${args[@]}") ;; +esac + +case "$mode" in cc|ccld) + # Add fortran flags + case $lang_flags in + F) + args=(${SPACK_FFLAGS[@]} "${args[@]}") ;; + esac + ;; +esac + # Read spack dependencies from the path environment variable IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do @@ -230,6 +295,12 @@ elif [[ $mode == ld ]]; then $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "${args[@]}") fi +# Add SPACK_LDLIBS to args +case "$mode" in + ld|ccld) + args=("${args[@]}" ${SPACK_LDLIBS[@]}) ;; +esac + # # Unset pesky environment variables that could affect build sanity. # @@ -237,28 +308,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=("" ".") -PATH="" -for dir in "${env_path[@]}"; do - addpath=true - for env_dir in "${spack_env_dirs[@]}"; do - if [[ $dir == $env_dir ]]; then - addpath=false - break - fi - done - if $addpath; then - PATH="${PATH:+$PATH:}$dir" - fi -done -export PATH - full_command=("$command" "${args[@]}") # In test command mode, write out full command for Spack tests. @@ -275,8 +324,8 @@ fi if [[ $SPACK_DEBUG == TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "[$mode] $command $input_command" >> $input_log - echo "[$mode] ${full_command[@]}" >> $output_log + echo "[$mode] $command $input_command" >> "$input_log" + echo "[$mode] ${full_command[@]}" >> "$output_log" fi exec "${full_command[@]}" diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6661a80f27..d72e8bae92 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -22,28 +22,28 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -__all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', - 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', - 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', - 'set_executable', 'copy_mode', 'unset_executable_mode', - 'remove_dead_links', 'remove_linked_tree', 'find_library_path', - 'fix_darwin_install_name'] - import os import glob -import sys import re import shutil import stat import errno import getpass from contextlib import contextmanager, closing -from tempfile import NamedTemporaryFile import subprocess import llnl.util.tty as tty -from spack.util.compression import ALLOWED_ARCHIVE_TYPES + +__all__ = ['set_install_permissions', 'install', 'install_tree', + 'traverse_tree', + 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', + 'force_remove', 'join_path', 'ancestor', 'can_access', + 'filter_file', + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'set_executable', 'copy_mode', 'unset_executable_mode', + 'remove_dead_links', 'remove_linked_tree', 'find_library_path', + 'fix_darwin_install_name', 'to_link_flags'] + def filter_file(regex, repl, *filenames, **kwargs): """Like sed, but uses python regular expressions. @@ -69,6 +69,7 @@ def filter_file(regex, repl, *filenames, **kwargs): # Allow strings to use \1, \2, etc. for replacement, like sed if not callable(repl): unescaped = repl.replace(r'\\', '\\') + def replace_groups_with_groupid(m): def groupid_to_group(x): return m.group(int(x.group(1))) @@ -157,9 +158,12 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: + dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: + dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: + dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) @@ -224,9 +228,10 @@ def force_remove(*paths): for path in paths: try: os.remove(path) - except OSError, e: + except OSError: pass + @contextmanager def working_dir(dirname, **kwargs): if kwargs.get('create', False): @@ -240,7 +245,7 @@ def working_dir(dirname, **kwargs): def touch(path): """Creates an empty file at the specified path.""" - with open(path, 'a') as file: + with open(path, 'a'): os.utime(path, None) @@ -253,7 +258,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError as e: + except OSError: os.remove(dest) os.symlink(src, dest) @@ -275,7 +280,7 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" - return os.access(file_name, os.R_OK|os.W_OK) + return os.access(file_name, os.R_OK | os.W_OK) def traverse_tree(source_root, dest_root, rel_path='', **kwargs): @@ -343,13 +348,14 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): # Treat as a directory if os.path.isdir(source_child) and ( - follow_links or not os.path.islink(source_child)): + follow_links or not os.path.islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): - tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) - for t in tuples: yield t + tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) # NOQA: ignore=E501 + for t in tuples: + yield t # Treat as a file. elif not ignore(os.path.join(rel_path, f)): @@ -379,6 +385,7 @@ def remove_dead_links(root): if not os.path.exists(real_path): os.unlink(path) + def remove_linked_tree(path): """ Removes a directory and its contents. If the directory is a @@ -402,28 +409,41 @@ def fix_darwin_install_name(path): Fix install name of dynamic libraries on Darwin to have full path. There are two parts of this task: (i) use install_name('-id',...) to change install name of a single lib; - (ii) use install_name('-change',...) to change the cross linking between libs. - The function assumes that all libraries are in one folder and currently won't - follow subfolders. + (ii) use install_name('-change',...) to change the cross linking between + libs. The function assumes that all libraries are in one folder and + currently won't follow subfolders. Args: path: directory in which .dylib files are alocated """ - libs = glob.glob(join_path(path,"*.dylib")) + libs = glob.glob(join_path(path, "*.dylib")) for lib in libs: # fix install name first: - subprocess.Popen(["install_name_tool", "-id",lib,lib], stdout=subprocess.PIPE).communicate()[0] - long_deps = subprocess.Popen(["otool", "-L",lib], stdout=subprocess.PIPE).communicate()[0].split('\n') + subprocess.Popen(["install_name_tool", "-id", lib, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501 + long_deps = subprocess.Popen(["otool", "-L", lib], stdout=subprocess.PIPE).communicate()[0].split('\n') # NOQA: ignore=E501 deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]] # fix all dependencies: for dep in deps: for loc in libs: if dep == os.path.basename(loc): - subprocess.Popen(["install_name_tool", "-change",dep,loc,lib], stdout=subprocess.PIPE).communicate()[0] + subprocess.Popen(["install_name_tool", "-change", dep, loc, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501 break +def to_link_flags(library): + """Transforms a path to a <library> into linking flags -L<dir> -l<name>. + + Return: + A string of linking flags. + """ + dir = os.path.dirname(library) + # Asume libXYZ.suffix + name = os.path.basename(library)[3:].split(".")[0] + res = '-L%s -l%s' % (dir, name) + return res + + def find_library_path(libname, *paths): """Searches for a file called <libname> in each path. diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index c638b113fd..ee81e11a20 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -64,12 +64,14 @@ def info(message, *args, **kwargs): format = kwargs.get('format', '*b') stream = kwargs.get('stream', sys.stdout) wrap = kwargs.get('wrap', False) + break_long_words = kwargs.get('break_long_words', False) cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream) for arg in args: if wrap: lines = textwrap.wrap( - str(arg), initial_indent=indent, subsequent_indent=indent) + str(arg), initial_indent=indent, subsequent_indent=indent, + break_long_words=break_long_words) for line in lines: stream.write(line + '\n') else: diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 429ba45882..81a83691d7 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -198,8 +198,13 @@ def colify(elts, **options): for col in xrange(cols): elt = col * rows + row width = config.widths[col] + cextra(elts[elt]) - fmt = '%%-%ds' % width - output.write(fmt % elts[elt]) + if col < cols - 1: + fmt = '%%-%ds' % width + output.write(fmt % elts[elt]) + else: + # Don't pad the rightmost column (sapces can wrap on + # small teriminals if one line is overlong) + output.write(elts[elt]) output.write("\n") row += 1 diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 164340bf0f..75ddca1abc 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,7 +39,9 @@ spack_file = join_path(spack_root, "bin", "spack") lib_path = join_path(spack_root, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") +operating_system_path = join_path(module_path, 'operating_systems') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(spack_root, "var", "spack") @@ -105,7 +107,7 @@ concretizer = DefaultConcretizer() # Version information from spack.version import Version -spack_version = Version("0.8.15") +spack_version = Version("0.9.1") # # Executables used by Spack diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 91d1d2003d..38cff62af4 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -35,8 +35,9 @@ class ABI(object): The current implementation is rather rough and could be improved.""" def architecture_compatible(self, parent, child): - """Returns true iff the parent and child specs have ABI compatible architectures.""" - return not parent.architecture or not child.architecture or parent.architecture == child.architecture + """Returns true iff the parent and child specs have ABI compatible targets.""" + return not parent.architecture or not child.architecture \ + or parent.architecture == child.architecture @memoized diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index b14cb2bea2..cbac7b41d6 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,68 +22,498 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. + +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the +platform. On Cray platforms or any other architecture that has different front +and back end environments, the operating system will determine the method of +compiler +detection. + +There are two different types of compiler detection: + 1. Through the $PATH env variable (front-end detection) + 2. Through the tcl module system. (back-end detection) + +Depending on which operating system is specified, the compiler will be detected +using one of those methods. + +For platforms such as linux and darwin, the operating system is autodetected +and the target is set to be x86_64. + +The command line syntax for specifying an architecture is as follows: + + target=<Target name> os=<OperatingSystem name> + +If the user wishes to use the defaults, either target or os can be left out of +the command line and Spack will concretize using the default. These defaults +are set in the 'platforms/' directory which contains the different subclasses +for platforms. If the machine has multiple architectures, the user can +also enter front-end, or fe or back-end or be. These settings will concretize +to their respective front-end and back-end targets and operating systems. +Additional platforms can be added by creating a subclass of Platform +and adding it inside the platform directory. + +Platforms are an abstract class that are extended by subclasses. If the user +wants to add a new type of platform (such as cray_xe), they can create a +subclass and set all the class attributes such as priority, front_target, +back_target, front_os, back_os. Platforms also contain a priority class +attribute. A lower number signifies higher priority. These numbers are +arbitrarily set and can be changed though often there isn't much need unless a +new platform is added and the user wants that to be detected first. + +Targets are created inside the platform subclasses. Most architecture +(like linux, and darwin) will have only one target (x86_64) but in the case of +Cray machines, there is both a frontend and backend processor. The user can +specify which targets are present on front-end and back-end architecture + +Depending on the platform, operating systems are either auto-detected or are +set. The user can set the front-end and back-end operating setting by the class +attributes front_os and back_os. The operating system as described earlier, +will be responsible for compiler detection. +""" import os -import re -import platform +import imp +import inspect -from llnl.util.lang import memoized +from llnl.util.lang import memoized, list_modules, key_ordering +from llnl.util.filesystem import join_path +import llnl.util.tty as tty import spack +import spack.compilers +from spack.util.naming import mod_to_class +from spack.util.environment import get_path +from spack.util.multiproc import parmap import spack.error as serr class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, - self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, - self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") + + +@key_ordering +class Target(object): + """ Target is the processor of the host machine. + The host machine may have different front-end and back-end targets, + especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also + recognize which platform they came from using the set_platform method. + Targets will have compiler finding strategies + """ + + def __init__(self, name, module_name=None): + self.name = name # case of cray "ivybridge" but if it's x86_64 + self.module_name = module_name # craype-ivybridge + + # Sets only the platform name to avoid recursiveness + + def _cmp_key(self): + return (self.name, self.module_name) + + def __repr__(self): + return self.__str__() + + def __str__(self): + return self.name + + +@key_ordering +class Platform(object): + """ Abstract class that each type of Platform will subclass. + Will return a instance of it once it + is returned + """ + + priority = None # Subclass sets number. Controls detection order + front_end = None + back_end = None + default = None # The default back end target. On cray ivybridge + + front_os = None + back_os = None + default_os = None + + def __init__(self, name): + self.targets = {} + self.operating_sys = {} + self.name = name + + def add_target(self, name, target): + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. + """ + if name in ['frontend', 'fe', 'backend', 'be', 'default_target']: + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of a target" % name) + self.targets[name] = target + + def target(self, name): + """This is a getter method for the target dictionary + that handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten + by a subclass for which we want to provide further aliasing options. + """ + if name == 'default_target': + name = self.default + elif name == 'frontend' or name == 'fe': + name = self.front_end + elif name == 'backend' or name == 'be': + name = self.back_end + + return self.targets.get(name, None) + + def add_operating_system(self, name, os_class): + """ Add the operating_system class object into the + platform.operating_sys dictionary + """ + if name in ['frontend', 'fe', 'backend', 'be', 'default_os']: + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of an OS" % name) + self.operating_sys[name] = os_class + + def operating_system(self, name): + if name == 'default_os': + name = self.default_os + if name == 'frontend' or name == "fe": + name = self.front_os + if name == 'backend' or name == 'be': + name = self.back_os + + return self.operating_sys.get(name, None) + + + @classmethod + def detect(self): + """ Subclass is responsible for implementing this method. + Returns True if the Platform class detects that + it is the current platform + and False if it's not. + """ + raise NotImplementedError() + + + def __repr__(self): + return self.__str__() + + + def __str__(self): + return self.name + + + def _cmp_key(self): + t_keys = ''.join(str(t._cmp_key()) for t in + sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in + sorted(self.operating_sys.values())) + return (self.name, + self.default, + self.front_end, + self.back_end, + self.default_os, + self.front_os, + self.back_os, + t_keys, + o_keys) + + +@key_ordering +class OperatingSystem(object): + """ Operating System will be like a class similar to platform extended + by subclasses for the specifics. Operating System will contain the + compiler finding logic. Instead of calling two separate methods to + find compilers we call find_compilers method for each operating system + """ + + def __init__(self, name, version): + self.name = name + self.version = version + + def __str__(self): + return self.name + self.version + + def __repr__(self): + return self.__str__() + + def _cmp_key(self): + return (self.name, self.version) + def find_compilers(self, *paths): + """ + Return a list of compilers found in the suppied paths. + This invokes the find() method for each Compiler class, + and appends the compilers detected to a list. + """ + if not paths: + paths = get_path('PATH') + # Make sure path elements exist, and include /bin directories + # under prefixes. + filtered_path = [] + for p in paths: + # Eliminate symlinks and just take the real directories. + p = os.path.realpath(p) + if not os.path.isdir(p): + continue + filtered_path.append(p) -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() + # Check for a bin directory, add it if it exists + bin = join_path(p, 'bin') + if os.path.isdir(bin): + filtered_path.append(os.path.realpath(bin)) + + # Once the paths are cleaned up, do a search for each type of + # compiler. We can spawn a bunch of parallel searches to reduce + # the overhead of spelunking all these directories. + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: + self.find_compiler(cmp_cls, *filtered_path), + types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x, y: x+y, compiler_lists) + return clist + + def find_compiler(self, cmp_cls, *path): + """Try to find the given type of compiler in the user's + environment. For each set of compilers found, this returns + compiler objects with the cc, cxx, f77, fc paths and the + version filled in. + + This will search for compilers with the names in cc_names, + cxx_names, etc. and it will group them if they have common + prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would + be grouped with g++-mp-4.7 and gfortran-mp-4.7. + """ + dicts = parmap( + lambda t: cmp_cls._find_matches_in_path(*t), + [(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path), + (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path), + (cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path), + (cmp_cls.fc_names, cmp_cls.fc_version) + tuple(path)]) + + all_keys = set() + for d in dicts: + all_keys.update(d) + + compilers = {} + for k in all_keys: + ver, pre, suf = k + + # Skip compilers with unknown version. + if ver == 'unknown': + continue + + paths = tuple(pn[k] if k in pn else None for pn in dicts) + spec = spack.spec.CompilerSpec(cmp_cls.name, ver) + + if ver in compilers: + prev = compilers[ver] + + # prefer the one with more compilers. + prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] + newcount = len([p for p in paths if p is not None]) + prevcount = len([p for p in prev_paths if p is not None]) + + # Don't add if it's not an improvement over prev compiler. + if newcount <= prevcount: + continue + + compilers[ver] = cmp_cls(spec, self, paths) + + return list(compilers.values()) + + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + return d + +@key_ordering +class Arch(object): + "Architecture is now a class to help with setting attributes" + + def __init__(self, platform=None, platform_os=None, target=None): + self.platform = platform + if platform and platform_os: + platform_os = self.platform.operating_system(platform_os) + self.platform_os = platform_os + if platform and target: + target = self.platform.target(target) + self.target = target + + # Hooks for parser to use when platform is set after target or os + self.target_string = None + self.os_string = None + + @property + def concrete(self): + return all((self.platform is not None, + isinstance(self.platform, Platform), + self.platform_os is not None, + isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target))) + + def __str__(self): + if self.platform or self.platform_os or self.target: + if self.platform.name == 'darwin': + os_name = self.platform_os.name if self.platform_os else "None" + else: + os_name = str(self.platform_os) + + return (str(self.platform) + "-" + + os_name + "-" + str(self.target)) + else: + return '' + + + def __contains__(self, string): + return string in str(self) + + + def _cmp_key(self): + if isinstance(self.platform, Platform): + platform = self.platform.name + else: + platform = self.platform + if isinstance(self.platform_os, OperatingSystem): + platform_os = self.platform_os.name + else: + platform_os = self.platform_os + if isinstance(self.target, Target): + target = self.target.name + else: + target = self.target + return (platform, platform_os, target) + + def to_dict(self): + d = {} + d['platform'] = str(self.platform) if self.platform else None + d['platform_os'] = str(self.platform_os) if self.platform_os else None + d['target'] = str(self.target) if self.target else None + + return d + + +def _target_from_dict(target_name, platform=None): + """ Creates new instance of target and assigns all the attributes of + that target from the dictionary + """ + if not platform: + platform = sys_type() + return platform.target(target_name) + + +def _operating_system_from_dict(os_name, platform=None): + """ uses platform's operating system method to grab the constructed + operating systems that are valid on the platform. + """ + if not platform: + platform = sys_type() + if isinstance(os_name, dict): + name = os_name['name'] + version = os_name['version'] + return platform.operating_system(name+version) else: - return spack.sys_type + return platform.operating_system(os_name) + +def _platform_from_dict(platform_name): + """ Constructs a platform from a dictionary. """ + platform_list = all_platforms() + for p in platform_list: + if platform_name.replace("_", "").lower() == p.__name__.lower(): + return p() -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') +def arch_from_dict(d): + """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict + helper methods to recreate the arch tuple from the dictionary read from + a yaml file + """ + arch = Arch() -def get_sys_type_from_platform(): - """Return the architecture from Python's platform module.""" - sys_type = platform.system() + '-' + platform.machine() - sys_type = re.sub(r'[^\w-]', '_', sys_type) - return sys_type.lower() + if isinstance(d, basestring): + # We have an old spec using a string for the architecture + arch.platform = Platform('spack_compatibility') + arch.platform_os = OperatingSystem('unknown', '') + arch.target = Target(d) + + arch.os_string = None + arch.target_string = None + else: + if d is None: + return None + platform_name = d['platform'] + os_name = d['platform_os'] + target_name = d['target'] + + if platform_name: + arch.platform = _platform_from_dict(platform_name) + else: + arch.platform = None + if target_name: + arch.target = _target_from_dict(target_name, arch.platform) + else: + arch.target = None + if os_name: + arch.platform_os = _operating_system_from_dict(os_name, + arch.platform) + else: + arch.platform_os = None + + arch.os_string = None + arch.target_string = None + + return arch @memoized -def sys_type(): - """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, - get_sys_type_from_platform] +def all_platforms(): + classes = [] + mod_path = spack.platform_path + parent_module = "spack.platforms" + + for name in list_modules(mod_path): + mod_name = '%s.%s' % (parent_module, name) + class_name = mod_to_class(name) + mod = __import__(mod_name, fromlist=[class_name]) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + + classes.append(cls) - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: - break + return classes - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) +@memoized +def sys_type(): + """ Gather a list of all available subclasses of platforms. + Sorts the list according to their priority looking. Priority is + an arbitrarily set number. Detects platform either using uname or + a file path (/opt/cray...) + """ + # Try to create a Platform object using the config file FIRST + platform_list = all_platforms() + platform_list.sort(key=lambda a: a.priority) - return sys_type + for platform in platform_list: + if platform.detect(): + return platform() diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5ce4cb1ce1..3fcfb151b8 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -51,15 +51,16 @@ There are two parts to the build environment: Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ -import multiprocessing import os -import platform -import shutil import sys +import shutil +import multiprocessing +import platform -import spack import llnl.util.tty as tty from llnl.util.filesystem import * + +import spack from spack.environment import EnvironmentModifications, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -74,20 +75,19 @@ SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE' # set_build_environment_variables and used to pass parameters to # Spack's compiler wrappers. # -SPACK_ENV_PATH = 'SPACK_ENV_PATH' -SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES' -SPACK_PREFIX = 'SPACK_PREFIX' -SPACK_INSTALL = 'SPACK_INSTALL' -SPACK_DEBUG = 'SPACK_DEBUG' -SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' -SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +SPACK_ENV_PATH = 'SPACK_ENV_PATH' +SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES' +SPACK_PREFIX = 'SPACK_PREFIX' +SPACK_INSTALL = 'SPACK_INSTALL' +SPACK_DEBUG = 'SPACK_DEBUG' +SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' +SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' # Platform-specific library suffix. dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' - class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallel or not on a per-invocation basis. Using @@ -98,6 +98,7 @@ class MakeExecutable(Executable): Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides everything. """ + def __init__(self, name, jobs): super(MakeExecutable, self).__init__(name) self.jobs = jobs @@ -113,30 +114,95 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + # Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, output=str, error=str).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[ + i + 1], output=str, error=str), '<string>', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, output=str, error=str) + exec(compile(load, '<string>', 'exec')) + + +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, output=str, error=str).split('\n') + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper() + '_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath + 6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L + 2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + # Unable to find module path + return None + + def set_compiler_environment_variables(pkg, env): - assert pkg.spec.concrete + assert(pkg.spec.concrete) + compiler = pkg.compiler + flags = pkg.spec.compiler_flags + # Set compiler variables used by CMake and autotools - assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in compiler.link_paths for key in ( + 'cc', 'cxx', 'f77', 'fc')) # Populate an object with the list of environment modifications # and return it - # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. + # TODO : add additional kwargs for better diagnostics, like requestor, + # ttyout, ttyerr, etc. link_dir = spack.build_env_path - env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) - env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) - env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) - env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) # Set SPACK compiler variables so that our wrapper knows what to call - compiler = pkg.compiler if compiler.cc: - env.set('SPACK_CC', compiler.cc) + env.set('SPACK_CC', compiler.cc) + env.set('CC', join_path(link_dir, compiler.link_paths['cc'])) if compiler.cxx: env.set('SPACK_CXX', compiler.cxx) + env.set('CXX', join_path(link_dir, compiler.link_paths['cxx'])) if compiler.f77: env.set('SPACK_F77', compiler.f77) + env.set('F77', join_path(link_dir, compiler.link_paths['f77'])) if compiler.fc: env.set('SPACK_FC', compiler.fc) + env.set('FC', join_path(link_dir, compiler.link_paths['fc'])) # Set SPACK compiler rpath flags so that our wrapper knows what to use env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg) @@ -144,7 +210,17 @@ def set_compiler_environment_variables(pkg, env): env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg) env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg) + # Add every valid compiler flag to the environment, prefixed with "SPACK_" + for flag in spack.spec.FlagMap.valid_compiler_flags(): + # Concreteness guarantees key safety here + if flags[flag] != []: + env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag])) + env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) + + for mod in compiler.modules: + load_module(mod) + return env @@ -163,7 +239,8 @@ def set_build_environment_variables(pkg, env): # handled by putting one in the <build_env_path>/case-insensitive # directory. Add that to the path too. env_paths = [] - for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + compiler_specific = join_path(spack.build_env_path, pkg.compiler.name) + for item in [spack.build_env_path, compiler_specific]: env_paths.append(item) ci = join_path(item, 'case-insensitive') if os.path.isdir(ci): @@ -176,7 +253,8 @@ def set_build_environment_variables(pkg, env): # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] env.set_path(SPACK_DEPENDENCIES, dep_prefixes) - env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH + # Add dependencies to CMAKE_PREFIX_PATH + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Install prefix env.set(SPACK_PREFIX, pkg.prefix) @@ -192,7 +270,8 @@ def set_build_environment_variables(pkg, env): env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + bin_dirs = reversed( + filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) for item in bin_dirs: env.prepend_path('PATH', item) @@ -203,13 +282,15 @@ def set_build_environment_variables(pkg, env): env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH - pkg_config_dirs = [] - for p in dep_prefixes: - for maybe in ('lib', 'lib64', 'share'): - pcdir = join_path(p, maybe, 'pkgconfig') + for pre in dep_prefixes: + for directory in ('lib', 'lib64', 'share'): + pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - pkg_config_dirs.append(pcdir) - env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) + # pkg_config_dirs.append(pcdir) + env.prepend_path('PKG_CONFIG_PATH', pcdir) + + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) return env @@ -229,7 +310,7 @@ def set_module_variables_for_package(pkg, module): m.make_jobs = jobs # TODO: make these build deps that can be installed if not found. - m.make = MakeExecutable('make', jobs) + m.make = MakeExecutable('make', jobs) m.gmake = MakeExecutable('gmake', jobs) # easy shortcut to os.environ @@ -253,33 +334,34 @@ def set_module_variables_for_package(pkg, module): # Set up CMake rpath m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % + ":".join(get_rpaths(pkg))) # Put spack compiler paths in module scope. link_dir = spack.build_env_path - m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) + m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) - m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) + m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) # Emulate some shell commands for convenience - m.pwd = os.getcwd - m.cd = os.chdir - m.mkdir = os.mkdir - m.makedirs = os.makedirs - m.remove = os.remove - m.removedirs = os.removedirs - m.symlink = os.symlink - - m.mkdirp = mkdirp - m.install = install + m.pwd = os.getcwd + m.cd = os.chdir + m.mkdir = os.mkdir + m.makedirs = os.makedirs + m.remove = os.remove + m.removedirs = os.removedirs + m.symlink = os.symlink + + m.mkdirp = mkdirp + m.install = install m.install_tree = install_tree - m.rmtree = shutil.rmtree - m.move = shutil.move + m.rmtree = shutil.rmtree + m.move = shutil.move # Useful directories within the prefix are encapsulated in # a Prefix object. - m.prefix = pkg.prefix + m.prefix = pkg.prefix # Platform-specific library suffix. m.dso_suffix = dso_suffix @@ -292,26 +374,40 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) + # Second module is our compiler mod name. We use that to get rpaths from + # module show output. + if pkg.compiler.modules and len(pkg.compiler.modules) > 1: + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths def parent_class_modules(cls): - """Get list of super class modules that are all descend from spack.Package""" + """ + Get list of super class modules that are all descend from spack.Package + """ if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): return [] result = [] module = sys.modules.get(cls.__module__) if module: - result = [ module ] + result = [module] for c in cls.__bases__: result.extend(parent_class_modules(c)) return result +def load_external_modules(pkg): + """ traverse the spec list and find any specs that have external modules. + """ + for dep in list(pkg.spec.traverse()): + if dep.external_module: + load_module(dep.external_module) + + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() - run_env = EnvironmentModifications() + run_env = EnvironmentModifications() # Before proceeding, ensure that specs and packages are consistent # @@ -327,11 +423,12 @@ def setup_package(pkg): # throwaway environment, but it is kind of dirty. # # TODO: Think about how to avoid this fix and do something cleaner. - for s in pkg.spec.traverse(): s.package.spec = s + for s in pkg.spec.traverse(): + s.package.spec = s set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - + load_external_modules(pkg) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False): @@ -415,7 +512,9 @@ def fork(pkg, function): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) + message = "Installation process had nonzero exit code : {code}" + strcode = str(returncode) + raise InstallError(message.format(code=strcode)) class InstallError(spack.error.SpackError): diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index dc96dd0faa..cf2f96fd21 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -28,8 +28,4 @@ import spack.architecture as architecture description = "Print the architecture for this machine" def arch(parser, args): - configured_sys_type = architecture.get_sys_type_from_spack_globals() - if not configured_sys_type: - configured_sys_type = "autodetect" - print "Configured sys_type: %s" % configured_sys_type - print "Autodetected default sys_type: %s" % architecture.sys_type() + print architecture.sys_type() diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index dc7731a290..c95045ef85 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -70,7 +70,7 @@ def setup_parser(subparser): def compiler_find(args): - """Search either $PATH or a list of paths for compilers and add them + """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" paths = args.add_paths if not paths: @@ -78,7 +78,6 @@ def compiler_find(args): compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) if c.spec not in spack.compilers.all_compilers(scope=args.scope)] - if compilers: spack.compilers.add_compilers_to_config(compilers, scope=args.scope) n = len(compilers) @@ -93,7 +92,6 @@ def compiler_find(args): def compiler_remove(args): cspec = CompilerSpec(args.compiler_spec) compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) - if not compilers: tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: @@ -121,6 +119,8 @@ def compiler_info(args): print "\tcxx = %s" % c.cxx print "\tf77 = %s" % c.f77 print "\tfc = %s" % c.fc + print "\tmodules = %s" % c.modules + print "\toperating system = %s" % c.operating_system def compiler_list(args): @@ -135,10 +135,10 @@ def compiler_list(args): def compiler(parser, args): - action = { 'add' : compiler_find, - 'find' : compiler_find, - 'remove' : compiler_remove, - 'rm' : compiler_remove, - 'info' : compiler_info, - 'list' : compiler_list } + action = {'add' : compiler_find, + 'find' : compiler_find, + 'remove' : compiler_remove, + 'rm' : compiler_remove, + 'info' : compiler_info, + 'list' : compiler_list } action[args.compiler_command](args) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7071af2686..41bfa741f6 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -1,3 +1,4 @@ +_copyright = """\ ############################################################################## # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -22,6 +23,7 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +""" import string import os import hashlib @@ -47,22 +49,22 @@ from spack.stage import Stage description = "Create a new package file from an archive URL" -package_template = string.Template("""\ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. +package_template = string.Template( + _copyright + """ # -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: # # spack install ${name} # -# You can always get back here to change things with: +# You can edit this file again by typing: # # spack edit ${name} # -# See the spack documentation for more information on building -# packages. +# See the Spack documentation for more information on packaging. +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. # from spack import * diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index a99012a275..3ec671f93f 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -22,57 +22,86 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys -import collections -import itertools import argparse -from StringIO import StringIO +import sys import llnl.util.tty as tty +import spack +import spack.spec +from llnl.util.lang import * from llnl.util.tty.colify import * from llnl.util.tty.color import * from llnl.util.lang import * -import spack -import spack.spec +description = "Find installed spack packages" -description ="Find installed spack packages" def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() + format_group.add_argument('-s', '--short', + action='store_const', + dest='mode', + const='short', + help='Show only specs (default)') + format_group.add_argument('-p', '--paths', + action='store_const', + dest='mode', + const='paths', + help='Show paths to package install directories') format_group.add_argument( - '-s', '--short', action='store_const', dest='mode', const='short', - help='Show only specs (default)') - format_group.add_argument( - '-p', '--paths', action='store_const', dest='mode', const='paths', - help='Show paths to package install directories') - format_group.add_argument( - '-d', '--deps', action='store_const', dest='mode', const='deps', + '-d', '--deps', + action='store_const', + dest='mode', + const='deps', help='Show full dependency DAG of installed packages') + subparser.add_argument('-l', '--long', + action='store_true', + dest='long', + help='Show dependency hashes as well as versions.') + subparser.add_argument('-L', '--very-long', + action='store_true', + dest='very_long', + help='Show dependency hashes as well as versions.') + subparser.add_argument('-f', '--show-flags', + action='store_true', + dest='show_flags', + help='Show spec compiler flags.') + subparser.add_argument( - '-l', '--long', action='store_true', - help='Show dependency hashes as well as versions.') + '-e', '--explicit', + action='store_true', + help='Show only specs that were installed explicitly') subparser.add_argument( - '-L', '--very-long', action='store_true', - help='Show dependency hashes as well as versions.') - + '-E', '--implicit', + action='store_true', + help='Show only specs that were installed as dependencies') subparser.add_argument( - '-u', '--unknown', action='store_true', + '-u', '--unknown', + action='store_true', + dest='unknown', help='Show only specs Spack does not have a package for.') subparser.add_argument( - '-m', '--missing', action='store_true', + '-m', '--missing', + action='store_true', + dest='missing', help='Show missing dependencies as well as installed specs.') subparser.add_argument( - '-M', '--only-missing', action='store_true', - help='Show only missing dependencies.') - subparser.add_argument( - '-N', '--namespace', action='store_true', - help='Show fully qualified package names.') - - subparser.add_argument( - 'query_specs', nargs=argparse.REMAINDER, - help='optional specs to filter results') + '-v', '--variants', + action='store_true', + dest='variants', + help='Show variants in output (can be long)') + subparser.add_argument('-M', '--only-missing', + action='store_true', + dest='only_missing', + help='Show only missing dependencies.') + subparser.add_argument('-N', '--namespace', + action='store_true', + help='Show fully qualified package names.') + + subparser.add_argument('query_specs', + nargs=argparse.REMAINDER, + help='optional specs to filter results') def gray_hash(spec, length): @@ -83,29 +112,36 @@ def display_specs(specs, **kwargs): mode = kwargs.get('mode', 'short') hashes = kwargs.get('long', False) namespace = kwargs.get('namespace', False) + flags = kwargs.get('show_flags', False) + variants = kwargs.get('variants', False) hlen = 7 if kwargs.get('very_long', False): hashes = True hlen = None + nfmt = '.' if namespace else '_' + ffmt = '$%+' if flags else '' + vfmt = '$+' if variants else '' + format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt) + # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) # Traverse the index and print out each package for i, (architecture, compiler) in enumerate(sorted(index)): - if i > 0: print + if i > 0: + print - header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, architecture, - spack.spec.compiler_color, compiler) + header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color, + architecture, spack.spec.compiler_color, + compiler) tty.hline(colorize(header), char='-') - specs = index[(architecture,compiler)] + specs = index[(architecture, compiler)] specs.sort() - nfmt = '.' if namespace else '_' - abbreviated = [s.format('$%s$@$+' % nfmt, color=True) for s in specs] + abbreviated = [s.format(format_string, color=True) for s in specs] if mode == 'paths': # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) @@ -114,38 +150,71 @@ def display_specs(specs, **kwargs): for abbrv, spec in zip(abbreviated, specs): if hashes: - print gray_hash(spec, hlen), - print format % (abbrv, spec.prefix) + print(gray_hash(spec, hlen), ) + print(format % (abbrv, spec.prefix)) elif mode == 'deps': for spec in specs: - print spec.tree( - format='$%s$@$+' % nfmt, + print(spec.tree( + format=format_string, color=True, indent=4, - prefix=(lambda s: gray_hash(s, hlen)) if hashes else None) + prefix=(lambda s: gray_hash(s, hlen)) if hashes else None)) elif mode == 'short': - def fmt(s): - string = "" - if hashes: - string += gray_hash(s, hlen) + ' ' - string += s.format('$-%s$@$+' % nfmt, color=True) + # Print columns of output if not printing flags + if not flags: + + def fmt(s): + string = "" + if hashes: + string += gray_hash(s, hlen) + ' ' + string += s.format('$-%s$@%s' % (nfmt, vfmt), color=True) + + return string - return string - colify(fmt(s) for s in specs) + colify(fmt(s) for s in specs) + # Print one entry per line if including flags + else: + for spec in specs: + # Print the hash if necessary + hsh = gray_hash(spec, hlen) + ' ' if hashes else '' + print(hsh + spec.format(format_string, color=True) + '\n') else: raise ValueError( - "Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode) + "Invalid mode for display_specs: %s. Must be one of (paths," + "deps, short)." % mode) # NOQA: ignore=E501 +def query_arguments(args): + # Check arguments + if args.explicit and args.implicit: + tty.error('You can\'t pass -E and -e options simultaneously.') + raise SystemExit(1) + + # Set up query arguments. + installed, known = True, any + if args.only_missing: + installed = False + elif args.missing: + installed = any + if args.unknown: + known = False + explicit = any + if args.explicit: + explicit = True + if args.implicit: + explicit = False + q_args = {'installed': installed, 'known': known, "explicit": explicit} + return q_args + def find(parser, args): # Filter out specs that don't exist. query_specs = spack.cmd.parse_specs(args.query_specs) query_specs, nonexisting = partition_list( - query_specs, lambda s: spack.repo.exists(s.name)) + query_specs, lambda s: spack.repo.exists(s.name) or not s.name) if nonexisting: msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '') @@ -155,21 +224,14 @@ def find(parser, args): if not query_specs: return - # Set up query arguments. - installed, known = True, any - if args.only_missing: - installed = False - elif args.missing: - installed = any - if args.unknown: - known = False - q_args = { 'installed' : installed, 'known' : known } + q_args = query_arguments(args) # Get all the specs the user asked for if not query_specs: specs = set(spack.installed_db.query(**q_args)) else: - results = [set(spack.installed_db.query(qs, **q_args)) for qs in query_specs] + results = [set(spack.installed_db.query(qs, **q_args)) + for qs in query_specs] specs = set.union(*results) if not args.mode: @@ -177,7 +239,10 @@ def find(parser, args): if sys.stdout.isatty(): tty.msg("%d installed packages." % len(specs)) - display_specs(specs, mode=args.mode, + display_specs(specs, + mode=args.mode, long=args.long, very_long=args.very_long, - namespace=args.namespace) + show_flags=args.show_flags, + namespace=args.namespace, + variants=args.variants) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index fef21f15ba..9d3175786b 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -78,4 +78,5 @@ def install(parser, args): ignore_deps=args.ignore_deps, make_jobs=args.jobs, verbose=args.verbose, - fake=args.fake) + fake=args.fake, + explicit=True) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 3bffc2633b..a6f08d09ed 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -39,6 +39,13 @@ error_message = """You can either: b) use spack uninstall -a to uninstall ALL matching specs. """ +# Arguments for display_specs when we find ambiguity +display_args = { + 'long': True, + 'show_flags': True, + 'variants':True +} + def ask_for_confirmation(message): while True: @@ -92,7 +99,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): if not allow_multiple_matches and len(matching) > 1: tty.error("%s matches multiple packages:" % spec) print() - display_specs(matching, long=True) + display_specs(matching, **display_args) print() has_errors = True @@ -172,7 +179,7 @@ def uninstall(parser, args): tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True)) print('') print("The following packages depend on it:") - display_specs(lst, long=True) + display_specs(lst, **display_args) print('') has_error = True elif args.dependents: @@ -186,7 +193,7 @@ def uninstall(parser, args): if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") print('') - display_specs(uninstall_list, long=True) + display_specs(uninstall_list, **display_args) print('') ask_for_confirmation('Do you want to proceed ? ') diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py new file mode 100644 index 0000000000..8f1fc9be74 --- /dev/null +++ b/lib/spack/spack/cmd/view.py @@ -0,0 +1,295 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +'''Produce a "view" of a Spack DAG. + +A "view" is file hierarchy representing the union of a number of +Spack-installed package file hierarchies. The union is formed from: + +- specs resolved from the package names given by the user (the seeds) + +- all depenencies of the seeds unless user specifies `--no-depenencies` + +- less any specs with names matching the regular expressions given by + `--exclude` + +The `view` can be built and tore down via a number of methods (the "actions"): + +- symlink :: a file system view which is a directory hierarchy that is + the union of the hierarchies of the installed packages in the DAG + where installed files are referenced via symlinks. + +- hardlink :: like the symlink view but hardlinks are used. + +- statlink :: a view producing a status report of a symlink or + hardlink view. + +The file system view concept is imspired by Nix, implemented by +brett.viren@gmail.com ca 2016. + +''' +# Implementation notes: +# +# This is implemented as a visitor pattern on the set of package specs. +# +# The command line ACTION maps to a visitor_*() function which takes +# the set of package specs and any args which may be specific to the +# ACTION. +# +# To add a new view: +# 1. add a new cmd line args sub parser ACTION +# 2. add any action-specific options/arguments, most likely a list of specs. +# 3. add a visitor_MYACTION() function +# 4. add any visitor_MYALIAS assignments to match any command line aliases + +import os +import re +import spack +import spack.cmd +import llnl.util.tty as tty + +description = "Produce a single-rooted directory view of a spec." + + +def setup_parser(sp): + setup_parser.parser = sp + + sp.add_argument( + '-v', '--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument( + '-e', '--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument( + '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'], + default='true', + help="Follow dependencies.") + + ssp = sp.add_subparsers(metavar='ACTION', dest='action') + + specs_opts = dict(metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + + # The action parameterizes the command but in keeping with Spack + # patterns we make it a subcommand. + file_system_view_actions = [ + ssp.add_parser( + 'symlink', aliases=['add', 'soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser( + 'hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser( + 'remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser( + 'statlink', aliases=['status', 'check'], + help='Check status of packages in a filesystem view.') + ] + # All these options and arguments are common to every action. + for act in file_system_view_actions: + act.add_argument('path', nargs=1, + help="Path to file system view directory.") + act.add_argument('specs', **specs_opts) + + return + + +def assuredir(path): + 'Assure path exists as a directory' + if not os.path.exists(path): + os.makedirs(path) + + +def relative_to(prefix, path): + 'Return end of `path` relative to `prefix`' + assert 0 == path.find(prefix) + reldir = path[len(prefix):] + if reldir.startswith('/'): + reldir = reldir[1:] + return reldir + + +def transform_path(spec, path, prefix=None): + 'Return the a relative path corresponding to given path spec.prefix' + if os.path.isabs(path): + path = relative_to(spec.prefix, path) + subdirs = path.split(os.path.sep) + if subdirs[0] == '.spack': + lst = ['.spack', spec.name] + subdirs[1:] + path = os.path.join(*lst) + if prefix: + path = os.path.join(prefix, path) + return path + + +def purge_empty_directories(path): + '''Ascend up from the leaves accessible from `path` + and remove empty directories.''' + for dirpath, subdirs, files in os.walk(path, topdown=False): + for sd in subdirs: + sdp = os.path.join(dirpath, sd) + try: + os.rmdir(sdp) + except OSError: + pass + + +def filter_exclude(specs, exclude): + 'Filter specs given sequence of exclude regex' + to_exclude = [re.compile(e) for e in exclude] + + def exclude(spec): + for e in to_exclude: + if e.match(spec.name): + return True + return False + return [s for s in specs if not exclude(s)] + + +def flatten(seeds, descend=True): + 'Normalize and flattend seed specs and descend hiearchy' + flat = set() + for spec in seeds: + if not descend: + flat.add(spec) + continue + flat.update(spec.normalized().traverse()) + return flat + + +def check_one(spec, path, verbose=False): + 'Check status of view in path against spec' + dotspack = os.path.join(path, '.spack', spec.name) + if os.path.exists(os.path.join(dotspack)): + tty.info('Package in view: "%s"' % spec.name) + return + tty.info('Package not in view: "%s"' % spec.name) + return + + +def remove_one(spec, path, verbose=False): + 'Remove any files found in `spec` from `path` and purge empty directories.' + + if not os.path.exists(path): + return # done, short circuit + + dotspack = transform_path(spec, '.spack', path) + if not os.path.exists(dotspack): + if verbose: + tty.info('Skipping nonexistent package: "%s"' % spec.name) + return + + if verbose: + tty.info('Removing package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): + if not filenames: + continue + targdir = transform_path(spec, dirpath, path) + for fname in filenames: + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + continue + os.unlink(dst) + + +def link_one(spec, path, link=os.symlink, verbose=False): + 'Link all files in `spec` into directory `path`.' + + dotspack = transform_path(spec, '.spack', path) + if os.path.exists(dotspack): + tty.warn('Skipping existing package: "%s"' % spec.name) + return + + if verbose: + tty.info('Linking package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + targdir = transform_path(spec, dirpath, path) + assuredir(targdir) + + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these + tty.warn("Skipping existing file: %s" % dst) + continue + link(src, dst) + + +def visitor_symlink(specs, args): + 'Symlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, verbose=args.verbose) +visitor_add = visitor_symlink +visitor_soft = visitor_symlink + + +def visitor_hardlink(specs, args): + 'Hardlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, os.link, verbose=args.verbose) +visitor_hard = visitor_hardlink + + +def visitor_remove(specs, args): + 'Remove all files and directories found in specs from args.path' + path = args.path[0] + for spec in specs: + remove_one(spec, path, verbose=args.verbose) + purge_empty_directories(path) +visitor_rm = visitor_remove + + +def visitor_statlink(specs, args): + 'Give status of view in args.path relative to specs' + path = args.path[0] + for spec in specs: + check_one(spec, path, verbose=args.verbose) +visitor_status = visitor_statlink +visitor_check = visitor_statlink + + +def view(parser, args): + 'Produce a view of a set of packages.' + + # Process common args + seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] + specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true']) + specs = filter_exclude(specs, args.exclude) + + # Execute the visitation. + try: + visitor = globals()['visitor_' + args.action] + except KeyError: + tty.error('Unknown action: "%s"' % args.action) + visitor(specs, args) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e2da272212..ce4555bc56 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -33,6 +33,7 @@ from llnl.util.filesystem import join_path import spack.error import spack.spec +import spack.architecture from spack.util.multiproc import parmap from spack.util.executable import * from spack.util.environment import get_path @@ -107,22 +108,46 @@ class Compiler(object): @property def fc_rpath_arg(self): return '-Wl,-rpath,' + # Cray PrgEnv name that can be used to load this compiler + PrgEnv = None + # Name of module used to switch versions of this compiler + PrgEnv_compiler = None - - def __init__(self, cspec, cc, cxx, f77, fc): + def __init__(self, cspec, operating_system, + paths, modules=[], alias=None, **kwargs): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) - + self.cc = check(paths[0]) + self.cxx = check(paths[1]) + if len(paths) > 2: + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) + + #self.cc = check(cc) + #self.cxx = check(cxx) + #self.f77 = check(f77) + #self.fc = check(fc) + + # Unfortunately have to make sure these params are accepted + # in the same order they are returned by sorted(flags) + # in compilers/__init__.py + self.flags = {} + for flag in spack.spec.FlagMap.valid_compiler_flags(): + value = kwargs.get(flag, None) + if value is not None: + self.flags[flag] = value.split() + + self.operating_system = operating_system self.spec = cspec - + self.modules = modules + self.alias = alias @property def version(self): @@ -188,7 +213,6 @@ class Compiler(object): def fc_version(cls, fc): return cls.default_version(fc) - @classmethod def _find_matches_in_path(cls, compiler_names, detect_version, *path): """Finds compilers in the paths supplied. @@ -250,57 +274,6 @@ class Compiler(object): successful.reverse() return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def find(cls, *path): - """Try to find this type of compiler in the user's - environment. For each set of compilers found, this returns - compiler objects with the cc, cxx, f77, fc paths and the - version filled in. - - This will search for compilers with the names in cc_names, - cxx_names, etc. and it will group them if they have common - prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would - be grouped with g++-mp-4.7 and gfortran-mp-4.7. - """ - dicts = parmap( - lambda t: cls._find_matches_in_path(*t), - [(cls.cc_names, cls.cc_version) + tuple(path), - (cls.cxx_names, cls.cxx_version) + tuple(path), - (cls.f77_names, cls.f77_version) + tuple(path), - (cls.fc_names, cls.fc_version) + tuple(path)]) - - all_keys = set() - for d in dicts: - all_keys.update(d) - - compilers = {} - for k in all_keys: - ver, pre, suf = k - - # Skip compilers with unknown version. - if ver == 'unknown': - continue - - paths = tuple(pn[k] if k in pn else None for pn in dicts) - spec = spack.spec.CompilerSpec(cls.name, ver) - - if ver in compilers: - prev = compilers[ver] - - # prefer the one with more compilers. - prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) - prevcount = len([p for p in prev_paths if p is not None]) - - # Don't add if it's not an improvement over prev compiler. - if newcount <= prevcount: - continue - - compilers[ver] = cls(spec, *paths) - - return list(compilers.values()) - - def __repr__(self): """Return a string representation of the compiler toolchain.""" return self.__str__() @@ -309,7 +282,7 @@ class Compiler(object): def __str__(self): """Return a string representation of the compiler toolchain.""" return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system))))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 692e5518aa..ae72b743b2 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -28,6 +28,11 @@ system and configuring Spack to use multiple compilers. import imp import os import platform +import copy +import hashlib +import base64 +import yaml +import sys from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -45,7 +50,9 @@ from spack.util.naming import mod_to_class from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' -_required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_other_instance_vars = ['modules', 'operating_system'] +_cache_config_file = [] # TODO: customize order in config file if platform.system() == 'Darwin': @@ -64,107 +71,105 @@ def _auto_compiler_spec(function): def _to_dict(compiler): """Return a dict version of compiler suitable to insert in YAML.""" - return { - str(compiler.spec) : dict( - (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars) - } + d = {} + d['spec'] = str(compiler.spec) + d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) + d['operating_system'] = str(compiler.operating_system) + d['modules'] = compiler.modules if compiler.modules else [] + if compiler.alias: + d['alias'] = compiler.alias -def get_compiler_config(arch=None, scope=None): + return {'compiler': d} + + +def get_compiler_config(scope=None): """Return the compiler configuration for the specified architecture. """ - # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.sys_type() - if arch is None: - arch = my_arch - def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch] = {} - compilers = find_compilers(*get_path('PATH')) + compilers = find_compilers() + compilers_dict = [] for compiler in compilers: - config[arch].update(_to_dict(compiler)) - spack.config.update_config('compilers', config, scope=scope) + compilers_dict.append(_to_dict(compiler)) + spack.config.update_config('compilers', compilers_dict, scope=scope) config = spack.config.get_config('compilers', scope=scope) - # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if arch == my_arch and arch not in config: + if not config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) elif scope == 'user': # Check the site config and update the user config if # nothing is configured at the site level. site_config = spack.config.get_config('compilers', scope='site') if not site_config: init_compiler_config() - - return config[arch] if arch in config else {} + config = spack.config.get_config('compilers', scope=scope) + return config + elif config: + return config + else: + return [] # Return empty list which we will later append to. -def add_compilers_to_config(compilers, arch=None, scope=None): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.sys_type() - - compiler_config = get_compiler_config(arch, scope) + compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config[str(compiler.spec)] = dict( - (c, getattr(compiler, c, "None")) - for c in _required_instance_vars) - - update = { arch : compiler_config } - spack.config.update_config('compilers', update, scope) + compiler_config.append(_to_dict(compiler)) + global _cache_config_file + _cache_config_file = compiler_config + spack.config.update_config('compilers', compiler_config, scope) @_auto_compiler_spec -def remove_compiler_from_config(compiler_spec, arch=None, scope=None): +def remove_compiler_from_config(compiler_spec, scope=None): """Remove compilers from the config, by spec. Arguments: - compiler_specs: a list of CompilerSpec objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.sys_type() - - compiler_config = get_compiler_config(arch, scope) - del compiler_config[str(compiler_spec)] - update = { arch : compiler_config } - - spack.config.update_config('compilers', update, scope) - - -def all_compilers_config(arch=None, scope=None): + compiler_config = get_compiler_config(scope) + config_length = len(compiler_config) + + filtered_compiler_config = [comp for comp in compiler_config + if spack.spec.CompilerSpec(comp['compiler']['spec']) != compiler_spec] + # Need a better way for this + global _cache_config_file + _cache_config_file = filtered_compiler_config # Update the cache for changes + if len(filtered_compiler_config) == config_length: # No items removed + CompilerSpecInsufficientlySpecificError(compiler_spec) + spack.config.update_config('compilers', filtered_compiler_config, scope) + + +def all_compilers_config(scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. - arch_config = get_compiler_config(arch, scope) - - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - merged_config = get_compiler_config('all', scope=scope) - merged_config = spack.config._merge_yaml(merged_config, arch_config) - - return merged_config + global _cache_config_file #Create a cache of the config file so we don't load all the time. + if not _cache_config_file: + _cache_config_file = get_compiler_config(scope) + return _cache_config_file + else: + return _cache_config_file -def all_compilers(arch=None, scope=None): +def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s) - for s in all_compilers_config(arch, scope)] + return [spack.spec.CompilerSpec(s['compiler']['spec']) + for s in all_compilers_config(scope)] def default_compiler(): @@ -179,36 +184,18 @@ def default_compiler(): return sorted(versions)[-1] -def find_compilers(*path): +def find_compilers(*paths): """Return a list of compilers found in the suppied paths. - This invokes the find() method for each Compiler class, - and appends the compilers detected to a list. + This invokes the find_compilers() method for each operating + system associated with the host platform, and appends + the compilers detected to a list. """ - # Make sure path elements exist, and include /bin directories - # under prefixes. - filtered_path = [] - for p in path: - # Eliminate symlinks and just take the real directories. - p = os.path.realpath(p) - if not os.path.isdir(p): - continue - filtered_path.append(p) - - # Check for a bin directory, add it if it exists - bin = join_path(p, 'bin') - if os.path.isdir(bin): - filtered_path.append(os.path.realpath(bin)) - - # Once the paths are cleaned up, do a search for each type of - # compiler. We can spawn a bunch of parallel searches to reduce - # the overhead of spelunking all these directories. - types = all_compiler_types() - compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types) - - # ensure all the version calls we made are cached in the parent - # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) - return clist + # Find compilers for each operating system class + oss = all_os_classes() + compiler_lists = [] + for o in oss: + compiler_lists.extend(o.find_compilers(*paths)) + return compiler_lists def supported_compilers(): @@ -227,47 +214,83 @@ def supported(compiler_spec): @_auto_compiler_spec -def find(compiler_spec, arch=None, scope=None): +def find(compiler_spec, scope=None): """Return specs of available compilers that match the supplied compiler spec. Return an list if nothing found.""" - return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] + return [c for c in all_compilers(scope) if c.satisfies(compiler_spec)] @_auto_compiler_spec -def compilers_for_spec(compiler_spec, arch=None, scope=None): +def compilers_for_spec(compiler_spec, scope=None, **kwargs): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = all_compilers_config(arch, scope) + platform = kwargs.get("platform", None) + config = all_compilers_config(scope) + + def get_compilers(cspec): + compilers = [] - def get_compiler(cspec): - items = config[str(cspec)] + for items in config: + if items['compiler']['spec'] != str(cspec): + continue + items = items['compiler'] - if not all(n in items for n in _required_instance_vars): - raise InvalidCompilerConfigurationError(cspec) + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): + raise InvalidCompilerConfigurationError(cspec) - cls = class_for_compiler_name(cspec.name) - compiler_paths = [] - for c in _required_instance_vars: - compiler_path = items[c] - if compiler_path != "None": - compiler_paths.append(compiler_path) + cls = class_for_compiler_name(cspec.name) + + compiler_paths = [] + for c in _path_instance_vars: + compiler_path = items['paths'][c] + if compiler_path != "None": + compiler_paths.append(compiler_path) + else: + compiler_paths.append(None) + + mods = items.get('modules') + if mods == 'None': + mods = [] + + if 'operating_system' in items: + operating_system = spack.architecture._operating_system_from_dict(items['operating_system'], platform) else: - compiler_paths.append(None) + operating_system = None + - return cls(cspec, *compiler_paths) + alias = items['alias'] if 'alias' in items else None - matches = find(compiler_spec, arch, scope) - return [get_compiler(cspec) for cspec in matches] + flags = {} + for f in spack.spec.FlagMap.valid_compiler_flags(): + if f in items: + flags[f] = items[f] + + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) + + return compilers + + matches = set(find(compiler_spec, scope)) + compilers = [] + for cspec in matches: + compilers.extend(get_compilers(cspec)) + return compilers +# return [get_compilers(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec): +def compiler_for_spec(compiler_spec, arch): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" + operating_system = arch.platform_os assert(compiler_spec.concrete) - compilers = compilers_for_spec(compiler_spec) - assert(len(compilers) == 1) + + compilers = [c for c in compilers_for_spec(compiler_spec, platform=arch.platform) + if c.operating_system == operating_system] + if len(compilers) < 1: + raise NoCompilerForSpecError(compiler_spec, operating_system) + if len(compilers) > 1: + raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] @@ -285,6 +308,19 @@ def class_for_compiler_name(compiler_name): return cls +def all_os_classes(): + """ + Return the list of classes for all operating systems available on + this platform + """ + classes = [] + + platform = spack.architecture.sys_type() + for os_class in platform.operating_sys.values(): + classes.append(os_class) + + return classes + def all_compiler_types(): return [class_for_compiler_name(c) for c in supported_compilers()] @@ -294,9 +330,19 @@ class InvalidCompilerConfigurationError(spack.error.SpackError): super(InvalidCompilerConfigurationError, self).__init__( "Invalid configuration for [compiler \"%s\"]: " % compiler_spec, "Compiler configuration must contain entries for all compilers: %s" - % _required_instance_vars) + % _path_instance_vars) class NoCompilersError(spack.error.SpackError): def __init__(self): super(NoCompilersError, self).__init__("Spack could not find any compilers!") + +class NoCompilerForSpecError(spack.error.SpackError): + def __init__(self, compiler_spec, target): + super(NoCompilerForSpecError, self).__init__("No compilers for operating system %s satisfy spec %s" % ( + target, compiler_spec)) + +class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): + def __init__(self, compiler_spec): + super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s", + compiler_spec) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 072bcd065f..00b406d820 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -73,7 +73,7 @@ class Clang(Compiler): return "-std=c++11" @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '--version' option works for clang compilers. On most platforms, output looks like this:: diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py new file mode 100644 index 0000000000..4ba8b110ec --- /dev/null +++ b/lib/spack/spack/compilers/craype.py @@ -0,0 +1,58 @@ +##############################################################################} +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty + +#from spack.build_environment import load_module +from spack.compiler import * +#from spack.version import ver + +class Craype(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + link_paths = { 'cc' : 'cc', + 'cxx' : 'c++', + 'f77' : 'f77', + 'fc' : 'fc'} + + @classmethod + def default_version(cls, comp): + return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') + diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 164bddeb3f..3f552eaece 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -49,6 +49,9 @@ class Gcc(Compiler): 'f77' : 'gcc/gfortran', 'fc' : 'gcc/gfortran' } + PrgEnv = 'PrgEnv-gnu' + PrgEnv_compiler = 'gcc' + @property def openmp_flag(self): return "-fopenmp" @@ -74,9 +77,9 @@ class Gcc(Compiler): return get_compiler_version( fc, '-dumpversion', # older gfortran versions don't have simple dumpversion output. - r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)') + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module) @classmethod def f77_version(cls, f77): - return cls.fc_version(f77) + return cls.fc_version(f77, module) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 5007ece645..6cad03ff47 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -45,6 +45,9 @@ class Intel(Compiler): 'f77' : 'intel/ifort', 'fc' : 'intel/ifort' } + PrgEnv = 'PrgEnv-intel' + PrgEnv_compiler = 'intel' + @property def openmp_flag(self): if self.version < ver('16.0'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d42148dc49..6d36d8bfa6 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -44,6 +44,12 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } + + + PrgEnv = 'PrgEnv-pgi' + PrgEnv_compiler = 'pgi' + + @property def openmp_flag(self): return "-mp" @@ -52,7 +58,6 @@ class Pgi(Compiler): def cxx11_flag(self): return "-std=c++11" - @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index bda2de4b87..b1431436ad 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -56,8 +56,9 @@ class Xl(Compiler): else: return "-qlanglvl=extended0x" + @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '-qversion' is the standard option fo XL compilers. Output looks like this:: @@ -83,6 +84,7 @@ class Xl(Compiler): return get_compiler_version( comp, '-qversion',r'([0-9]?[0-9]\.[0-9])') + @classmethod def fc_version(cls, fc): """The fortran and C/C++ versions of the XL compiler are always two units apart. diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f5e1c10b48..1f37455c77 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -44,6 +44,7 @@ from spec import DependencyMap from itertools import chain from spack.config import * + class DefaultConcretizer(object): """This class doesn't have any state, it just provides some methods for concretization. You can subclass it to override just some of the @@ -83,7 +84,8 @@ class DefaultConcretizer(object): raise NoBuildError(spec) def cmp_externals(a, b): - if a.name != b.name: + if a.name != b.name and (not a.external or a.external_module and + not b.external and b.external_module): # We're choosing between different providers, so # maintain order from provider sort return candidates.index(a) - candidates.index(b) @@ -186,31 +188,64 @@ class DefaultConcretizer(object): return True # Things changed + def _concretize_operating_system(self, spec): + platform = spec.architecture.platform + if spec.architecture.platform_os is not None and isinstance( + spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False - def concretize_architecture(self, spec): - """If the spec already had an architecture, return. Otherwise if - the root of the DAG has an architecture, then use that. - Otherwise take the system's default architecture. - - Intuition: Architectures won't be set a lot, and generally you - want the host system's architecture. When architectures are - mised in a spec, it is likely because the tool requries a - cross-compiled component, e.g. for tools that run on BlueGene - or Cray machines. These constraints will likely come directly - from packages, so require the user to be explicit if they want - to mess with the architecture, and revert to the default when - they're not explicit. - """ - if spec.architecture is not None: + if spec.root.architecture and spec.root.architecture.platform_os: + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + spec.architecture.platform_os = spec.root.architecture.platform_os + else: + spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') + return True #changed + + def _concretize_target(self, spec): + platform = spec.architecture.platform + if spec.architecture.target is not None and isinstance( + spec.architecture.target, spack.architecture.Target): return False + if spec.root.architecture and spec.root.architecture.target: + if isinstance(spec.root.architecture.target,spack.architecture.Target): + spec.architecture.target = spec.root.architecture.target + else: + spec.architecture.target = spec.architecture.platform.target('default_target') + return True #changed - if spec.root.architecture: - spec.architecture = spec.root.architecture + def _concretize_platform(self, spec): + if spec.architecture.platform is not None and isinstance( + spec.architecture.platform, spack.architecture.Platform): + return False + if spec.root.architecture and spec.root.architecture.platform: + if isinstance(spec.root.architecture.platform,spack.architecture.Platform): + spec.architecture.platform = spec.root.architecture.platform else: - spec.architecture = spack.architecture.sys_type() + spec.architecture.platform = spack.architecture.sys_type() + return True #changed? + + def concretize_architecture(self, spec): + """If the spec is empty provide the defaults of the platform. If the + architecture is not a basestring, then check if either the platform, + target or operating system are concretized. If any of the fields are + changed then return True. If everything is concretized (i.e the + architecture attribute is a namedtuple of classes) then return False. + If the target is a string type, then convert the string into a + concretized architecture. If it has no architecture and the root of the + DAG has an architecture, then use the root otherwise use the defaults + on the platform. + """ + if spec.architecture is None: + # Set the architecture to all defaults + spec.architecture = spack.architecture.Arch() + return True + + # Concretize the operating_system and target based of the spec + ret = any((self._concretize_platform(spec), + self._concretize_operating_system(spec), + self._concretize_target(spec))) + return ret - assert(spec.architecture is not None) - return True # changed def concretize_variants(self, spec): @@ -237,6 +272,23 @@ class DefaultConcretizer(object): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ + # Pass on concretizing the compiler if the target is not yet determined + if not spec.architecture.platform_os: + #Although this usually means changed, this means awaiting other changes + return True + + # Only use a matching compiler if it is of the proper style + # Takes advantage of the proper logic already existing in compiler_for_spec + # Should think whether this can be more efficient + def _proper_compiler_style(cspec, arch): + platform = arch.platform + compilers = spack.compilers.compilers_for_spec(cspec, + platform=platform) + return filter(lambda c: c.operating_system == + arch.platform_os, compilers) + #return compilers + + all_compilers = spack.compilers.all_compilers() if (spec.compiler and @@ -246,6 +298,7 @@ class DefaultConcretizer(object): #Find the another spec that has a compiler, or the root if none do other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) + if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler @@ -264,11 +317,75 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + index = 0 + while not _proper_compiler_style(matches[index], spec.architecture): + index += 1 + if index == len(matches) - 1: + raise NoValidVersionError(spec) + spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. + def concretize_compiler_flags(self, spec): + """ + The compiler flags are updated to match those of the spec whose + compiler is used, defaulting to no compiler flags in the spec. + Default specs set at the compiler level will still be added later. + """ + + + if not spec.architecture.platform_os: + #Although this usually means changed, this means awaiting other changes + return True + + ret = False + for flag in spack.spec.FlagMap.valid_compiler_flags(): + try: + nearest = next(p for p in spec.traverse(direction='parents') + if ((p.compiler == spec.compiler and p is not spec) + and flag in p.compiler_flags)) + if not flag in spec.compiler_flags or \ + not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): + if flag in spec.compiler_flags: + spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | + set(nearest.compiler_flags[flag])) + else: + spec.compiler_flags[flag] = nearest.compiler_flags[flag] + ret = True + + except StopIteration: + if (flag in spec.root.compiler_flags and ((not flag in spec.compiler_flags) or + sorted(spec.compiler_flags[flag]) != sorted(spec.root.compiler_flags[flag]))): + if flag in spec.compiler_flags: + spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | + set(spec.root.compiler_flags[flag])) + else: + spec.compiler_flags[flag] = spec.root.compiler_flags[flag] + ret = True + else: + if not flag in spec.compiler_flags: + spec.compiler_flags[flag] = [] + + # Include the compiler flag defaults from the config files + # This ensures that spack will detect conflicts that stem from a change + # in default compiler flags. + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) + for flag in compiler.flags: + if flag not in spec.compiler_flags: + spec.compiler_flags[flag] = compiler.flags[flag] + if compiler.flags[flag] != []: + ret = True + else: + if ((sorted(spec.compiler_flags[flag]) != sorted(compiler.flags[flag])) and + (not set(spec.compiler_flags[flag]) >= set(compiler.flags[flag]))): + ret = True + spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | + set(compiler.flags[flag])) + + return ret + + def find_spec(spec, condition): """Searches the dag from spec in an intelligent order and looks for a spec that matches a condition""" @@ -330,7 +447,6 @@ def cmp_specs(lhs, rhs): return 0 - class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec37bd290c..db0787edc6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -135,7 +135,7 @@ from yaml.error import MarkedYAMLError # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml - +from spack.build_environment import get_path_from_module """Dict from section names -> schema for that section.""" section_schemas = { @@ -146,18 +146,17 @@ section_schemas = { 'additionalProperties': False, 'patternProperties': { 'compilers:?': { # optional colon for overriding site config. - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*': { # architecture + 'type': 'array', + 'items': { + 'compiler': { 'type': 'object', 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*@\w[\w-]*': { # compiler spec + 'required': ['paths', 'spec', 'modules', 'operating_system'], + 'properties': { + 'paths': { 'type': 'object', - 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], + 'additionalProperties': False, 'properties': { 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, @@ -167,8 +166,27 @@ section_schemas = { {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, - },},},},},},},}, - + 'cflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxxflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cppflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldlibs': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}}}, + 'spec': { 'type': 'string'}, + 'operating_system': { 'type': 'string'}, + 'alias': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }]}, + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} + },},},},},}, 'mirrors': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack mirror configuration file schema', @@ -194,7 +212,6 @@ section_schemas = { 'default': [], 'items': { 'type': 'string'},},},}, - 'packages': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack package configuration file schema', @@ -224,6 +241,10 @@ section_schemas = { 'type': 'boolean', 'default': True, }, + 'modules': { + 'type' : 'object', + 'default' : {}, + }, 'providers': { 'type': 'object', 'default': {}, @@ -563,8 +584,7 @@ def _merge_yaml(dest, source): # Source list is prepended (for precedence) if they_are(list): - seen = set(source) - dest[:] = source + [x for x in dest if x not in seen] + dest[:] = source + [x for x in dest if x not in source] return dest # Source dict is merged into dest. @@ -667,7 +687,8 @@ def spec_externals(spec): external_specs = [] pkg_paths = allpkgs.get(name, {}).get('paths', None) - if not pkg_paths: + pkg_modules = allpkgs.get(name, {}).get('modules', None) + if (not pkg_paths) and (not pkg_modules): return [] for external_spec, path in pkg_paths.iteritems(): @@ -678,6 +699,17 @@ def spec_externals(spec): external_spec = spack.spec.Spec(external_spec, external=path) if external_spec.satisfies(spec): external_specs.append(external_spec) + + for external_spec, module in pkg_modules.iteritems(): + if not module: + continue + + path = get_path_from_module(module) + + external_spec = spack.spec.Spec(external_spec, external=path, external_module=module) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index ee4473e079..f941346bb1 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -40,7 +40,6 @@ filesystem. """ import os -import time import socket import yaml @@ -56,11 +55,12 @@ from spack.spec import Spec from spack.error import SpackError from spack.repository import UnknownPackageError + # DB goes in this directory underneath the root _db_dirname = '.spack-db' # DB version. This is stuck in the DB file to track changes in format. -_db_version = Version('0.9') +_db_version = Version('0.9.1') # Default timeout for spack database locks is 5 min. _db_lock_timeout = 60 @@ -69,10 +69,12 @@ _db_lock_timeout = 60 def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" + def converter(self, spec_like, *args, **kwargs): if not isinstance(spec_like, spack.spec.Spec): spec_like = spack.spec.Spec(spec_like) return function(self, spec_like, *args, **kwargs) + return converter @@ -92,22 +94,28 @@ class InstallRecord(object): dependents left. """ - def __init__(self, spec, path, installed, ref_count=0): + + def __init__(self, spec, path, installed, ref_count=0, explicit=False): self.spec = spec self.path = str(path) self.installed = bool(installed) self.ref_count = ref_count + self.explicit = explicit def to_dict(self): - return { 'spec' : self.spec.to_node_dict(), - 'path' : self.path, - 'installed' : self.installed, - 'ref_count' : self.ref_count } + return { + 'spec': self.spec.to_node_dict(), + 'path': self.path, + 'installed': self.installed, + 'ref_count': self.ref_count, + 'explicit': self.explicit + } @classmethod def from_dict(cls, spec, dictionary): d = dictionary - return InstallRecord(spec, d['path'], d['installed'], d['ref_count']) + return InstallRecord(spec, d['path'], d['installed'], d['ref_count'], + d.get('explicit', False)) class Database(object): @@ -142,7 +150,7 @@ class Database(object): # Set up layout of database files within the db dir self._index_path = join_path(self._db_dir, 'index.yaml') - self._lock_path = join_path(self._db_dir, 'lock') + self._lock_path = join_path(self._db_dir, 'lock') # Create needed directories and files if not os.path.exists(self._db_dir): @@ -155,17 +163,14 @@ class Database(object): self.lock = Lock(self._lock_path) self._data = {} - def write_transaction(self, timeout=_db_lock_timeout): """Get a write lock context manager for use in a `with` block.""" return WriteTransaction(self, self._read, self._write, timeout) - def read_transaction(self, timeout=_db_lock_timeout): """Get a read lock context manager for use in a `with` block.""" return ReadTransaction(self, self._read, None, timeout) - def _write_to_yaml(self, stream): """Write out the databsae to a YAML file. @@ -181,9 +186,9 @@ class Database(object): # different paths, it can't differentiate. # TODO: fix this before we support multiple install locations. database = { - 'database' : { - 'installs' : installs, - 'version' : str(_db_version) + 'database': { + 'installs': installs, + 'version': str(_db_version) } } @@ -192,32 +197,33 @@ class Database(object): except YAMLError as e: raise SpackYAMLError("error writing YAML database:", str(e)) - def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): """Recursively construct a spec from a hash in a YAML database. Does not do any locking. """ - if hash_key not in installs: - parent = read_spec(installs[parent_key]['path']) - spec_dict = installs[hash_key]['spec'] + # Install records don't include hash with spec, so we add it in here + # to ensure it is read properly. + for name in spec_dict: + spec_dict[name]['hash'] = hash_key + # Build spec from dict first. spec = Spec.from_node_dict(spec_dict) # Add dependencies from other records in the install DB to # form a full spec. - for dep_hash in spec_dict[spec.name]['dependencies'].values(): - child = self._read_spec_from_yaml(dep_hash, installs, hash_key) - spec._add_dependency(child) + if 'dependencies' in spec_dict[spec.name]: + for dep_hash in spec_dict[spec.name]['dependencies'].values(): + child = self._read_spec_from_yaml(dep_hash, installs, hash_key) + spec._add_dependency(child) # Specs from the database need to be marked concrete because # they represent actual installations. spec._mark_concrete() return spec - def _read_from_yaml(self, stream): """ Fill database from YAML, do not maintain old data @@ -239,22 +245,27 @@ class Database(object): return def check(cond, msg): - if not cond: raise CorruptDatabaseError(self._index_path, msg) + if not cond: + raise CorruptDatabaseError(self._index_path, msg) check('database' in yfile, "No 'database' attribute in YAML.") # High-level file checks db = yfile['database'] check('installs' in db, "No 'installs' in YAML DB.") - check('version' in db, "No 'version' in YAML DB.") + check('version' in db, "No 'version' in YAML DB.") + + installs = db['installs'] # TODO: better version checking semantics. version = Version(db['version']) - if version != _db_version: + if version > _db_version: raise InvalidDatabaseVersionError(_db_version, version) + elif version < _db_version: + self.reindex(spack.install_layout) + installs = dict((k, v.to_dict()) for k, v in self._data.items()) # Iterate through database and check each record. - installs = db['installs'] data = {} for hash_key, rec in installs.items(): try: @@ -265,25 +276,26 @@ class Database(object): # hashes are the same. spec_hash = spec.dag_hash() if not spec_hash == hash_key: - tty.warn("Hash mismatch in database: %s -> spec with hash %s" - % (hash_key, spec_hash)) - continue # TODO: is skipping the right thing to do? + tty.warn( + "Hash mismatch in database: %s -> spec with hash %s" % + (hash_key, spec_hash)) + continue # TODO: is skipping the right thing to do? # Insert the brand new spec in the database. Each # spec has its own copies of its dependency specs. - # TODO: would a more immmutable spec implementation simplify this? + # TODO: would a more immmutable spec implementation simplify + # this? data[hash_key] = InstallRecord.from_dict(spec, rec) except Exception as e: tty.warn("Invalid database reecord:", "file: %s" % self._index_path, "hash: %s" % hash_key, - "cause: %s" % str(e)) + "cause: %s: %s" % (type(e).__name__, str(e))) raise self._data = data - def reindex(self, directory_layout): """Build database index from scratch based from a directory layout. @@ -299,7 +311,11 @@ class Database(object): for spec in directory_layout.all_specs(): # Create a spec for each known package and add it. path = directory_layout.path_for_spec(spec) - self._add(spec, path, directory_layout) + old_info = old_data.get(spec.dag_hash()) + explicit = False + if old_info is not None: + explicit = old_info.explicit + self._add(spec, path, directory_layout, explicit=explicit) self._check_ref_counts() @@ -308,7 +324,6 @@ class Database(object): self._data = old_data raise - def _check_ref_counts(self): """Ensure consistency of reference counts in the DB. @@ -330,9 +345,8 @@ class Database(object): found = rec.ref_count if not expected == found: raise AssertionError( - "Invalid ref_count: %s: %d (expected %d), in DB %s" - % (key, found, expected, self._index_path)) - + "Invalid ref_count: %s: %d (expected %d), in DB %s" % + (key, found, expected, self._index_path)) def _write(self): """Write the in-memory database index to its file path. @@ -354,7 +368,6 @@ class Database(object): os.remove(temp_file) raise - def _read(self): """Re-read Database from the data in the set location. @@ -369,8 +382,7 @@ class Database(object): # reindex() takes its own write lock, so no lock here. self.reindex(spack.install_layout) - - def _add(self, spec, path, directory_layout=None): + def _add(self, spec, path, directory_layout=None, explicit=False): """Add an install record for spec at path to the database. This assumes that the spec is not already installed. It @@ -392,11 +404,11 @@ class Database(object): rec.path = path else: - self._data[key] = InstallRecord(spec, path, True) + self._data[key] = InstallRecord(spec, path, True, + explicit=explicit) for dep in spec.dependencies.values(): self._increment_ref_count(dep, directory_layout) - def _increment_ref_count(self, spec, directory_layout=None): """Recursively examine dependencies and update their DB entries.""" key = spec.dag_hash() @@ -415,7 +427,7 @@ class Database(object): self._data[key].ref_count += 1 @_autospec - def add(self, spec, path): + def add(self, spec, path, explicit=False): """Add spec at path to database, locking and reading DB to sync. ``add()`` will lock and read from the DB on disk. @@ -424,30 +436,27 @@ class Database(object): # TODO: ensure that spec is concrete? # Entire add is transactional. with self.write_transaction(): - self._add(spec, path) - + self._add(spec, path, explicit=explicit) def _get_matching_spec_key(self, spec, **kwargs): """Get the exact spec OR get a single spec that matches.""" key = spec.dag_hash() - if not key in self._data: + if key not in self._data: match = self.query_one(spec, **kwargs) if match: return match.dag_hash() raise KeyError("No such spec in database! %s" % spec) return key - @_autospec def get_record(self, spec, **kwargs): key = self._get_matching_spec_key(spec, **kwargs) return self._data[key] - def _decrement_ref_count(self, spec): key = spec.dag_hash() - if not key in self._data: + if key not in self._data: # TODO: print something here? DB is corrupt, but # not much we can do. return @@ -460,7 +469,6 @@ class Database(object): for dep in spec.dependencies.values(): self._decrement_ref_count(dep) - def _remove(self, spec): """Non-locking version of remove(); does real work. """ @@ -479,7 +487,6 @@ class Database(object): # query spec was passed in. return rec.spec - @_autospec def remove(self, spec): """Removes a spec from the database. To be called on uninstall. @@ -496,7 +503,6 @@ class Database(object): with self.write_transaction(): return self._remove(spec) - @_autospec def installed_extensions_for(self, extendee_spec): """ @@ -507,13 +513,12 @@ class Database(object): try: if s.package.extends(extendee_spec): yield s.package - except UnknownPackageError as e: + except UnknownPackageError: continue # skips unknown packages # TODO: conditional way to do this instead of catching exceptions - - def query(self, query_spec=any, known=any, installed=True): + def query(self, query_spec=any, known=any, installed=True, explicit=any): """Run a query on the database. ``query_spec`` @@ -553,14 +558,16 @@ class Database(object): for key, rec in self._data.items(): if installed is not any and rec.installed != installed: continue - if known is not any and spack.repo.exists(rec.spec.name) != known: + if explicit is not any and rec.explicit != explicit: + continue + if known is not any and spack.repo.exists( + rec.spec.name) != known: continue if query_spec is any or rec.spec.satisfies(query_spec): results.append(rec.spec) return sorted(results) - def query_one(self, query_spec, known=any, installed=True): """Query for exactly one spec that matches the query spec. @@ -572,10 +579,9 @@ class Database(object): assert len(concrete_specs) <= 1 return concrete_specs[0] if concrete_specs else None - def missing(self, spec): with self.read_transaction(): - key = spec.dag_hash() + key = spec.dag_hash() return key in self._data and not self._data[key].installed @@ -587,7 +593,10 @@ class _Transaction(object): Timeout for lock is customizable. """ - def __init__(self, db, acquire_fn=None, release_fn=None, + + def __init__(self, db, + acquire_fn=None, + release_fn=None, timeout=_db_lock_timeout): self._db = db self._timeout = timeout @@ -622,11 +631,11 @@ class WriteTransaction(_Transaction): class CorruptDatabaseError(SpackError): def __init__(self, path, msg=''): super(CorruptDatabaseError, self).__init__( - "Spack database is corrupt: %s. %s" %(path, msg)) + "Spack database is corrupt: %s. %s" % (path, msg)) class InvalidDatabaseVersionError(SpackError): def __init__(self, expected, found): super(InvalidDatabaseVersionError, self).__init__( - "Expected database version %s but found version %s" - % (expected, found)) + "Expected database version %s but found version %s" % + (expected, found)) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 74ee7b0add..ca8f21dc08 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -45,11 +45,8 @@ The available directives are: * ``resource`` """ -__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', - 'variant', 'resource'] import re -import inspect import os.path import functools @@ -67,6 +64,9 @@ from spack.spec import Spec, parse_anonymous_spec from spack.resource import Resource from spack.fetch_strategy import from_kwargs +__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', 'variant', + 'resource'] + # # This is a list of all directives, built up as they are defined in # this file. @@ -122,15 +122,14 @@ class directive(object): def __init__(self, dicts=None): if isinstance(dicts, basestring): - dicts = (dicts,) + dicts = (dicts, ) elif type(dicts) not in (list, tuple): raise TypeError( - "dicts arg must be list, tuple, or string. Found %s" - % type(dicts)) + "dicts arg must be list, tuple, or string. Found %s" % + type(dicts)) self.dicts = dicts - def ensure_dicts(self, pkg): """Ensure that a package has the dicts required by this directive.""" for d in self.dicts: @@ -142,7 +141,6 @@ class directive(object): raise spack.error.SpackError( "Package %s has non-dict %s attribute!" % (pkg, d)) - def __call__(self, directive_function): directives[directive_function.__name__] = self @@ -259,11 +257,12 @@ def variant(pkg, name, default=False, description=""): """Define a variant for the package. Packager can specify a default value (on or off) as well as a text description.""" - default = bool(default) + default = default description = str(description).strip() if not re.match(spack.spec.identifier_re, name): - raise DirectiveError("Invalid variant name in %s: '%s'" % (pkg.name, name)) + raise DirectiveError("Invalid variant name in %s: '%s'" % + (pkg.name, name)) pkg.variants[name] = Variant(default, description) @@ -271,31 +270,37 @@ def variant(pkg, name, default=False, description=""): @directive('resources') def resource(pkg, **kwargs): """ - Define an external resource to be fetched and staged when building the package. Based on the keywords present in the - dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their - own folder inside spack stage area, and then linked into the stage area of the package that needs them. + Define an external resource to be fetched and staged when building the + package. Based on the keywords present in the dictionary the appropriate + FetchStrategy will be used for the resource. Resources are fetched and + staged in their own folder inside spack stage area, and then linked into + the stage area of the package that needs them. List of recognized keywords: - * 'when' : (optional) represents the condition upon which the resource is needed - * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage - area. - * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage - area. + * 'when' : (optional) represents the condition upon which the resource is + needed + * 'destination' : (optional) path where to link the resource. This path + must be relative to the main package stage area. + * 'placement' : (optional) gives the possibility to fine tune how the + resource is linked into the main package stage area. """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") placement = kwargs.get('placement', None) # Check if the path is relative if os.path.isabs(destination): - message = "The destination keyword of a resource directive can't be an absolute path.\n" + message = "The destination keyword of a resource directive can't be" + " an absolute path.\n" message += "\tdestination : '{dest}\n'".format(dest=destination) raise RuntimeError(message) # Check if the path falls within the main package stage area - test_path = 'stage_folder_root/' - normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path + test_path = 'stage_folder_root' + normalized_destination = os.path.normpath(join_path(test_path, destination) + ) # Normalized absolute path if test_path not in normalized_destination: - message = "The destination folder of a resource must fall within the main package stage directory.\n" + message = "The destination folder of a resource must fall within the" + " main package stage directory.\n" message += "\tdestination : '{dest}'\n".format(dest=destination) raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) @@ -307,6 +312,7 @@ def resource(pkg, **kwargs): class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" + def __init__(self, directive, message): super(DirectiveError, self).__init__(message) self.directive = directive @@ -314,6 +320,7 @@ class DirectiveError(spack.error.SpackError): class CircularReferenceError(DirectiveError): """This is raised when something depends on itself.""" + def __init__(self, directive, package): super(CircularReferenceError, self).__init__( directive, diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 32d27d7bd0..7e20365b0f 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -165,7 +165,7 @@ class DirectoryLayout(object): class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: <install root>/ - <architecture>/ + <target>/ <compiler>-<compiler version>/ <name>-<version>-<variants>-<hash> @@ -207,8 +207,7 @@ class YamlDirectoryLayout(DirectoryLayout): spec.version, spec.dag_hash(self.hash_len)) - path = join_path( - spec.architecture, + path = join_path(spec.architecture, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 11998ad8d2..30c6228ca4 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,4 +1,4 @@ -############################################################################## +# # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -21,14 +21,17 @@ # You should have received a copy of the GNU Lesser General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## +# import collections import inspect +import json import os import os.path +import subprocess class NameModifier(object): + def __init__(self, name, **kwargs): self.name = name self.args = {'name': name} @@ -36,57 +39,68 @@ class NameModifier(object): class NameValueModifier(object): + def __init__(self, name, value, **kwargs): self.name = name self.value = value - self.args = {'name': name, 'value': value} + self.separator = kwargs.get('separator', ':') + self.args = {'name': name, 'value': value, 'delim': self.separator} self.args.update(kwargs) class SetEnv(NameValueModifier): + def execute(self): os.environ[self.name] = str(self.value) class UnsetEnv(NameModifier): + def execute(self): # Avoid throwing if the variable was not set os.environ.pop(self.name, None) class SetPath(NameValueModifier): + def execute(self): - string_path = concatenate_paths(self.value) + string_path = concatenate_paths(self.value, separator=self.separator) os.environ[self.name] = string_path class AppendPath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') - directories = environment_value.split(':') if environment_value else [] + directories = environment_value.split( + self.separator) if environment_value else [] directories.append(os.path.normpath(self.value)) - os.environ[self.name] = ':'.join(directories) + os.environ[self.name] = self.separator.join(directories) class PrependPath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') - directories = environment_value.split(':') if environment_value else [] + directories = environment_value.split( + self.separator) if environment_value else [] directories = [os.path.normpath(self.value)] + directories - os.environ[self.name] = ':'.join(directories) + os.environ[self.name] = self.separator.join(directories) class RemovePath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') - directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) - for x in directories + directories = environment_value.split( + self.separator) if environment_value else [] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] - os.environ[self.name] = ':'.join(directories) + os.environ[self.name] = self.separator.join(directories) class EnvironmentModifications(object): + """ Keeps track of requests to modify the current environment. @@ -237,18 +251,140 @@ class EnvironmentModifications(object): for x in actions: x.execute() + @staticmethod + def from_sourcing_files(*args, **kwargs): + """ + Creates an instance of EnvironmentModifications that, if executed, + has the same effect on the environment as sourcing the files passed as + parameters + + Args: + *args: list of files to be sourced -def concatenate_paths(paths): + Returns: + instance of EnvironmentModifications + """ + env = EnvironmentModifications() + # Check if the files are actually there + if not all(os.path.isfile(file) for file in args): + raise RuntimeError('trying to source non-existing files') + # Relevant kwd parameters and formats + info = dict(kwargs) + info.setdefault('shell', '/bin/bash') + info.setdefault('shell_options', '-c') + info.setdefault('source_command', 'source') + info.setdefault('suppress_output', '&> /dev/null') + info.setdefault('concatenate_on_success', '&&') + + shell = '{shell}'.format(**info) + shell_options = '{shell_options}'.format(**info) + source_file = '{source_command} {file} {concatenate_on_success}' + dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' # NOQA: ignore=E501 + # Construct the command that will be executed + command = [source_file.format(file=file, **info) for file in args] + command.append(dump_environment) + command = ' '.join(command) + command = [ + shell, + shell_options, + command + ] + + # Try to source all the files, + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, env=os.environ) + proc.wait() + if proc.returncode != 0: + raise RuntimeError('sourcing files returned a non-zero exit code') + output = ''.join([line for line in proc.stdout]) + # Construct a dictionary with all the variables in the new environment + after_source_env = dict(json.loads(output)) + this_environment = dict(os.environ) + + # Filter variables that are not related to sourcing a file + to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD' + for d in after_source_env, this_environment: + for name in to_be_filtered: + d.pop(name, None) + + # Fill the EnvironmentModifications instance + + # New variables + new_variables = set(after_source_env) - set(this_environment) + for x in new_variables: + env.set(x, after_source_env[x]) + # Variables that have been unset + unset_variables = set(this_environment) - set(after_source_env) + for x in unset_variables: + env.unset(x) + # Variables that have been modified + common_variables = set(this_environment).intersection(set(after_source_env)) # NOQA: ignore=E501 + modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501 + + def return_separator_if_any(first_value, second_value): + separators = ':', ';' + for separator in separators: + if separator in first_value and separator in second_value: + return separator + return None + + for x in modified_variables: + current = this_environment[x] + modified = after_source_env[x] + sep = return_separator_if_any(current, modified) + if sep is None: + # We just need to set the variable to the new value + env.set(x, after_source_env[x]) + else: + current_list = current.split(sep) + modified_list = modified.split(sep) + # Paths that have been removed + remove_list = [ + ii for ii in current_list if ii not in modified_list] + # Check that nothing has been added in the middle of vurrent + # list + remaining_list = [ + ii for ii in current_list if ii in modified_list] + start = modified_list.index(remaining_list[0]) + end = modified_list.index(remaining_list[-1]) + search = sep.join(modified_list[start:end + 1]) + if search not in current: + # We just need to set the variable to the new value + env.set(x, after_source_env[x]) + break + else: + try: + prepend_list = modified_list[:start] + except KeyError: + prepend_list = [] + try: + append_list = modified_list[end + 1:] + except KeyError: + append_list = [] + + for item in remove_list: + env.remove_path(x, item) + for item in append_list: + env.append_path(x, item) + for item in prepend_list: + env.prepend_path(x, item) + + return env + + +def concatenate_paths(paths, separator=':'): """ - Concatenates an iterable of paths into a string of column separated paths + Concatenates an iterable of paths into a string of paths separated by + separator, defaulting to colon Args: paths: iterable of paths + separator: the separator to use, default ':' Returns: string """ - return ':'.join(str(item) for item in paths) + return separator.join(str(item) for item in paths) def set_or_unset_not_first(variable, changes, errstream): @@ -256,16 +392,13 @@ def set_or_unset_not_first(variable, changes, errstream): Check if we are going to set or unset something after other modifications have already been requested """ - indexes = [ii - for ii, item in enumerate(changes) + indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] if indexes: good = '\t \t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}' message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501 - errstream( - message.format( - var=variable)) + errstream(message.format(var=variable)) for ii, item in enumerate(changes): print_format = nogood if ii in indexes else good errstream(print_format.format(**item.args)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index e05cb13c1e..1953d7c1b3 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -1,4 +1,4 @@ -############################################################################## +# # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -21,7 +21,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## +# """ Fetch strategies are used to download source code into a staging area in order to build it. They need to define the following methods: @@ -57,7 +57,6 @@ from spack.version import Version, ver from spack.util.compression import decompressor_for, extension import spack.util.pattern as pattern - """List of all fetch strategies, created by FetchStrategy metaclass.""" all_strategies = [] @@ -76,19 +75,24 @@ def _needs_stage(fun): class FetchStrategy(object): + """Superclass of all fetch strategies.""" enabled = False # Non-abstract subclasses should be enabled. required_attributes = None # Attributes required in version() args. class __metaclass__(type): + """This metaclass registers all fetch strategies in a list.""" + def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.enabled: all_strategies.append(cls) + if cls.enabled: + all_strategies.append(cls) def __init__(self): - # The stage is initialized late, so that fetch strategies can be constructed - # at package construction time. This is where things will be fetched. + # The stage is initialized late, so that fetch strategies can be + # constructed at package construction time. This is where things + # will be fetched. self.stage = None def set_stage(self, stage): @@ -97,15 +101,20 @@ class FetchStrategy(object): self.stage = stage # Subclasses need to implement these methods - def fetch(self): pass # Return True on success, False on fail. + def fetch(self): + pass # Return True on success, False on fail. - def check(self): pass # Do checksum. + def check(self): + pass # Do checksum. - def expand(self): pass # Expand archive. + def expand(self): + pass # Expand archive. - def reset(self): pass # Revert to freshly downloaded state. + def reset(self): + pass # Revert to freshly downloaded state. - def archive(self, destination): pass # Used to create tarball for mirror. + def archive(self, destination): + pass # Used to create tarball for mirror. def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @@ -119,6 +128,7 @@ class FetchStrategy(object): @pattern.composite(interface=FetchStrategy) class FetchStrategyComposite(object): + """ Composite for a FetchStrategy object. Implements the GoF composite pattern. """ @@ -127,6 +137,7 @@ class FetchStrategyComposite(object): class URLFetchStrategy(FetchStrategy): + """FetchStrategy that pulls source code from a URL for an archive, checks the archive against a checksum,and decompresses the archive. """ @@ -139,10 +150,12 @@ class URLFetchStrategy(FetchStrategy): # If URL or digest are provided in the kwargs, then prefer # those values. self.url = kwargs.get('url', None) - if not self.url: self.url = url + if not self.url: + self.url = url self.digest = kwargs.get('md5', None) - if not self.digest: self.digest = digest + if not self.digest: + self.digest = digest self.expand_archive = kwargs.get('expand', True) @@ -167,16 +180,20 @@ class URLFetchStrategy(FetchStrategy): tty.msg("Trying to fetch from %s" % self.url) if partial_file: - save_args = ['-C', '-', # continue partial downloads - '-o', partial_file] # use a .part file + save_args = ['-C', + '-', # continue partial downloads + '-o', + partial_file] # use a .part file else: save_args = ['-O'] curl_args = save_args + [ - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', # resolve 3xx redirects - self.url, ] + '-f', # fail on >400 errors + '-D', + '-', # print out HTML headers + '-L', # resolve 3xx redirects + self.url, + ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty @@ -184,8 +201,7 @@ class URLFetchStrategy(FetchStrategy): curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers - headers = spack.curl( - *curl_args, output=str, fail_on_error=False) + headers = spack.curl(*curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -198,34 +214,38 @@ class URLFetchStrategy(FetchStrategy): if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( - self.url, "URL %s was not found!" % self.url) + self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( - self.url, - "Curl was unable to fetch due to invalid certificate. " - "This is either an attack, or your cluster's SSL configuration " - "is bad. If you believe your SSL configuration is bad, you " - "can try running spack -k, which will not check SSL certificates." - "Use this at your own risk.") + self.url, + "Curl was unable to fetch due to invalid certificate. " + "This is either an attack, or your cluster's SSL " + "configuration is bad. If you believe your SSL " + "configuration is bad, you can try running spack -k, " + "which will not check SSL certificates." + "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( - self.url, "Curl failed with error %d" % spack.curl.returncode) + self.url, + "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle # redirects properly. content_types = re.findall(r'Content-Type:[^\r\n]+', headers) if content_types and 'text/html' in content_types[-1]: - tty.warn("The contents of " + self.archive_file + " look like HTML.", + tty.warn("The contents of ", + (self.archive_file if self.archive_file is not None + else "the archive"), + " look like HTML.", "The checksum will likely be bad. If it is, you can use", - "'spack clean <package>' to remove the bad archive, then fix", - "your internet gateway issue and install again.") - + "'spack clean <package>' to remove the bad archive, then", + "fix your internet gateway issue and install again.") if save_file: os.rename(partial_file, save_file) @@ -247,14 +267,16 @@ class URLFetchStrategy(FetchStrategy): self.stage.chdir() if not self.archive_file: - raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", - "Failed on expand() for URL %s" % self.url) + raise NoArchiveFileError( + "URLFetchStrategy couldn't find archive file", + "Failed on expand() for URL %s" % self.url) decompress = decompressor_for(self.archive_file) # Expand all tarballs in their own directory to contain # exploding tarballs. - tarball_container = os.path.join(self.stage.path, "spack-expanded-archive") + tarball_container = os.path.join(self.stage.path, + "spack-expanded-archive") mkdirp(tarball_container) os.chdir(tarball_container) decompress(self.archive_file) @@ -295,20 +317,25 @@ class URLFetchStrategy(FetchStrategy): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" if not self.digest: - raise NoDigestError("Attempt to check URLFetchStrategy with no digest.") + raise NoDigestError( + "Attempt to check URLFetchStrategy with no digest.") checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s" % (checker.hash_name, self.archive_file), - "Expected %s but got %s" % (self.digest, checker.sum)) + "%s checksum failed for %s" % + (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): - """Removes the source path if it exists, then re-expands the archive.""" + """ + Removes the source path if it exists, then re-expands the archive. + """ if not self.archive_file: - raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching", - "Failed on reset() for URL %s" % self.url) + raise NoArchiveFileError( + "Tried to reset URLFetchStrategy before fetching", + "Failed on reset() for URL %s" % self.url) # Remove everythigng but the archive from the stage for filename in os.listdir(self.stage.path): @@ -331,20 +358,23 @@ class URLFetchStrategy(FetchStrategy): class VCSFetchStrategy(FetchStrategy): + def __init__(self, name, *rev_types, **kwargs): super(VCSFetchStrategy, self).__init__() self.name = name # Set a URL based on the type of fetch strategy. self.url = kwargs.get(name, None) - if not self.url: raise ValueError( + if not self.url: + raise ValueError( "%s requires %s argument." % (self.__class__, name)) # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s" % ( - comma_or(rev_types), name)) + "Supply only one of %s to fetch with %s" % ( + comma_or(rev_types), name + )) # Set attributes for each rev type. for rt in rev_types: @@ -382,32 +412,95 @@ class VCSFetchStrategy(FetchStrategy): return "%s<%s>" % (self.__class__, self.url) +class GoFetchStrategy(VCSFetchStrategy): + + """ + Fetch strategy that employs the `go get` infrastructure + Use like this in a package: + + version('name', + go='github.com/monochromegane/the_platinum_searcher/...') + + Go get does not natively support versions, they can be faked with git + """ + enabled = True + required_attributes = ('go', ) + + def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next + # call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + + super(GoFetchStrategy, self).__init__('go', **forwarded_args) + self._go = None + + @property + def go_version(self): + vstring = self.go('version', output=str).split(' ')[2] + return Version(vstring) + + @property + def go(self): + if not self._go: + self._go = which('go', required=True) + return self._go + + @_needs_stage + def fetch(self): + self.stage.chdir() + + tty.msg("Trying to get go resource:", self.url) + + try: + os.mkdir('go') + except OSError: + pass + env = dict(os.environ) + env['GOPATH'] = os.path.join(os.getcwd(), 'go') + self.go('get', '-v', '-d', self.url, env=env) + + def archive(self, destination): + super(GoFetchStrategy, self).archive(destination, exclude='.git') + + @_needs_stage + def reset(self): + self.stage.chdir_to_source() + self.go('clean') + + def __str__(self): + return "[go] %s" % self.url + + class GitFetchStrategy(VCSFetchStrategy): - """Fetch strategy that gets source code from a git repository. - Use like this in a package: - version('name', git='https://github.com/project/repo.git') + """ + Fetch strategy that gets source code from a git repository. + Use like this in a package: + + version('name', git='https://github.com/project/repo.git') - Optionally, you can provide a branch, or commit to check out, e.g.: + Optionally, you can provide a branch, or commit to check out, e.g.: - version('1.1', git='https://github.com/project/repo.git', tag='v1.1') + version('1.1', git='https://github.com/project/repo.git', tag='v1.1') - You can use these three optional attributes in addition to ``git``: + You can use these three optional attributes in addition to ``git``: - * ``branch``: Particular branch to build from (default is master) - * ``tag``: Particular tag to check out - * ``commit``: Particular commit hash in the repo + * ``branch``: Particular branch to build from (default is master) + * ``tag``: Particular tag to check out + * ``commit``: Particular commit hash in the repo """ enabled = True - required_attributes = ('git',) + required_attributes = ('git', ) def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **forwarded_args) + 'git', 'tag', 'branch', 'commit', **forwarded_args) self._git = None @property @@ -501,6 +594,7 @@ class GitFetchStrategy(VCSFetchStrategy): class SvnFetchStrategy(VCSFetchStrategy): + """Fetch strategy that gets source code from a subversion repository. Use like this in a package: @@ -515,12 +609,13 @@ class SvnFetchStrategy(VCSFetchStrategy): required_attributes = ['svn'] def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **forwarded_args) + 'svn', 'revision', **forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) @@ -576,32 +671,36 @@ class SvnFetchStrategy(VCSFetchStrategy): class HgFetchStrategy(VCSFetchStrategy): - """Fetch strategy that gets source code from a Mercurial repository. - Use like this in a package: - version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2') + """ + Fetch strategy that gets source code from a Mercurial repository. + Use like this in a package: + + version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2') - Optionally, you can provide a branch, or revision to check out, e.g.: + Optionally, you can provide a branch, or revision to check out, e.g.: - version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus') + version('torus', + hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus') - You can use the optional 'revision' attribute to check out a - branch, tag, or particular revision in hg. To prevent - non-reproducible builds, using a moving target like a branch is - discouraged. + You can use the optional 'revision' attribute to check out a + branch, tag, or particular revision in hg. To prevent + non-reproducible builds, using a moving target like a branch is + discouraged. - * ``revision``: Particular revision, branch, or tag. + * ``revision``: Particular revision, branch, or tag. """ enabled = True required_attributes = ['hg'] def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **forwarded_args) + 'hg', 'revision', **forwarded_args) self._hg = None @property @@ -675,7 +774,8 @@ def from_kwargs(**kwargs): return fetcher(**kwargs) # Raise an error in case we can't instantiate any known strategy message = "Cannot instantiate any FetchStrategy" - long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs) + long_message = message + " from the given arguments : {arguments}".format( + srguments=kwargs) raise FetchError(message, long_message) @@ -687,7 +787,7 @@ def for_package_version(pkg, version): """Determine a fetch strategy based on the arguments supplied to version() in the package description.""" # If it's not a known version, extrapolate one. - if not version in pkg.versions: + if version not in pkg.versions: url = pkg.url_for_version(version) if not url: raise InvalidArgsError(pkg, version) @@ -716,37 +816,44 @@ def for_package_version(pkg, version): class FetchError(spack.error.SpackError): + def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) class FailedDownloadError(FetchError): + """Raised wen a download fails.""" def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( - "Failed to fetch file from URL: %s" % url, msg) + "Failed to fetch file from URL: %s" % url, msg) self.url = url class NoArchiveFileError(FetchError): + def __init__(self, msg, long_msg): super(NoArchiveFileError, self).__init__(msg, long_msg) class NoDigestError(FetchError): + def __init__(self, msg, long_msg=None): super(NoDigestError, self).__init__(msg, long_msg) class InvalidArgsError(FetchError): + def __init__(self, pkg, version): - msg = "Could not construct a fetch strategy for package %s at version %s" + msg = ("Could not construct a fetch strategy for package %s at " + "version %s") msg %= (pkg.name, version) super(InvalidArgsError, self).__init__(msg) class ChecksumError(FetchError): + """Raised when archive fails to checksum.""" def __init__(self, message, long_msg=None): @@ -754,8 +861,10 @@ class ChecksumError(FetchError): class NoStageError(FetchError): + """Raised when fetch operations are called before set_stage().""" def __init__(self, method): super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) + "Must call FetchStrategy.set_stage() before calling %s" % + method.__name__) diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index 0f63b0e05a..9010b84154 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -26,7 +26,7 @@ import os import spack import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import join_path, mkdirp def pre_install(pkg): @@ -154,6 +154,9 @@ def symlink_license(pkg): target = pkg.global_license_file for filename in pkg.license_files: link_name = join_path(pkg.prefix, filename) + license_dir = os.path.dirname(link_name) + if not os.path.exists(license_dir): + mkdirp(license_dir) if os.path.exists(target): os.symlink(target, link_name) tty.msg("Added local symlink %s to global license file" % diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 83d67ea225..cb0ad42b14 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -24,7 +24,6 @@ ############################################################################## import os -from llnl.util.filesystem import * import llnl.util.tty as tty import spack @@ -34,6 +33,7 @@ import spack.modules # here, as it is the shortest I could find on a modern OS. shebang_limit = 127 + def shebang_too_long(path): """Detects whether a file has a shebang line that is too long.""" with open(path, 'r') as script: @@ -57,16 +57,10 @@ def filter_shebang(path): if original.startswith(new_sbang_line): return - backup = path + ".shebang.bak" - os.rename(path, backup) - with open(path, 'w') as new_file: new_file.write(new_sbang_line) new_file.write(original) - copy_mode(backup, path) - unset_executable_mode(backup) - tty.warn("Patched overly long shebang in %s" % path) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index a35e21c3db..d2b819e80a 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -485,9 +485,9 @@ class TclModule(EnvModule): path = join_path(spack.share_path, "modules") environment_modifications_formats = { - PrependPath: 'prepend-path {name} \"{value}\"\n', - AppendPath: 'append-path {name} \"{value}\"\n', - RemovePath: 'remove-path {name} \"{value}\"\n', + PrependPath: 'prepend-path --delim "{delim}" {name} \"{value}\"\n', + AppendPath: 'append-path --delim "{delim}" {name} \"{value}\"\n', + RemovePath: 'remove-path --delim "{delim}" {name} \"{value}\"\n', SetEnv: 'setenv {name} \"{value}\"\n', UnsetEnv: 'unsetenv {name}\n' } diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 5fda9328d6..170ef3cea2 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -146,12 +146,12 @@ class when(object): def install(self, prefix): # Do default install - @when('=chaos_5_x86_64_ib') + @when('arch=chaos_5_x86_64_ib') def install(self, prefix): # This will be executed instead of the default install if # the package's sys_type() is chaos_5_x86_64_ib. - @when('=bgqos_0") + @when('arch=bgqos_0") def install(self, prefix): # This will be executed if the package's sys_type is bgqos_0 diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/operating_systems/__init__.py diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py new file mode 100644 index 0000000000..c160a60be8 --- /dev/null +++ b/lib/spack/spack/operating_systems/cnl.py @@ -0,0 +1,62 @@ +import re +import os + +from spack.architecture import OperatingSystem +from spack.util.executable import * +import spack.spec +from spack.util.multiproc import parmap +import spack.compilers + +class Cnl(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(Cnl, self).__init__(name, version) + + + def find_compilers(self, *paths): + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + + def find_compiler(self, cmp_cls, *paths): + compilers = [] + if cmp_cls.PrgEnv: + if not cmp_cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Save the environment variable to restore later + old_modulepath = os.environ['MODULEPATH'] + # if given any explicit paths, search them for module files too + if paths: + module_paths = ':' + ':'.join(p for p in paths) + os.environ['MODULEPATH'] = module_paths + + output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + for name, version in matches: + v = version + comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + + compilers.append(comp) + + # Restore modulepath environment variable + if paths: + os.environ['MODULEPATH'] = old_modulepath + + return compilers diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py new file mode 100644 index 0000000000..2e3c72719b --- /dev/null +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -0,0 +1,22 @@ +import re +import platform as py_platform +from spack.architecture import OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + distname, version, _ = py_platform.linux_distribution( + full_distribution_name=False) + + # Grabs major version from tuple on redhat; on other platforms + # grab the first legal identifier in the version field. On + # debian you get things like 'wheezy/sid'; sid means unstable. + # We just record 'wheezy' and don't get quite so detailed. + version = re.split(r'[^\w-]', version)[0] + + super(LinuxDistro, self).__init__(distname, version) diff --git a/lib/spack/spack/operating_systems/mac_os.py b/lib/spack/spack/operating_systems/mac_os.py new file mode 100644 index 0000000000..f35b3ca577 --- /dev/null +++ b/lib/spack/spack/operating_systems/mac_os.py @@ -0,0 +1,29 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class MacOs(OperatingSystem): + """This class represents the macOS operating system. This will be + auto detected using the python platform.mac_ver. The macOS + platform will be represented using the major version operating + system name, i.e el capitan, yosemite...etc. + """ + + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan", + "10.12": "sierra"} + + mac_ver = py_platform.mac_ver()[0][:-2] + name = mac_releases.get(mac_ver, "macos") + super(MacOs, self).__init__(name, mac_ver) + + def __str__(self): + return self.name diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8167341127..98fd51b262 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -398,13 +398,19 @@ class Package(object): spack.repo.get(self.extendee_spec)._check_extendable() @property + def global_license_dir(self): + """Returns the directory where global license files for all + packages are stored.""" + spack_root = ancestor(__file__, 4) + return join_path(spack_root, 'etc', 'spack', 'licenses') + + @property def global_license_file(self): - """Returns the path where a global license file should be stored.""" + """Returns the path where a global license file for this + particular package should be stored.""" if not self.license_files: return - spack_root = ancestor(__file__, 4) - global_license_dir = join_path(spack_root, 'etc', 'spack', 'licenses') - return join_path(global_license_dir, self.name, + return join_path(self.global_license_dir, self.name, os.path.basename(self.license_files[0])) @property @@ -676,11 +682,13 @@ class Package(object): return self.spec.prefix @property + #TODO: Change this to architecture def compiler(self): """Get the spack.compiler.Compiler object used to build this package""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler) + return spack.compilers.compiler_for_spec(self.spec.compiler, + self.spec.architecture) def url_version(self, version): """ @@ -857,7 +865,8 @@ class Package(object): skip_patch=False, verbose=False, make_jobs=None, - fake=False): + fake=False, + explicit=False): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -887,6 +896,11 @@ class Package(object): # Ensure package is not already installed if spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) + rec = spack.installed_db.get_record(self.spec) + if (not rec.explicit) and explicit: + with spack.installed_db.write_transaction(): + rec = spack.installed_db.get_record(self.spec) + rec.explicit = True return tty.msg("Installing %s" % self.name) @@ -995,7 +1009,7 @@ class Package(object): # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + spack.installed_db.add(self.spec, self.prefix, explicit=explicit) def sanity_check_prefix(self): """This function checks whether install succeeded.""" diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/platforms/__init__.py diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py new file mode 100644 index 0000000000..e0eb76f336 --- /dev/null +++ b/lib/spack/spack/platforms/bgq.py @@ -0,0 +1,18 @@ +import os +from spack.architecture import Platform, Target + +class Bgq(Platform): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('bgq') + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end,)) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py new file mode 100644 index 0000000000..e710303e23 --- /dev/null +++ b/lib/spack/spack/platforms/cray_xc.py @@ -0,0 +1,46 @@ +import os +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + +class CrayXc(Platform): + priority = 20 + front_end = 'sandybridge' + back_end = 'ivybridge' + default = 'ivybridge' + + front_os = "SuSE11" + back_os = "CNL10" + default_os = "CNL10" + + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' + super(CrayXc, self).__init__('cray_xc') + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default + + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge', Target('sandybridge')) + self.add_target('ivybridge', + Target('ivybridge', 'craype-ivybridge')) + self.add_target('haswell', + Target('haswell','craype-haswell')) + + self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', Cnl()) + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py new file mode 100644 index 0000000000..d47dd640f9 --- /dev/null +++ b/lib/spack/spack/platforms/darwin.py @@ -0,0 +1,26 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.mac_os import MacOs + +class Darwin(Platform): + priority = 89 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Darwin, self).__init__('darwin') + self.add_target(self.default, Target(self.default)) + mac_os = MacOs() + + self.default_os = str(mac_os) + self.front_os = str(mac_os) + self.back_os = str(mac_os) + + self.add_operating_system(str(mac_os), mac_os) + + @classmethod + def detect(self): + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'darwin' in platform.strip().lower() diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py new file mode 100644 index 0000000000..4d8adac384 --- /dev/null +++ b/lib/spack/spack/platforms/linux.py @@ -0,0 +1,24 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro + +class Linux(Platform): + priority = 90 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Linux, self).__init__('linux') + self.add_target(self.default, Target(self.default)) + linux_dist = LinuxDistro() + self.default_os = str(linux_dist) + self.front_os = self.default_os + self.back_os = self.default_os + self.add_operating_system(str(linux_dist), linux_dist) + + @classmethod + def detect(self): + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'linux' in platform.strip().lower() diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py new file mode 100644 index 0000000000..8fa2585a7a --- /dev/null +++ b/lib/spack/spack/platforms/test.py @@ -0,0 +1,28 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + + +class Test(Platform): + priority = 1000000 + front_end = 'x86_32' + back_end = 'x86_64' + default = 'x86_64' + + back_os = 'CNL10' + default_os = 'CNL10' + + def __init__(self): + super(Test, self).__init__('test') + self.add_target(self.default, Target(self.default)) + self.add_target(self.front_end, Target(self.front_end)) + + self.add_operating_system(self.default_os, Cnl()) + linux_dist = LinuxDistro() + self.front_os = linux_dist.name + self.add_operating_system(self.front_os, linux_dist) + + @classmethod + def detect(self): + return True diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 89a023a750..54219ec1b4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -18,9 +18,9 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and # conditions of the GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """ Spack allows very fine-grained control over how packages are installed and @@ -72,7 +72,9 @@ Here is the EBNF grammar for a spec:: dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =architecture } + %compiler | arch=architecture | [ flag ]=value} + flag = { cflags | cxxflags | fcflags | fflags | cppflags | + ldflags | ldlibs } variant = id architecture = id compiler = id [ version-list ] @@ -80,6 +82,9 @@ Here is the EBNF grammar for a spec:: version = id | id: | :id | id:id id = [A-Za-z0-9_][A-Za-z0-9_.-]* +Identifiers using the <name>=<value> command, such as architectures and +compiler flags, require a space before the name. + There is one context-sensitive part: ids in versions may contain '.', while other ids may not. @@ -94,6 +99,7 @@ import sys import itertools import hashlib import base64 +import imp from StringIO import StringIO from operator import attrgetter import yaml @@ -102,39 +108,45 @@ from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.lang import * from llnl.util.tty.color import * +from llnl.util.filesystem import join_path import spack +import spack.architecture import spack.parse import spack.error import spack.compilers as compilers +# TODO: move display_specs to some other location. +from spack.cmd.find import display_specs from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix +from spack.util.naming import mod_to_class from spack.virtual import ProviderIndex +from spack.build_environment import get_path_from_module, load_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' # Convenient names for color formats so that other things can use them -compiler_color = '@g' -version_color = '@c' -architecture_color = '@m' -enabled_variant_color = '@B' +compiler_color = '@g' +version_color = '@c' +architecture_color = '@m' +enabled_variant_color = '@B' disabled_variant_color = '@r' -dependency_color = '@.' -hash_color = '@K' +dependency_color = '@.' +hash_color = '@K' """This map determines the coloring of specs when using color output. We make the fields different colors to enhance readability. See spack.color for descriptions of the color codes. """ -color_formats = {'%' : compiler_color, - '@' : version_color, - '=' : architecture_color, - '+' : enabled_variant_color, - '~' : disabled_variant_color, - '^' : dependency_color, - '#' : hash_color } +color_formats = {'%': compiler_color, + '@': version_color, + '=': architecture_color, + '+': enabled_variant_color, + '~': disabled_variant_color, + '^': dependency_color, + '#': hash_color} """Regex used for splitting by spec field separators.""" _separators = '[%s]' % ''.join(color_formats.keys()) @@ -151,7 +163,7 @@ def index_specs(specs): """ spec_dict = {} for spec in specs: - if not spec.name in spec_dict: + if spec.name not in spec_dict: spec_dict[spec.name] = [] spec_dict[spec.name].append(spec) return spec_dict @@ -198,8 +210,8 @@ class CompilerSpec(object): else: raise TypeError( - "Can only build CompilerSpec from string or CompilerSpec." + - " Found %s" % type(arg)) + "Can only build CompilerSpec from string or " + + "CompilerSpec. Found %s" % type(arg)) elif nargs == 2: name, version = args @@ -211,23 +223,19 @@ class CompilerSpec(object): raise TypeError( "__init__ takes 1 or 2 arguments. (%d given)" % nargs) - def _add_version(self, version): self.versions.add(version) - def _autospec(self, compiler_spec_like): if isinstance(compiler_spec_like, CompilerSpec): return compiler_spec_like return CompilerSpec(compiler_spec_like) - def satisfies(self, other, strict=False): other = self._autospec(other) return (self.name == other.name and self.versions.satisfies(other.versions, strict=strict)) - def constrain(self, other): """Intersect self's versions with other. @@ -241,44 +249,37 @@ class CompilerSpec(object): return self.versions.intersect(other.versions) - @property def concrete(self): """A CompilerSpec is concrete if its versions are concrete and there is an available compiler with the right version.""" return self.versions.concrete - @property def version(self): if not self.concrete: raise SpecError("Spec is not concrete: " + str(self)) return self.versions[0] - def copy(self): clone = CompilerSpec.__new__(CompilerSpec) clone.name = self.name clone.versions = self.versions.copy() return clone - def _cmp_key(self): return (self.name, self.versions) - def to_dict(self): - d = {'name' : self.name} + d = {'name': self.name} d.update(self.versions.to_dict()) - return { 'compiler' : d } - + return {'compiler': d} @staticmethod def from_dict(d): d = d['compiler'] return CompilerSpec(d['name'], VersionList.from_dict(d)) - def __str__(self): out = self.name if self.versions and self.versions != _any_version: @@ -292,43 +293,43 @@ class CompilerSpec(object): @key_ordering class VariantSpec(object): + """Variants are named, build-time options for a package. Names depend on the particular package being built, and each named variant can be enabled or disabled. """ - def __init__(self, name, enabled): + def __init__(self, name, value): self.name = name - self.enabled = enabled - + self.value = value def _cmp_key(self): - return (self.name, self.enabled) - + return (self.name, self.value) def copy(self): - return VariantSpec(self.name, self.enabled) - + return VariantSpec(self.name, self.value) def __str__(self): - out = '+' if self.enabled else '~' - return out + self.name + if self.value in [True, False]: + out = '+' if self.value else '~' + return out + self.name + else: + return ' ' + self.name + "=" + self.value class VariantMap(HashableMap): + def __init__(self, spec): super(VariantMap, self).__init__() self.spec = spec - def satisfies(self, other, strict=False): if strict or self.spec._concrete: - return all(k in self and self[k].enabled == other[k].enabled + return all(k in self and self[k].value == other[k].value for k in other) else: - return all(self[k].enabled == other[k].enabled + return all(self[k].value == other[k].value for k in other if k in self) - def constrain(self, other): """Add all variants in other that aren't in self to self. @@ -343,11 +344,11 @@ class VariantMap(HashableMap): changed = False for k in other: if k in self: - if self[k].enabled != other[k].enabled: + if self[k].value != other[k].value: raise UnsatisfiableVariantSpecError(self[k], other[k]) else: self[k] = other[k].copy() - changed =True + changed = True return changed @property @@ -355,27 +356,92 @@ class VariantMap(HashableMap): return self.spec._concrete or all( v in self for v in self.spec.package_class.variants) - def copy(self): clone = VariantMap(None) for name, variant in self.items(): clone[name] = variant.copy() return clone - def __str__(self): sorted_keys = sorted(self.keys()) return ''.join(str(self[key]) for key in sorted_keys) +_valid_compiler_flags = [ + 'cflags', 'cxxflags', 'fflags', 'ldflags', 'ldlibs', 'cppflags'] + + +class FlagMap(HashableMap): + + def __init__(self, spec): + super(FlagMap, self).__init__() + self.spec = spec + + def satisfies(self, other, strict=False): + if strict or (self.spec and self.spec._concrete): + return all(f in self and set(self[f]) <= set(other[f]) + for f in other) + else: + return all(set(self[f]) <= set(other[f]) + for f in other if (other[f] != [] and f in self)) + + def constrain(self, other): + """Add all flags in other that aren't in self to self. + + Return whether the spec changed. + """ + if other.spec and other.spec._concrete: + for k in self: + if k not in other: + raise UnsatisfiableCompilerFlagSpecError( + self[k], '<absent>') + + changed = False + for k in other: + if k in self and not set(self[k]) <= set(other[k]): + raise UnsatisfiableCompilerFlagSpecError( + ' '.join(f for f in self[k]), + ' '.join(f for f in other[k])) + elif k not in self: + self[k] = other[k] + changed = True + return changed + + @staticmethod + def valid_compiler_flags(): + return _valid_compiler_flags + + @property + def concrete(self): + return all(flag in self for flag in _valid_compiler_flags) + + def copy(self): + clone = FlagMap(None) + for name, value in self.items(): + clone[name] = value + return clone + + def _cmp_key(self): + return ''.join(str(key) + ' '.join(str(v) for v in value) + for key, value in sorted(self.items())) + + def __str__(self): + sorted_keys = filter( + lambda flag: self[flag] != [], sorted(self.keys())) + cond_symbol = ' ' if len(sorted_keys) > 0 else '' + return cond_symbol + ' '.join(str(key) + '=\"' + ' '.join(str(f) + for f in self[key]) + '\"' + for key in sorted_keys) + + class DependencyMap(HashableMap): + """Each spec has a DependencyMap containing specs for its dependencies. The DependencyMap is keyed by name. """ @property def concrete(self): return all(d.concrete for d in self.values()) - def __str__(self): return ''.join( ["^" + str(self[name]) for name in sorted(self.keys())]) @@ -383,6 +449,7 @@ class DependencyMap(HashableMap): @key_ordering class Spec(object): + def __init__(self, spec_like, *dep_like, **kwargs): # Copy if spec_like is a Spec. if isinstance(spec_like, Spec): @@ -409,20 +476,24 @@ class Spec(object): self.versions = other.versions self.architecture = other.architecture self.compiler = other.compiler + self.compiler_flags = other.compiler_flags + self.compiler_flags.spec = self self.dependencies = other.dependencies self.variants = other.variants self.variants.spec = self self.namespace = other.namespace + self._hash = other._hash # Specs are by default not assumed to be normal, but in some # cases we've read them from a file want to assume normal. # This allows us to manipulate specs that Spack doesn't have # package.py files for. - self._normal = kwargs.get('normal', False) + self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) # Allow a spec to be constructed with an external path. self.external = kwargs.get('external', None) + self.external_module = kwargs.get('external_module', None) # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -431,7 +502,6 @@ class Spec(object): spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - # # Private routines here are called by the parser when building a spec. # @@ -439,32 +509,111 @@ class Spec(object): """Called by the parser to add an allowable version.""" self.versions.add(version) - - def _add_variant(self, name, enabled): + def _add_variant(self, name, value): """Called by the parser to add a variant.""" - if name in self.variants: raise DuplicateVariantError( + if name in self.variants: + raise DuplicateVariantError( "Cannot specify variant '%s' twice" % name) - self.variants[name] = VariantSpec(name, enabled) - + if isinstance(value, basestring) and value.upper() == 'TRUE': + value = True + elif isinstance(value, basestring) and value.upper() == 'FALSE': + value = False + self.variants[name] = VariantSpec(name, value) + + def _add_flag(self, name, value): + """Called by the parser to add a known flag. + Known flags currently include "arch" + """ + valid_flags = FlagMap.valid_compiler_flags() + if name == 'arch' or name == 'architecture': + parts = value.split('-') + if len(parts) == 3: + platform, op_sys, target = parts + else: + platform, op_sys, target = None, None, value + + assert(self.architecture.platform is None) + assert(self.architecture.platform_os is None) + assert(self.architecture.target is None) + assert(self.architecture.os_string is None) + assert(self.architecture.target_string is None) + self._set_platform(platform) + self._set_os(op_sys) + self._set_target(target) + elif name == 'platform': + self._set_platform(value) + elif name == 'os' or name == 'operating_system': + if self.architecture.platform: + self._set_os(value) + else: + self.architecture.os_string = value + elif name == 'target': + if self.architecture.platform: + self._set_target(value) + else: + self.architecture.target_string = value + elif name in valid_flags: + assert(self.compiler_flags is not None) + self.compiler_flags[name] = value.split() + else: + self._add_variant(name, value) def _set_compiler(self, compiler): """Called by the parser to set the compiler.""" - if self.compiler: raise DuplicateCompilerSpecError( + if self.compiler: + raise DuplicateCompilerSpecError( "Spec for '%s' cannot have two compilers." % self.name) self.compiler = compiler + def _set_platform(self, value): + """Called by the parser to set the architecture platform""" + if isinstance(value, basestring): + mod_path = spack.platform_path + mod_string = 'spack.platformss' + names = list_modules(mod_path) + if value in names: + # Create a platform object from the name + mod_name = mod_string + value + path = join_path(mod_path, value) + '.py' + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(value) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + platform = cls() + else: + tty.die("No platform class %s defined." % value) + else: + # The value is a platform + platform = value + + self.architecture.platform = platform + + # Set os and target if we previously got strings for them + if self.architecture.os_string: + self._set_os(self.architecture.os_string) + self.architecture.os_string = None + if self.architecture.target_string: + self._set_target(self.architecture.target_string) + self.architecture.target_string = None - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_os(self, value): + """Called by the parser to set the architecture operating system""" + if self.architecture.platform: + self.architecture.platform_os = self.architecture.platform.operating_system(value) + def _set_target(self, value): + """Called by the parser to set the architecture target""" + if self.architecture.platform: + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" if spec.name in self.dependencies: - raise DuplicateDependencyError("Cannot depend on '%s' twice" % spec) + raise DuplicateDependencyError( + "Cannot depend on '%s' twice" % spec) self.dependencies[spec.name] = spec spec.dependents[self.name] = self @@ -473,8 +622,8 @@ class Spec(object): # @property def fullname(self): - return '%s.%s' % (self.namespace, self.name) if self.namespace else self.name - + return (('%s.%s' % (self.namespace, self.name)) if self.namespace else + (self.name if self.name else '')) @property def root(self): @@ -494,12 +643,10 @@ class Spec(object): assert(all(first_root is d.root for d in depiter)) return first_root - @property def package(self): return spack.repo.get(self) - @property def package_class(self): """Internal package call gets only the class object for a package. @@ -507,7 +654,6 @@ class Spec(object): """ return spack.repo.get_pkg_class(self.name) - @property def virtual(self): """Right now, a spec is virtual if no package exists with its name. @@ -519,12 +665,10 @@ class Spec(object): """ return Spec.is_virtual(self.name) - @staticmethod def is_virtual(name): """Test if a name is virtual without requiring a Spec.""" - return not spack.repo.exists(name) - + return (name is not None) and (not spack.repo.exists(name)) @property def concrete(self): @@ -540,12 +684,12 @@ class Spec(object): and self.versions.concrete and self.variants.concrete and self.architecture + and self.architecture.concrete and self.compiler and self.compiler.concrete + and self.compiler_flags.concrete and self.dependencies.concrete) - return self._concrete - def traverse(self, visited=None, d=0, **kwargs): """Generic traversal of the DAG represented by this spec. This will yield each node in the spec. Options: @@ -581,7 +725,7 @@ class Spec(object): in the traversal. root [=True] - If false, this won't yield the root node, just its descendents. + If False, this won't yield the root node, just its descendents. direction [=children|parents] If 'children', does a traversal of this spec's children. If @@ -589,14 +733,14 @@ class Spec(object): """ # get initial values for kwargs - depth = kwargs.get('depth', False) - key_fun = kwargs.get('key', id) + depth = kwargs.get('depth', False) + key_fun = kwargs.get('key', id) if isinstance(key_fun, basestring): key_fun = attrgetter(key_fun) yield_root = kwargs.get('root', True) - cover = kwargs.get('cover', 'nodes') - direction = kwargs.get('direction', 'children') - order = kwargs.get('order', 'pre') + cover = kwargs.get('cover', 'nodes') + direction = kwargs.get('direction', 'children') + order = kwargs.get('order', 'pre') # Make sure kwargs have legal values; raise ValueError if not. def validate(name, val, allowed_values): @@ -633,47 +777,50 @@ class Spec(object): visited.add(key) for name in sorted(successors): child = successors[name] - for elt in child.traverse(visited, d+1, **kwargs): + for elt in child.traverse(visited, d + 1, **kwargs): yield elt # Postorder traversal yields after successors if yield_me and order == 'post': yield result - @property def short_spec(self): """Returns a version of the spec with the dependencies hashed instead of completely enumerated.""" return self.format('$_$@$%@$+$=$#') - @property def cshort_spec(self): """Returns a version of the spec with the dependencies hashed instead of completely enumerated.""" return self.format('$_$@$%@$+$=$#', color=True) - @property def prefix(self): return Prefix(spack.install_layout.path_for_spec(self)) - def dag_hash(self, length=None): """ Return a hash of the entire spec DAG, including connectivity. """ - yaml_text = yaml.dump( - self.to_node_dict(), default_flow_style=True, width=sys.maxint) - sha = hashlib.sha1(yaml_text) - return base64.b32encode(sha.digest()).lower()[:length] - + if self._hash: + return self._hash[:length] + else: + yaml_text = yaml.dump( + self.to_node_dict(), default_flow_style=True, width=sys.maxint) + sha = hashlib.sha1(yaml_text) + b32_hash = base64.b32encode(sha.digest()).lower()[:length] + if self.concrete: + self._hash = b32_hash + return b32_hash def to_node_dict(self): + params = dict((name, v.value) for name, v in self.variants.items()) + params.update(dict((name, value) + for name, value in self.compiler_flags.items())) d = { - 'variants' : dict( - (name,v.enabled) for name, v in self.variants.items()), + 'parameters' : params, 'arch' : self.architecture, 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) @@ -684,13 +831,20 @@ class Spec(object): if not self.concrete or self.namespace: d['namespace'] = self.namespace + if self.architecture: + # TODO: Fix the target.to_dict to account for the tuple + # Want it to be a dict of dicts + d['arch'] = self.architecture.to_dict() + else: + d['arch'] = None + if self.compiler: d.update(self.compiler.to_dict()) else: d['compiler'] = None d.update(self.versions.to_dict()) - return { self.name : d } + return {self.name: d} def to_yaml(self, stream=None): node_list = [] @@ -698,10 +852,9 @@ class Spec(object): node = s.to_node_dict() node[s.name]['hash'] = s.dag_hash() node_list.append(node) - return yaml.dump({ 'spec' : node_list }, + return yaml.dump({'spec': node_list}, stream=stream, default_flow_style=False) - @staticmethod def from_node_dict(node): name = next(iter(node)) @@ -710,19 +863,34 @@ class Spec(object): spec = Spec(name) spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) - spec.architecture = node['arch'] + + if 'hash' in node: + spec._hash = node['hash'] + + spec.architecture = spack.architecture.arch_from_dict(node['arch']) if node['compiler'] is None: spec.compiler = None else: spec.compiler = CompilerSpec.from_dict(node) - for name, enabled in node['variants'].items(): - spec.variants[name] = VariantSpec(name, enabled) + if 'parameters' in node: + for name, value in node['parameters'].items(): + if name in _valid_compiler_flags: + spec.compiler_flags[name] = value + else: + spec.variants[name] = VariantSpec(name, value) + elif 'variants' in node: + for name, value in node['variants'].items(): + spec.variants[name] = VariantSpec(name, value) + for name in FlagMap.valid_compiler_flags(): + spec.compiler_flags[name] = [] + else: + raise SpackRecordError( + "Did not find a valid format for variants in YAML file") return spec - @staticmethod def from_yaml(stream): """Construct a spec from YAML. @@ -755,15 +923,16 @@ class Spec(object): deps[name].dependencies[dep_name] = deps[dep_name] return spec - def _concretize_helper(self, presets=None, visited=None): """Recursive helper function for concretize(). This concretizes everything bottom-up. As things are concretized, they're added to the presets, and ancestors will prefer the settings of their children. """ - if presets is None: presets = {} - if visited is None: visited = set() + if presets is None: + presets = {} + if visited is None: + visited = set() if self.name in visited: return False @@ -776,22 +945,23 @@ class Spec(object): if self.name in presets: changed |= self.constrain(presets[self.name]) - else: # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. - changed |= any( - (spack.concretizer.concretize_architecture(self), - spack.concretizer.concretize_compiler(self), - spack.concretizer.concretize_version(self), - spack.concretizer.concretize_variants(self))) + if not self.virtual: + changed |= any( + (spack.concretizer.concretize_architecture(self), + spack.concretizer.concretize_compiler(self), + spack.concretizer.concretize_compiler_flags( + self), # has to be concretized after compiler + spack.concretizer.concretize_version(self), + spack.concretizer.concretize_variants(self))) presets[self.name] = self visited.add(self.name) return changed - def _replace_with(self, concrete): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) @@ -803,7 +973,6 @@ class Spec(object): if concrete.name not in dependent.dependencies: dependent._add_dependency(concrete) - def _replace_node(self, replacement): """Replace this spec with another. @@ -821,7 +990,6 @@ class Spec(object): del dep.dependents[self.name] del self.dependencies[dep.name] - def _expand_virtual_packages(self): """Find virtual packages in this spec, replace them with providers, and normalize again to include the provider's (potentially virtual) @@ -841,11 +1009,12 @@ class Spec(object): """ # Make an index of stuff this spec already provides self_index = ProviderIndex(self.traverse(), restrict=True) - changed = False done = False + while not done: done = True + for spec in list(self.traverse()): replacement = None if spec.virtual: @@ -854,12 +1023,14 @@ class Spec(object): # TODO: may break if in-place on self but # shouldn't happen if root is traversed first. spec._replace_with(replacement) - done=False + done = False break if not replacement: - # Get a list of possible replacements in order of preference. - candidates = spack.concretizer.choose_virtual_or_external(spec) + # Get a list of possible replacements in order of + # preference. + candidates = spack.concretizer.choose_virtual_or_external( + spec) # Try the replacements in order, skipping any that cause # satisfiability problems. @@ -872,20 +1043,22 @@ class Spec(object): copy[spec.name]._dup(replacement.copy(deps=False)) try: - # If there are duplicate providers or duplicate provider - # deps, consolidate them and merge constraints. + # If there are duplicate providers or duplicate + # provider deps, consolidate them and merge + # constraints. copy.normalize(force=True) break - except SpecError as e: + except SpecError: # On error, we'll try the next replacement. continue # If replacement is external then trim the dependencies - if replacement.external: + if replacement.external or replacement.external_module: if (spec.dependencies): changed = True spec.dependencies = DependencyMap() replacement.dependencies = DependencyMap() + replacement.architecture = self.architecture # TODO: could this and the stuff in _dup be cleaned up? def feq(cfield, sfield): @@ -897,9 +1070,9 @@ class Spec(object): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external)): + feq(replacement.external, spec.external) and + feq(replacement.external_module, spec.external_module)): continue - # Refine this spec to the candidate. This uses # replace_with AND dup so that it can work in # place. TODO: make this more efficient. @@ -910,12 +1083,11 @@ class Spec(object): changed = True self_index.update(spec) - done=False + done = False break return changed - def concretize(self): """A spec is concrete if it describes one build of a package uniquely. This will ensure that this spec is concrete. @@ -924,10 +1096,12 @@ class Spec(object): of a package, this will add constraints to make it concrete. Some rigorous validation and checks are also performed on the spec. - Concretizing ensures that it is self-consistent and that it's consistent - with requirements of its pacakges. See flatten() and normalize() for - more details on this. + Concretizing ensures that it is self-consistent and that it's + consistent with requirements of its pacakges. See flatten() and + normalize() for more details on this. """ + if not self.name: + raise SpecError("Attempting to concretize anonymous spec") if self._concrete: return @@ -940,7 +1114,7 @@ class Spec(object): self._expand_virtual_packages(), self._concretize_helper()) changed = any(changes) - force=True + force = True for s in self.traverse(): # After concretizing, assign namespaces to anything left. @@ -954,10 +1128,18 @@ class Spec(object): if s.namespace is None: s.namespace = spack.repo.repo_for_pkg(s.name).namespace + + for s in self.traverse(root=False): + if s.external_module: + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture) + for mod in compiler.modules: + load_module(mod) + + s.external = get_path_from_module(s.external_module) + # Mark everything in the spec as concrete, as well. self._mark_concrete() - def _mark_concrete(self): """Mark this spec and its dependencies as concrete. @@ -968,7 +1150,6 @@ class Spec(object): s._normal = True s._concrete = True - def concretized(self): """This is a non-destructive version of concretize(). First clones, then returns a concrete version of this package without modifying @@ -977,7 +1158,6 @@ class Spec(object): clone.concretize() return clone - def flat_dependencies(self, **kwargs): """Return a DependencyMap containing all of this spec's dependencies with their constraints merged. @@ -1016,7 +1196,6 @@ class Spec(object): # parser doesn't allow it. Spack must be broken! raise InconsistentSpecError("Invalid Spec DAG: %s" % e.message) - def index(self): """Return DependencyMap that points to all the dependencies in this spec.""" @@ -1025,7 +1204,6 @@ class Spec(object): dm[spec.name] = spec return dm - def flatten(self): """Pull all dependencies up to the root (this spec). Merge constraints for dependencies with the same name, and if they @@ -1033,7 +1211,6 @@ class Spec(object): for dep in self.flat_dependencies(copy=False): self._add_dependency(dep) - def _evaluate_dependency_conditions(self, name): """Evaluate all the conditions on a dependency with this name. @@ -1054,12 +1231,11 @@ class Spec(object): try: dep.constrain(dep_spec) except UnsatisfiableSpecError, e: - e.message = ("Conflicting conditional dependencies on package " - "%s for spec %s" % (self.name, self)) + e.message = ("Conflicting conditional dependencies on" + "package %s for spec %s" % (self.name, self)) raise e return dep - def _find_provider(self, vdep, provider_index): """Find provider for a virtual spec in the provider index. Raise an exception if there is a conflicting virtual @@ -1071,6 +1247,12 @@ class Spec(object): # If there is a provider for the vpkg, then use that instead of # the virtual package. if providers: + # Remove duplicate providers that can concretize to the same + # result. + for provider in providers: + for spec in providers: + if spec is not provider and provider.satisfies(spec): + providers.remove(spec) # Can't have multiple providers for the same thing in one spec. if len(providers) > 1: raise MultipleProviderError(vdep, providers) @@ -1085,11 +1267,10 @@ class Spec(object): elif required: raise UnsatisfiableProviderSpecError(required[0], vdep) - def _merge_dependency(self, dep, visited, spec_deps, provider_index): """Merge the dependency into this spec. - This is the core of the normalize() method. There are a few basic steps: + This is the core of normalize(). There are some basic steps: * If dep is virtual, evaluate whether it corresponds to an existing concrete dependency, and merge if so. @@ -1123,19 +1304,17 @@ class Spec(object): if required: raise UnsatisfiableProviderSpecError(required[0], dep) provider_index.update(dep) - # If the spec isn't already in the set of dependencies, clone # it from the package description. if dep.name not in spec_deps: spec_deps[dep.name] = dep.copy() changed = True - # Constrain package information with spec info try: changed |= spec_deps[dep.name].constrain(dep) except UnsatisfiableSpecError, e: - e.message = "Invalid spec: '%s'. " + e.message = "Invalid spec: '%s'. " e.message += "Package %s requires %s %s, but spec asked for %s" e.message %= (spec_deps[dep.name], dep.name, e.constraint_type, e.required, e.provided) @@ -1146,10 +1325,10 @@ class Spec(object): if dep.name not in self.dependencies: self._add_dependency(dependency) - changed |= dependency._normalize_helper(visited, spec_deps, provider_index) + changed |= dependency._normalize_helper( + visited, spec_deps, provider_index) return changed - def _normalize_helper(self, visited, spec_deps, provider_index): """Recursive helper function for _normalize.""" if self.name in visited: @@ -1158,7 +1337,7 @@ class Spec(object): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual or self.external: + if self.virtual or self.external or self.external_module: return False # Combine constraints from package deps with constraints from @@ -1172,7 +1351,6 @@ class Spec(object): for dep_name in pkg.dependencies: # Do we depend on dep_name? If so pkg_dep is not None. pkg_dep = self._evaluate_dependency_conditions(dep_name) - # If pkg_dep is a dependency, merge it. if pkg_dep: changed |= self._merge_dependency( @@ -1181,30 +1359,31 @@ class Spec(object): return any_change - def normalize(self, force=False): """When specs are parsed, any dependencies specified are hanging off the root, and ONLY the ones that were explicitly provided are there. Normalization turns a partial flat spec into a DAG, where: 1. Known dependencies of the root package are in the DAG. - 2. Each node's dependencies dict only contains its known direct deps. + 2. Each node's dependencies dict only contains its known direct + deps. 3. There is only ONE unique spec for each package in the DAG. * This includes virtual packages. If there a non-virtual package that provides a virtual package that is in the spec, then we replace the virtual package with the non-virtual one. - TODO: normalize should probably implement some form of cycle detection, - to ensure that the spec is actually a DAG. - + TODO: normalize should probably implement some form of cycle + detection, to ensure that the spec is actually a DAG. """ + if not self.name: + raise SpecError("Attempting to normalize anonymous spec") + if self._normal and not force: return False # Ensure first that all packages & compilers in the DAG exist. self.validate_names() - # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) @@ -1228,14 +1407,14 @@ class Spec(object): self._normal = True return any_change - def normalized(self): - """Return a normalized copy of this spec without modifying this spec.""" + """ + Return a normalized copy of this spec without modifying this spec. + """ clone = self.copy() clone.normalize() return clone - def validate_names(self): """This checks that names of packages and compilers in this spec are real. If they're not, it will raise either UnknownPackageError or @@ -1243,7 +1422,7 @@ class Spec(object): """ for spec in self.traverse(): # Don't get a package for a virtual name. - if not spec.virtual: + if (not spec.virtual) and spec.name: spack.repo.get(spec.fullname) # validate compiler in addition to the package name. @@ -1256,7 +1435,6 @@ class Spec(object): if vname not in spec.package_class.variants: raise UnknownVariantError(spec.name, vname) - def constrain(self, other, deps=True): """Merge the constraints of other with self. @@ -1264,26 +1442,40 @@ class Spec(object): """ other = self._autospec(other) - if not self.name == other.name: + if not (self.name == other.name or + (not self.name) or + (not other.name)): raise UnsatisfiableSpecNameError(self.name, other.name) - if other.namespace is not None: - if self.namespace is not None and other.namespace != self.namespace: - raise UnsatisfiableSpecNameError(self.fullname, other.fullname) + if (other.namespace is not None and + self.namespace is not None and + other.namespace != self.namespace): + raise UnsatisfiableSpecNameError(self.fullname, other.fullname) if not self.versions.overlaps(other.versions): raise UnsatisfiableVersionSpecError(self.versions, other.versions) for v in other.variants: if (v in self.variants and - self.variants[v].enabled != other.variants[v].enabled): + self.variants[v].value != other.variants[v].value): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) + # TODO: Check out the logic here if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.architecture.platform is not None and other.architecture.platform is not None: + if self.architecture.platform != other.architecture.platform: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.platform_os is not None and other.architecture.platform_os is not None: + if self.architecture.platform_os != other.architecture.platform_os: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.target is not None and other.architecture.target is not None: + if self.architecture.target != other.architecture.target: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + changed = False if self.compiler is not None and other.compiler is not None: @@ -1295,16 +1487,25 @@ class Spec(object): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.architecture - self.architecture = self.architecture or other.architecture - changed |= (self.architecture != old) + changed |= self.compiler_flags.constrain(other.compiler_flags) + + old = str(self.architecture) + if self.architecture is None or other.architecture is None: + self.architecture = self.architecture or other.architecture + else: + if self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + if self.architecture.platform_os is None or other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + if self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target + changed |= (str(self.architecture) != old) if deps: changed |= self._constrain_dependencies(other) return changed - def _constrain_dependencies(self, other): """Apply constraints of other spec's dependencies to this spec.""" other = self._autospec(other) @@ -1323,7 +1524,6 @@ class Spec(object): for name in self.common_dependencies(other): changed |= self[name].constrain(other[name], deps=False) - # Update with additional constraints from other spec for name in other.dep_difference(self): self._add_dependency(other[name].copy()) @@ -1331,7 +1531,6 @@ class Spec(object): return changed - def common_dependencies(self, other): """Return names of dependencies that self an other have in common.""" common = set( @@ -1340,14 +1539,12 @@ class Spec(object): s.name for s in other.traverse(root=False)) return common - def constrained(self, other, deps=True): """Return a constrained copy without modifying this spec.""" clone = self.copy(deps=deps) clone.constrain(other, deps) return clone - def dep_difference(self, other): """Returns dependencies in self that are not in other.""" mine = set(s.name for s in self.traverse(root=False)) @@ -1355,21 +1552,24 @@ class Spec(object): s.name for s in other.traverse(root=False)) return mine - def _autospec(self, spec_like): - """Used to convert arguments to specs. If spec_like is a spec, returns it. - If it's a string, tries to parse a string. If that fails, tries to parse - a local spec from it (i.e. name is assumed to be self's name). + """ + Used to convert arguments to specs. If spec_like is a spec, returns + it. If it's a string, tries to parse a string. If that fails, tries + to parse a local spec from it (i.e. name is assumed to be self's name). """ if isinstance(spec_like, spack.spec.Spec): return spec_like try: - return spack.spec.Spec(spec_like) + spec = spack.spec.Spec(spec_like) + if not spec.name: + raise SpecError( + "anonymous package -- this will always be handled") + return spec except SpecError: return parse_anonymous_spec(spec_like, self.name) - def satisfies(self, other, deps=True, strict=False): """Determine if this spec satisfies all constraints of another. @@ -1396,14 +1596,14 @@ class Spec(object): return False # Otherwise, first thing we care about is whether the name matches - if self.name != other.name: + if self.name != other.name and self.name and other.name: return False # namespaces either match, or other doesn't require one. - if other.namespace is not None: - if self.namespace is not None and self.namespace != other.namespace: - return False - + if (other.namespace is not None and + self.namespace is not None and + self.namespace != other.namespace): + return False if self.versions and other.versions: if not self.versions.satisfies(other.versions, strict=strict): return False @@ -1417,26 +1617,43 @@ class Spec(object): elif strict and (other.compiler and not self.compiler): return False - if not self.variants.satisfies(other.variants, strict=strict): + var_strict = strict + if (not self.name) or (not other.name): + var_strict = True + if not self.variants.satisfies(other.variants, strict=var_strict): return False # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. if self.architecture and other.architecture: - if self.architecture != other.architecture: + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or + (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False - elif strict and (other.architecture and not self.architecture): + elif strict and ((other.architecture and not self.architecture) or + (other.architecture.platform and not self.architecture.platform) or + (other.architecture.platform_os and not self.architecture.platform_os) or + (other.architecture.target and not self.architecture.target)): + return False + + if not self.compiler_flags.satisfies( + other.compiler_flags, + strict=strict): return False # If we need to descend into dependencies, do it, otherwise we're done. if deps: - return self.satisfies_dependencies(other, strict=strict) + deps_strict = strict + if not (self.name and other.name): + deps_strict = True + return self.satisfies_dependencies(other, strict=deps_strict) else: return True - def satisfies_dependencies(self, other, strict=False): - """This checks constraints on common dependencies against each other.""" + """ + This checks constraints on common dependencies against each other. + """ other = self._autospec(other) if strict: @@ -1447,7 +1664,8 @@ class Spec(object): return False elif not self.dependencies or not other.dependencies: - # if either spec doesn't restrict dependencies then both are compatible. + # if either spec doesn't restrict dependencies then both are + # compatible. return True # Handle first-order constraints directly @@ -1463,11 +1681,12 @@ class Spec(object): if not self_index.satisfies(other_index): return False - # These two loops handle cases where there is an overly restrictive vpkg - # in one spec for a provider in the other (e.g., mpi@3: is not compatible - # with mpich2) + # These two loops handle cases where there is an overly restrictive + # vpkg in one spec for a provider in the other (e.g., mpi@3: is not + # compatible with mpich2) for spec in self.virtual_dependencies(): - if spec.name in other_index and not other_index.providers_for(spec): + if (spec.name in other_index and + not other_index.providers_for(spec)): return False for spec in other.virtual_dependencies(): @@ -1476,12 +1695,10 @@ class Spec(object): return True - def virtual_dependencies(self): """Return list of any virtual deps in this spec.""" return [spec for spec in self.traverse() if spec.virtual] - def _dup(self, other, **kwargs): """Copy the spec other into self. This is an overwriting copy. It does not copy any dependents (parents), but by default @@ -1491,16 +1708,17 @@ class Spec(object): Options: dependencies[=True] - Whether deps should be copied too. Set to false to copy a + Whether deps should be copied too. Set to False to copy a spec but not its dependencies. """ # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): - changed = (self.name != other.name and self.versions != other.versions and - self.architecture != other.architecture and self.compiler != other.compiler and - self.variants != other.variants and self._normal != other._normal and - self.concrete != other.concrete and self.external != other.external) + changed = (self.name != other.name and self.versions != other.versions and \ + self.architecture != other.architecture and self.compiler != other.compiler and \ + self.variants != other.variants and self._normal != other._normal and \ + self.concrete != other.concrete and self.external != other.external and \ + self.external_module != other.external_module and self.compiler_flags != other.compiler_flags) # Local node attributes get copied first. self.name = other.name @@ -1510,10 +1728,13 @@ class Spec(object): if kwargs.get('cleardeps', True): self.dependents = DependencyMap() self.dependencies = DependencyMap() + self.compiler_flags = other.compiler_flags.copy() self.variants = other.variants.copy() self.variants.spec = self self.external = other.external + self.external_module = other.external_module self.namespace = other.namespace + self._hash = other._hash # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1532,9 +1753,9 @@ class Spec(object): self._normal = other._normal self._concrete = other._concrete self.external = other.external + self.external_module = other.external_module return changed - def copy(self, **kwargs): """Return a copy of this spec. By default, returns a deep copy. Supply dependencies=False @@ -1544,14 +1765,12 @@ class Spec(object): clone._dup(self, **kwargs) return clone - @property def version(self): if not self.versions.concrete: raise SpecError("Spec version is not concrete: " + str(self)) return self.versions[0] - def __getitem__(self, name): """Get a dependency from the spec by its name.""" for spec in self.traverse(): @@ -1570,7 +1789,6 @@ class Spec(object): raise KeyError("No spec with name %s in %s" % (name, self)) - def __contains__(self, spec): """True if this spec satisfis the provided spec, or if any dependency does. If the spec has no name, then we parse this one first. @@ -1582,13 +1800,11 @@ class Spec(object): return False - def sorted_deps(self): """Return a list of all dependencies sorted by name.""" deps = self.flat_dependencies() return tuple(deps[name] for name in sorted(deps)) - def _eq_dag(self, other, vs, vo): """Recursive helper for eq_dag and ne_dag. Does the actual DAG traversal.""" @@ -1601,18 +1817,22 @@ class Spec(object): if len(self.dependencies) != len(other.dependencies): return False - ssorted = [self.dependencies[name] for name in sorted(self.dependencies)] - osorted = [other.dependencies[name] for name in sorted(other.dependencies)] + ssorted = [self.dependencies[name] + for name in sorted(self.dependencies)] + osorted = [other.dependencies[name] + for name in sorted(other.dependencies)] for s, o in zip(ssorted, osorted): visited_s = id(s) in vs visited_o = id(o) in vo # Check for duplicate or non-equal dependencies - if visited_s != visited_o: return False + if visited_s != visited_o: + return False # Skip visited nodes - if visited_s or visited_o: continue + if visited_s or visited_o: + continue # Recursive check for equality if not s._eq_dag(o, vs, vo): @@ -1620,17 +1840,14 @@ class Spec(object): return True - def eq_dag(self, other): """True if the full dependency DAGs of specs are equal""" return self._eq_dag(other, set(), set()) - def ne_dag(self, other): """True if the full dependency DAGs of specs are not equal""" return not self.eq_dag(other) - def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, @@ -1638,19 +1855,18 @@ class Spec(object): self.versions, self.variants, self.architecture, - self.compiler) + self.compiler, + self.compiler_flags) def eq_node(self, other): """Equality with another spec, not including dependencies.""" return self._cmp_node() == other._cmp_node() - def ne_node(self, other): """Inequality with another spec, not including dependencies.""" return self._cmp_node() != other._cmp_node() - def _cmp_key(self): """This returns a key for the spec *including* DAG structure. @@ -1662,52 +1878,56 @@ class Spec(object): tuple(hash(self.dependencies[name]) for name in sorted(self.dependencies)),) - def colorized(self): return colorize_spec(self) - - def format(self, format_string='$_$@$%@$+$=', **kwargs): - """Prints out particular pieces of a spec, depending on what is - in the format string. The format strings you can provide are:: - - $_ Package name - $. Full package name (with namespace) - $@ Version with '@' prefix - $% Compiler with '%' prefix - $%@ Compiler with '%' prefix & compiler version with '@' prefix - $+ Options - $= Architecture with '=' prefix - $# 7-char prefix of DAG hash with '-' prefix - $$ $ - - You can also use full-string versions, which leave off the prefixes: - - ${PACKAGE} Package name - ${VERSION} Version - ${COMPILER} Full compiler string - ${COMPILERNAME} Compiler name - ${COMPILERVER} Compiler version - ${OPTIONS} Options - ${ARCHITECTURE} Architecture - ${SHA1} Dependencies 8-char sha1 prefix - - ${SPACK_ROOT} The spack root directory - ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt - - Optionally you can provide a width, e.g. $20_ for a 20-wide name. - Like printf, you can provide '-' for left justification, e.g. - $-20_ for a left-justified name. - - Anything else is copied verbatim into the output stream. - - *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, arch, or compiler. - - TODO: allow, e.g., $6# to customize short hash length - TODO: allow, e.g., $## for full hash. - """ - color = kwargs.get('color', False) + def format(self, format_string='$_$@$%@+$+$=', **kwargs): + """ + Prints out particular pieces of a spec, depending on what is + in the format string. The format strings you can provide are:: + + $_ Package name + $. Full package name (with namespace) + $@ Version with '@' prefix + $% Compiler with '%' prefix + $%@ Compiler with '%' prefix & compiler version with '@' prefix + $%+ Compiler with '%' prefix & compiler flags prefixed by name + $%@+ Compiler, compiler version, and compiler flags with same + prefixes as above + $+ Options + $= Architecture prefixed by 'arch=' + $# 7-char prefix of DAG hash with '-' prefix + $$ $ + + You can also use full-string versions, which elide the prefixes: + + ${PACKAGE} Package name + ${VERSION} Version + ${COMPILER} Full compiler string + ${COMPILERNAME} Compiler name + ${COMPILERVER} Compiler version + ${COMPILERFLAGS} Compiler flags + ${OPTIONS} Options + ${ARCHITECTURE} Architecture + ${SHA1} Dependencies 8-char sha1 prefix + + ${SPACK_ROOT} The spack root directory + ${SPACK_INSTALL} The default spack install directory, + ${SPACK_PREFIX}/opt + + Optionally you can provide a width, e.g. $20_ for a 20-wide name. + Like printf, you can provide '-' for left justification, e.g. + $-20_ for a left-justified name. + + Anything else is copied verbatim into the output stream. + + *Example:* ``$_$@$+`` translates to the name, version, and options + of the package, but no dependencies, arch, or compiler. + + TODO: allow, e.g., $6# to customize short hash length + TODO: allow, e.g., $## for full hash. + """ + color = kwargs.get('color', False) length = len(format_string) out = StringIO() named = escape = compiler = False @@ -1734,7 +1954,8 @@ class Spec(object): fmt += 's' if c == '_': - out.write(fmt % self.name) + name = self.name if self.name else '' + out.write(fmt % name) elif c == '.': out.write(fmt % self.fullname) elif c == '@': @@ -1748,8 +1969,8 @@ class Spec(object): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: - write(fmt % (c + str(self.architecture)), c) + if self.architecture and str(self.architecture): + write(fmt % (' arch' + c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -1764,22 +1985,28 @@ class Spec(object): elif compiler: if c == '@': if (self.compiler and self.compiler.versions and - self.compiler.versions != _any_version): + self.compiler.versions != _any_version): write(c + str(self.compiler.versions), '%') + elif c == '+': + if self.compiler_flags: + write(fmt % str(self.compiler_flags), '%') + compiler = False elif c == '$': escape = True + compiler = False else: out.write(c) - compiler = False + compiler = False elif named: if not c == '}': if i == length - 1: - raise ValueError("Error: unterminated ${ in format: '%s'" - % format_string) + raise ValueError("Error: unterminated ${ in format:" + "'%s'" % format_string) named_str += c - continue; + continue if named_str == 'PACKAGE': + name = self.name if self.name else '' write(fmt % self.name, '@') if named_str == 'VERSION': if self.versions and self.versions != _any_version: @@ -1793,12 +2020,15 @@ class Spec(object): elif named_str == 'COMPILERVER': if self.compiler: write(fmt % self.compiler.versions, '%') + elif named_str == 'COMPILERFLAGS': + if self.compiler: + write(fmt % str(self.compiler_flags), '%') elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': - if self.architecture: - write(fmt % str(self.architecture), '=') + if self.architecture and str(self.architecture): + write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dag_hash(7))) @@ -1820,24 +2050,56 @@ class Spec(object): result = out.getvalue() return result - def dep_string(self): return ''.join("^" + dep.format() for dep in self.sorted_deps()) + def __cmp__(self, other): + #Package name sort order is not configurable, always goes alphabetical + if self.name != other.name: + return cmp(self.name, other.name) + + #Package version is second in compare order + pkgname = self.name + if self.versions != other.versions: + return spack.pkgsort.version_compare(pkgname, + self.versions, other.versions) + + #Compiler is third + if self.compiler != other.compiler: + return spack.pkgsort.compiler_compare(pkgname, + self.compiler, other.compiler) + + #Variants + if self.variants != other.variants: + return spack.pkgsort.variant_compare(pkgname, + self.variants, other.variants) + + #Target + if self.architecture != other.architecture: + return spack.pkgsort.architecture_compare(pkgname, + self.architecture, other.architecture) + + #Dependency is not configurable + if self.dependencies != other.dependencies: + return -1 if self.dependencies < other.dependencies else 1 + + #Equal specs + return 0 + + def __str__(self): return self.format() + self.dep_string() - def tree(self, **kwargs): """Prints out this spec and its dependencies, tree-formatted with indentation.""" - color = kwargs.pop('color', False) - depth = kwargs.pop('depth', False) + color = kwargs.pop('color', False) + depth = kwargs.pop('depth', False) showid = kwargs.pop('ids', False) - cover = kwargs.pop('cover', 'nodes') + cover = kwargs.pop('cover', 'nodes') indent = kwargs.pop('indent', 0) - fmt = kwargs.pop('format', '$_$@$%@$+$=') + fmt = kwargs.pop('format', '$_$@$%@+$+$=') prefix = kwargs.pop('prefix', None) check_kwargs(kwargs, self.tree) @@ -1861,7 +2123,6 @@ class Spec(object): out += node.format(fmt, color=color) + "\n" return out - def __repr__(self): return str(self) @@ -1869,80 +2130,143 @@ class Spec(object): # # These are possible token types in the spec grammar. # -DEP, AT, COLON, COMMA, ON, OFF, PCT, EQ, ID = range(9) +HASH, DEP, AT, COLON, COMMA, ON, OFF, PCT, EQ, QT, ID = range(11) + class SpecLexer(spack.parse.Lexer): + """Parses tokens that make up spack specs.""" + def __init__(self): super(SpecLexer, self).__init__([ - (r'\^', lambda scanner, val: self.token(DEP, val)), - (r'\@', lambda scanner, val: self.token(AT, val)), - (r'\:', lambda scanner, val: self.token(COLON, val)), - (r'\,', lambda scanner, val: self.token(COMMA, val)), - (r'\+', lambda scanner, val: self.token(ON, val)), - (r'\-', lambda scanner, val: self.token(OFF, val)), - (r'\~', lambda scanner, val: self.token(OFF, val)), - (r'\%', lambda scanner, val: self.token(PCT, val)), - (r'\=', lambda scanner, val: self.token(EQ, val)), + (r'/', lambda scanner, val: self.token(HASH, val)), + (r'\^', lambda scanner, val: self.token(DEP, val)), + (r'\@', lambda scanner, val: self.token(AT, val)), + (r'\:', lambda scanner, val: self.token(COLON, val)), + (r'\,', lambda scanner, val: self.token(COMMA, val)), + (r'\+', lambda scanner, val: self.token(ON, val)), + (r'\-', lambda scanner, val: self.token(OFF, val)), + (r'\~', lambda scanner, val: self.token(OFF, val)), + (r'\%', lambda scanner, val: self.token(PCT, val)), + (r'\=', lambda scanner, val: self.token(EQ, val)), # This is more liberal than identifier_re (see above). # Checked by check_identifier() for better error messages. + (r'([\"\'])(?:(?=(\\?))\2.)*?\1', + lambda scanner, val: self.token(QT, val)), (r'\w[\w.-]*', lambda scanner, val: self.token(ID, val)), - (r'\s+', lambda scanner, val: None)]) + (r'\s+', lambda scanner, val: None)]) + +# Lexer is always the same for every parser. +_lexer = SpecLexer() class SpecParser(spack.parse.Parser): - def __init__(self): - super(SpecParser, self).__init__(SpecLexer()) + def __init__(self): + super(SpecParser, self).__init__(_lexer) + self.previous = None def do_parse(self): specs = [] - try: while self.next: + # TODO: clean this parsing up a bit + if self.previous: + specs.append(self.spec(self.previous.value)) if self.accept(ID): - specs.append(self.spec()) + self.previous = self.token + if self.accept(EQ): + if not specs: + specs.append(self.spec(None)) + if self.accept(QT): + self.token.value = self.token.value[1:-1] + else: + self.expect(ID) + specs[-1]._add_flag( + self.previous.value, self.token.value) + else: + specs.append(self.spec(self.previous.value)) + self.previous = None + elif self.accept(HASH): + specs.append(self.spec_by_hash()) elif self.accept(DEP): if not specs: - self.last_token_error("Dependency has no package") - self.expect(ID) - specs[-1]._add_dependency(self.spec()) + self.previous = self.token + specs.append(self.spec(None)) + self.previous = None + if self.accept(HASH): + specs[-1]._add_dependency(self.spec_by_hash()) + else: + self.expect(ID) + specs[-1]._add_dependency(self.spec(self.token.value)) else: - self.unexpected_token() + # Attempt to construct an anonymous spec, but check that + # the first token is valid + # TODO: Is this check even necessary, or will it all be Lex + # errors now? + specs.append(self.spec(None, True)) + except spack.parse.ParseError, e: raise SpecParseError(e) - return specs + # If the spec has an os or a target and no platform, give it the default platform + for spec in specs: + for s in spec.traverse(): + if s.architecture.os_string or s.architecture.target_string: + s._set_platform(spack.architecture.sys_type()) + return specs def parse_compiler(self, text): self.setup(text) return self.compiler() + def spec_by_hash(self): + self.expect(ID) + + specs = spack.installed_db.query() + matches = [spec for spec in specs if + spec.dag_hash()[:len(self.token.value)] == self.token.value] - def spec(self): + if not matches: + tty.die("%s does not match any installed packages." % + self.token.value) + + if len(matches) != 1: + raise AmbiguousHashError( + "Multiple packages specify hash %s." % self.token.value, + *matches) + + return matches[0] + + def spec(self, name, check_valid_token=False): """Parse a spec out of the input. If a spec is supplied, then initialize and return it instead of creating a new one.""" - - spec_namespace, dot, spec_name = self.token.value.rpartition('.') - if not spec_namespace: + if name: + spec_namespace, dot, spec_name = name.rpartition('.') + if not spec_namespace: + spec_namespace = None + self.check_identifier(spec_name) + else: spec_namespace = None - - self.check_identifier(spec_name) + spec_name = None # This will init the spec without calling __init__. spec = Spec.__new__(Spec) spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.architecture = spack.architecture.Arch() spec.compiler = None spec.external = None - spec.dependents = DependencyMap() + spec.external_module = None + spec.compiler_flags = FlagMap(spec) + spec.dependents = DependencyMap() spec.dependencies = DependencyMap() spec.namespace = spec_namespace + spec._hash = None spec._normal = False spec._concrete = False @@ -1951,26 +2275,51 @@ class SpecParser(spack.parse.Parser): # unspecified or not. added_version = False + if self.previous and self.previous.value == DEP: + if self.accept(HASH): + spec.add_dependency(self.spec_by_hash()) + else: + self.expect(ID) + if self.accept(EQ): + raise SpecParseError(spack.parse.ParseError( + "", "", "Expected dependency received anonymous spec")) + spec.add_dependency(self.spec(self.token.value)) + while self.next: if self.accept(AT): vlist = self.version_list() for version in vlist: spec._add_version(version) added_version = True + check_valid_token = False elif self.accept(ON): spec._add_variant(self.variant(), True) + check_valid_token = False elif self.accept(OFF): spec._add_variant(self.variant(), False) + check_valid_token = False elif self.accept(PCT): spec._set_compiler(self.compiler()) + check_valid_token = False - elif self.accept(EQ): - spec._set_architecture(self.architecture()) + elif self.accept(ID): + self.previous = self.token + if self.accept(EQ): + if self.accept(QT): + self.token.value = self.token.value[1:-1] + else: + self.expect(ID) + spec._add_flag(self.previous.value, self.token.value) + self.previous = None + else: + return spec else: + if check_valid_token: + self.unexpected_token() break # If there was no version in the spec, consier it an open range @@ -1979,17 +2328,14 @@ class SpecParser(spack.parse.Parser): return spec - - def variant(self): - self.expect(ID) - self.check_identifier() - return self.token.value - - - def architecture(self): - self.expect(ID) - return self.token.value - + def variant(self, name=None): + # TODO: Make generalized variants possible + if name: + return name + else: + self.expect(ID) + self.check_identifier() + return self.token.value def version(self): start = None @@ -2007,11 +2353,12 @@ class SpecParser(spack.parse.Parser): # No colon and no id: invalid version. self.next_token_error("Invalid version specifier") - if start: start = Version(start) - if end: end = Version(end) + if start: + start = Version(start) + if end: + end = Version(end) return VersionRange(start, end) - def version_list(self): vlist = [] vlist.append(self.version()) @@ -2019,7 +2366,6 @@ class SpecParser(spack.parse.Parser): vlist.append(self.version()) return vlist - def compiler(self): self.expect(ID) self.check_identifier() @@ -2035,7 +2381,6 @@ class SpecParser(spack.parse.Parser): compiler.versions = VersionList(':') return compiler - def check_identifier(self, id=None): """The only identifiers that can contain '.' are versions, but version ids are context-sensitive so we have to check on a case-by-case @@ -2068,9 +2413,16 @@ def parse_anonymous_spec(spec_like, pkg_name): if isinstance(spec_like, str): try: anon_spec = Spec(spec_like) + if anon_spec.name != pkg_name: + raise SpecParseError(spack.parse.ParseError( + "", + "", + "Expected anonymous spec for package %s but found spec for" + "package %s" % (pkg_name, anon_spec.name))) except SpecParseError: - anon_spec = Spec(pkg_name + spec_like) - if anon_spec.name != pkg_name: raise ValueError( + anon_spec = Spec(pkg_name + ' ' + spec_like) + if anon_spec.name != pkg_name: + raise ValueError( "Invalid spec for package %s: %s" % (pkg_name, spec_like)) else: anon_spec = spec_like.copy() @@ -2083,13 +2435,17 @@ def parse_anonymous_spec(spec_like, pkg_name): class SpecError(spack.error.SpackError): + """Superclass for all errors that occur while constructing specs.""" + def __init__(self, message): super(SpecError, self).__init__(message) class SpecParseError(SpecError): + """Wrapper for ParseError for when we're parsing specs.""" + def __init__(self, parse_error): super(SpecParseError, self).__init__(parse_error.message) self.string = parse_error.string @@ -2097,61 +2453,79 @@ class SpecParseError(SpecError): class DuplicateDependencyError(SpecError): + """Raised when the same dependency occurs in a spec twice.""" + def __init__(self, message): super(DuplicateDependencyError, self).__init__(message) class DuplicateVariantError(SpecError): + """Raised when the same variant occurs in a spec twice.""" + def __init__(self, message): super(DuplicateVariantError, self).__init__(message) class DuplicateCompilerSpecError(SpecError): + """Raised when the same compiler occurs in a spec twice.""" + def __init__(self, message): super(DuplicateCompilerSpecError, self).__init__(message) class UnsupportedCompilerError(SpecError): + """Raised when the user asks for a compiler spack doesn't know about.""" + def __init__(self, compiler_name): super(UnsupportedCompilerError, self).__init__( "The '%s' compiler is not yet supported." % compiler_name) class UnknownVariantError(SpecError): + """Raised when the same variant occurs in a spec twice.""" + def __init__(self, pkg, variant): super(UnknownVariantError, self).__init__( "Package %s has no variant %s!" % (pkg, variant)) class DuplicateArchitectureError(SpecError): + """Raised when the same architecture occurs in a spec twice.""" + def __init__(self, message): super(DuplicateArchitectureError, self).__init__(message) class InconsistentSpecError(SpecError): + """Raised when two nodes in the same spec DAG have inconsistent constraints.""" + def __init__(self, message): super(InconsistentSpecError, self).__init__(message) class InvalidDependencyException(SpecError): + """Raised when a dependency in a spec is not actually a dependency of the package.""" + def __init__(self, message): super(InvalidDependencyException, self).__init__(message) class NoProviderError(SpecError): + """Raised when there is no package that provides a particular virtual dependency. """ + def __init__(self, vpkg): super(NoProviderError, self).__init__( "No providers found for virtual package: '%s'" % vpkg) @@ -2159,9 +2533,11 @@ class NoProviderError(SpecError): class MultipleProviderError(SpecError): + """Raised when there is no package that provides a particular virtual dependency. """ + def __init__(self, vpkg, providers): """Takes the name of the vpkg""" super(MultipleProviderError, self).__init__( @@ -2172,8 +2548,10 @@ class MultipleProviderError(SpecError): class UnsatisfiableSpecError(SpecError): + """Raised when a spec conflicts with package constraints. Provide the requirement that was violated when raising.""" + def __init__(self, provided, required, constraint_type): super(UnsatisfiableSpecError, self).__init__( "%s does not satisfy %s" % (provided, required)) @@ -2183,55 +2561,96 @@ class UnsatisfiableSpecError(SpecError): class UnsatisfiableSpecNameError(UnsatisfiableSpecError): + """Raised when two specs aren't even for the same package.""" + def __init__(self, provided, required): super(UnsatisfiableSpecNameError, self).__init__( provided, required, "name") class UnsatisfiableVersionSpecError(UnsatisfiableSpecError): + """Raised when a spec version conflicts with package constraints.""" + def __init__(self, provided, required): super(UnsatisfiableVersionSpecError, self).__init__( provided, required, "version") class UnsatisfiableCompilerSpecError(UnsatisfiableSpecError): + """Raised when a spec comiler conflicts with package constraints.""" + def __init__(self, provided, required): super(UnsatisfiableCompilerSpecError, self).__init__( provided, required, "compiler") class UnsatisfiableVariantSpecError(UnsatisfiableSpecError): + """Raised when a spec variant conflicts with package constraints.""" + def __init__(self, provided, required): super(UnsatisfiableVariantSpecError, self).__init__( provided, required, "variant") +class UnsatisfiableCompilerFlagSpecError(UnsatisfiableSpecError): + + """Raised when a spec variant conflicts with package constraints.""" + + def __init__(self, provided, required): + super(UnsatisfiableCompilerFlagSpecError, self).__init__( + provided, required, "compiler_flags") + + class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError): + """Raised when a spec architecture conflicts with package constraints.""" + def __init__(self, provided, required): super(UnsatisfiableArchitectureSpecError, self).__init__( provided, required, "architecture") class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): + """Raised when a provider is supplied but constraints don't match a vpkg requirement""" + def __init__(self, provided, required): super(UnsatisfiableProviderSpecError, self).__init__( provided, required, "provider") # TODO: get rid of this and be more specific about particular incompatible # dep constraints + + class UnsatisfiableDependencySpecError(UnsatisfiableSpecError): + """Raised when some dependency of constrained specs are incompatible""" + def __init__(self, provided, required): super(UnsatisfiableDependencySpecError, self).__init__( provided, required, "dependency") + class SpackYAMLError(spack.error.SpackError): + def __init__(self, msg, yaml_error): super(SpackYAMLError, self).__init__(msg, str(yaml_error)) + + +class SpackRecordError(spack.error.SpackError): + + def __init__(self, msg): + super(SpackRecordError, self).__init__(msg) + + +class AmbiguousHashError(SpecError): + + def __init__(self, msg, *specs): + super(AmbiguousHashError, self).__init__(msg) + for spec in specs: + print ' ', spec.format('$.$@$%@+$+$=$#') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 1668e271fa..fb91f24721 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,15 +31,16 @@ from llnl.util.filesystem import join_path from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', 'multimethod', 'install', 'package_sanity', 'config', 'directory_layout', 'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', 'optional_deps', 'make_executable', 'configure_guess', 'lock', 'database', - 'namespace_trie', 'yaml', 'sbang', 'environment', - 'cmd.uninstall', 'cmd.test_install'] + 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', + 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] def list_tests(): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py new file mode 100644 index 0000000000..a6847c5744 --- /dev/null +++ b/lib/spack/spack/test/architecture.py @@ -0,0 +1,112 @@ +""" Test checks if the architecture class is created correctly and also that + the functions are looking for the correct architecture name +""" +import unittest +import os +import platform as py_platform +import spack +from spack.architecture import * +from spack.spec import * +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.bgq import Bgq +from spack.platforms.darwin import Darwin + +from spack.test.mock_packages_test import * + +#class ArchitectureTest(unittest.TestCase): +class ArchitectureTest(MockPackagesTest): + + def setUp(self): + super(ArchitectureTest, self).setUp() + self.platform = sys_type() + + def tearDown(self): + super(ArchitectureTest, self).tearDown() + + def test_dict_functions_for_architecture(self): + arch = Arch() + arch.platform = spack.architecture.sys_type() + arch.platform_os = arch.platform.operating_system('default_os') + arch.target = arch.platform.target('default_target') + + d = arch.to_dict() + + new_arch = spack.architecture.arch_from_dict(d) + + self.assertEqual(arch, new_arch) + + self.assertTrue( isinstance(arch, Arch) ) + self.assertTrue( isinstance(arch.platform, Platform) ) + self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(arch.target, Target) ) + self.assertTrue( isinstance(new_arch, Arch) ) + self.assertTrue( isinstance(new_arch.platform, Platform) ) + self.assertTrue( isinstance(new_arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(new_arch.target, Target) ) + + + def test_sys_type(self): + output_platform_class = sys_type() + my_arch_class = None + if os.path.exists('/opt/cray/craype'): + my_platform_class = CrayXc() + elif os.path.exists('/bgsys'): + my_platform_class = Bgq() + elif 'Linux' in py_platform.system(): + my_platform_class = Linux() + elif 'Darwin' in py_platform.system(): + my_platform_class = Darwin() + + self.assertEqual(str(output_platform_class), str(my_platform_class)) + + def test_user_front_end_input(self): + """Test when user inputs just frontend that both the frontend target + and frontend operating system match + """ + frontend_os = self.platform.operating_system("frontend") + frontend_target = self.platform.target("frontend") + frontend_spec = Spec("libelf os=frontend target=frontend") + frontend_spec.concretize() + self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) + self.assertEqual(frontend_target, frontend_spec.architecture.target) + + def test_user_back_end_input(self): + """Test when user inputs backend that both the backend target and + backend operating system match + """ + backend_os = self.platform.operating_system("backend") + backend_target = self.platform.target("backend") + backend_spec = Spec("libelf os=backend target=backend") + backend_spec.concretize() + self.assertEqual(backend_os, backend_spec.architecture.platform_os) + self.assertEqual(backend_target, backend_spec.architecture.target) + + def test_user_defaults(self): + default_os = self.platform.operating_system("default_os") + default_target = self.platform.target("default_target") + + default_spec = Spec("libelf") # default is no args + default_spec.concretize() + self.assertEqual(default_os, default_spec.architecture.platform_os) + self.assertEqual(default_target, default_spec.architecture.target) + + def test_user_input_combination(self): + os_list = self.platform.operating_sys.keys() + target_list = self.platform.targets.keys() + additional = ["fe", "be", "frontend", "backend"] + + os_list.extend(additional) + target_list.extend(additional) + + combinations = itertools.product(os_list, target_list) + results = [] + for arch in combinations: + o,t = arch + spec = Spec("libelf os=%s target=%s" % (o, t)) + spec.concretize() + results.append(spec.architecture.platform_os == self.platform.operating_system(o)) + results.append(spec.architecture.target == self.platform.target(t)) + res = all(results) + + self.assertTrue(res) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index a630866143..ea2b164462 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -56,11 +56,16 @@ class CompilerTest(unittest.TestCase): self.cc = Executable(join_path(spack.build_env_path, "cc")) self.ld = Executable(join_path(spack.build_env_path, "ld")) self.cpp = Executable(join_path(spack.build_env_path, "cpp")) + self.cxx = Executable(join_path(spack.build_env_path, "c++")) + self.fc = Executable(join_path(spack.build_env_path, "fc")) self.realcc = "/bin/mycc" self.prefix = "/spack-test-prefix" os.environ['SPACK_CC'] = self.realcc + os.environ['SPACK_CXX'] = self.realcc + os.environ['SPACK_FC'] = self.realcc + os.environ['SPACK_PREFIX'] = self.prefix os.environ['SPACK_ENV_PATH']="test" os.environ['SPACK_DEBUG_LOG_DIR'] = "." @@ -102,6 +107,15 @@ class CompilerTest(unittest.TestCase): self.assertEqual(self.cc(*args, output=str).strip(), expected) + def check_cxx(self, command, args, expected): + os.environ['SPACK_TEST_COMMAND'] = command + self.assertEqual(self.cxx(*args, output=str).strip(), expected) + + def check_fc(self, command, args, expected): + os.environ['SPACK_TEST_COMMAND'] = command + self.assertEqual(self.fc(*args, output=str).strip(), expected) + + def check_ld(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command self.assertEqual(self.ld(*args, output=str).strip(), expected) @@ -142,6 +156,64 @@ class CompilerTest(unittest.TestCase): self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld") + def test_flags(self): + os.environ['SPACK_LDFLAGS'] = '-L foo' + os.environ['SPACK_LDLIBS'] = '-lfoo' + os.environ['SPACK_CPPFLAGS'] = '-g -O1' + os.environ['SPACK_CFLAGS'] = '-Wall' + os.environ['SPACK_CXXFLAGS'] = '-Werror' + os.environ['SPACK_FFLAGS'] = '-w' + + # Test ldflags added properly in ld mode + self.check_ld('dump-args', test_command, + "ld " + + '-rpath ' + self.prefix + '/lib ' + + '-rpath ' + self.prefix + '/lib64 ' + + '-L foo ' + + ' '.join(test_command) + ' ' + + '-lfoo') + + # Test cppflags added properly in cpp mode + self.check_cpp('dump-args', test_command, + "cpp " + + '-g -O1 ' + + ' '.join(test_command)) + + # Test ldflags, cppflags, and language specific flags are added in proper order + self.check_cc('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + '-g -O1 ' + + '-Wall ' + + '-L foo ' + + ' '.join(test_command) + ' ' + + '-lfoo') + + self.check_cxx('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + '-g -O1 ' + + '-Werror ' + + '-L foo ' + + ' '.join(test_command) + ' ' + + '-lfoo') + + self.check_fc('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + '-w ' + + '-g -O1 ' + + '-L foo ' + + ' '.join(test_command) + ' ' + + '-lfoo') + + os.environ['SPACK_LDFLAGS']='' + os.environ['SPACK_LDLIBS']='' + + def test_dep_rpath(self): """Ensure RPATHs for root package are added.""" self.check_cc('dump-args', test_command, diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py new file mode 100644 index 0000000000..371e9650e0 --- /dev/null +++ b/lib/spack/spack/test/cmd/find.py @@ -0,0 +1,60 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + + +import spack.cmd.find +import unittest + + +class Bunch(object): + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +class FindTest(unittest.TestCase): + + def test_query_arguments(self): + query_arguments = spack.cmd.find.query_arguments + # Default arguments + args = Bunch(only_missing=False, missing=False, + unknown=False, explicit=False, implicit=False) + q_args = query_arguments(args) + self.assertTrue('installed' in q_args) + self.assertTrue('known' in q_args) + self.assertTrue('explicit' in q_args) + self.assertEqual(q_args['installed'], True) + self.assertEqual(q_args['known'], any) + self.assertEqual(q_args['explicit'], any) + # Check that explicit works correctly + args.explicit = True + q_args = query_arguments(args) + self.assertEqual(q_args['explicit'], True) + args.explicit = False + args.implicit = True + q_args = query_arguments(args) + self.assertEqual(q_args['explicit'], False) + args.explicit = True + self.assertRaises(SystemExit, query_arguments, args) diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py new file mode 100644 index 0000000000..d89814154b --- /dev/null +++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py @@ -0,0 +1,81 @@ +import os +import shutil +from tempfile import mkdtemp + +from llnl.util.filesystem import set_executable, mkdirp + +import spack.spec +import spack.cmd.compiler +import spack.compilers +from spack.version import Version +from spack.test.mock_packages_test import * + +test_version = '4.5-spacktest' + +class MockArgs(object): + def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None): + self.add_paths = add_paths + self.scope = scope + self.compiler_spec = compiler_spec + self.all = all + + +def make_mock_compiler(): + """Make a directory containing a fake, but detectable compiler.""" + mock_compiler_dir = mkdtemp() + bin_dir = os.path.join(mock_compiler_dir, 'bin') + mkdirp(bin_dir) + + gcc_path = os.path.join(bin_dir, 'gcc') + gxx_path = os.path.join(bin_dir, 'g++') + gfortran_path = os.path.join(bin_dir, 'gfortran') + + with open(gcc_path, 'w') as f: + f.write("""\ +#!/bin/sh + +for arg in "$@"; do + if [ "$arg" = -dumpversion ]; then + echo '%s' + fi +done +""" % test_version) + + # Create some mock compilers in the temporary directory + set_executable(gcc_path) + shutil.copy(gcc_path, gxx_path) + shutil.copy(gcc_path, gfortran_path) + + return mock_compiler_dir + + +class CompilerCmdTest(MockPackagesTest): + """ Test compiler commands for add and remove """ + + + def test_compiler_remove(self): + args = MockArgs(all=True, compiler_spec='gcc@4.5.0') + spack.cmd.compiler.compiler_remove(args) + compilers = spack.compilers.all_compilers() + self.assertTrue(spack.spec.CompilerSpec("gcc@4.5.0") not in compilers) + + + def test_compiler_add(self): + # compilers available by default. + old_compilers = set(spack.compilers.all_compilers()) + + # add our new compiler and find again. + compiler_dir = make_mock_compiler() + + try: + args = MockArgs(add_paths=[compiler_dir]) + spack.cmd.compiler.compiler_find(args) + + # ensure new compiler is in there + new_compilers = set(spack.compilers.all_compilers()) + new_compiler = new_compilers - old_compilers + self.assertTrue(new_compiler) + self.assertTrue(new_compiler.pop().version == Version(test_version)) + + finally: + shutil.rmtree(compiler_dir, ignore_errors=True) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 799fdae3a9..ab201f406a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack +import spack.architecture from spack.spec import Spec, CompilerSpec from spack.version import ver from spack.concretize import find_spec @@ -38,11 +39,20 @@ class ConcretizeTest(MockPackagesTest): for name in abstract.variants: avariant = abstract.variants[name] cvariant = concrete.variants[name] - self.assertEqual(avariant.enabled, cvariant.enabled) + self.assertEqual(avariant.value, cvariant.value) + + if abstract.compiler_flags: + for flag in abstract.compiler_flags: + aflag = abstract.compiler_flags[flag] + cflag = concrete.compiler_flags[flag] + self.assertTrue(set(aflag) <= set(cflag)) for name in abstract.package.variants: self.assertTrue(name in concrete.variants) + for flag in concrete.compiler_flags.valid_compiler_flags(): + self.assertTrue(flag in concrete.compiler_flags) + if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) @@ -75,9 +85,14 @@ class ConcretizeTest(MockPackagesTest): def test_concretize_variant(self): self.check_concretize('mpich+debug') self.check_concretize('mpich~debug') + self.check_concretize('mpich debug=2') self.check_concretize('mpich') + def test_conretize_compiler_flags(self): + self.check_concretize('mpich cppflags="-O3"') + + def test_concretize_preferred_version(self): spec = self.check_concretize('python') self.assertEqual(spec.versions, ver('2.7.11')) @@ -231,7 +246,7 @@ class ConcretizeTest(MockPackagesTest): def test_external_package(self): - spec = Spec('externaltool') + spec = Spec('externaltool%gcc') spec.concretize() self.assertEqual(spec['externaltool'].external, '/path/to/external_tool') @@ -239,6 +254,19 @@ class ConcretizeTest(MockPackagesTest): self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + def test_external_package_module(self): + # No tcl modules on darwin/linux machines + # TODO: improved way to check for this. + if (spack.architecture.sys_type().name == 'darwin' or + spack.architecture.sys_type().name == 'linux'): + return + + spec = Spec('externalmodule') + spec.concretize() + self.assertEqual(spec['externalmodule'].external_module, 'external-module') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externalmodule'].compiler.satisfies('gcc')) + def test_nobuild_package(self): got_error = False spec = Spec('externaltool%clang') diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index eff482f4c6..252d77e66b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -32,45 +32,75 @@ from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data -a_comps = { - "x86_64_E5v2_IntelIB": { - "gcc@4.7.3" : { +a_comps = [ + {'compiler': { + 'paths': { "cc" : "/gcc473", "cxx": "/g++473", "f77": None, - "fc" : None }, - "gcc@4.5.0" : { + "fc" : None + }, + 'modules': None, + 'spec': 'gcc@4.7.3', + 'operating_system': 'CNL10' + }}, + {'compiler': { + 'paths': { "cc" : "/gcc450", "cxx": "/g++450", - "f77": "/gfortran", - "fc" : "/gfortran" }, - "clang@3.3" : { + "f77": 'gfortran', + "fc" : 'gfortran' + }, + 'modules': None, + 'spec': 'gcc@4.5.0', + 'operating_system': 'CNL10' + }}, + {'compiler': { + 'paths': { "cc" : "<overwritten>", "cxx": "<overwritten>", - "f77": "<overwritten>", - "fc" : "<overwritten>" } - } -} - -b_comps = { - "x86_64_E5v3": { - "icc@10.0" : { + "f77": '<overwritten>', + "fc" : '<overwritten>' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': 'CNL10' + }} +] + +b_comps = [ + {'compiler': { + 'paths': { "cc" : "/icc100", - "cxx": "/icc100", + "cxx": "/icp100", "f77": None, - "fc" : None }, - "icc@11.1" : { + "fc" : None + }, + 'modules': None, + 'spec': 'icc@10.0', + 'operating_system': 'CNL10' + }}, + {'compiler': { + 'paths': { "cc" : "/icc111", "cxx": "/icp111", - "f77": "/ifort", - "fc" : "/ifort" }, - "clang@3.3" : { - "cc" : "/clang", - "cxx": "/clang++", - "f77": None, - "fc" : None} - } -} + "f77": 'ifort', + "fc" : 'ifort' + }, + 'modules': None, + 'spec': 'icc@11.1', + 'operating_system': 'CNL10' + }}, + {'compiler': { + 'paths': { + "cc" : "<overwritten>", + "cxx": "<overwritten>", + "f77": '<overwritten>', + "fc" : '<overwritten>' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': 'CNL10' + }} +] # Some Sample repo data repos_low = [ "/some/path" ] @@ -89,15 +119,28 @@ class ConfigTest(MockPackagesTest): super(ConfigTest, self).tearDown() shutil.rmtree(self.tmp_dir, True) - def check_config(self, comps, arch, *compiler_names): + + def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] - for key in compiler_names: - for c in compiler_list: - expected = comps[arch][key][c] - actual = config[arch][key][c] - self.assertEqual(expected, actual) + param_list = ['modules', 'paths', 'spec', 'operating_system'] + for compiler in config: + conf = compiler['compiler'] + if conf['spec'] in compiler_names: + comp = None + for c in comps: + if c['compiler']['spec'] == conf['spec']: + comp = c['compiler'] + break + if not comp: + self.fail('Bad config spec') + for p in param_list: + self.assertEqual(conf[p], comp[p]) + for c in compiler_list: + expected = comp['paths'][c] + actual = conf['paths'][c] + self.assertEqual(expected, actual) def test_write_list_in_memory(self): spack.config.update_config('repos', repos_low, 'test_low_priority') @@ -111,8 +154,9 @@ class ConfigTest(MockPackagesTest): spack.config.update_config('compilers', b_comps, 'test_high_priority') # Make sure the config looks how we expect. - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') + def test_write_key_to_disk(self): # Write b_comps "on top of" a_comps. @@ -123,8 +167,8 @@ class ConfigTest(MockPackagesTest): spack.config.clear_config_caches() # Same check again, to ensure consistency. - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') def test_write_to_same_priority_file(self): # Write b_comps in the same file as a_comps. @@ -135,5 +179,5 @@ class ConfigTest(MockPackagesTest): spack.config.clear_config_caches() # Same check again, to ensure consistency. - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh new file mode 100644 index 0000000000..800f639ac8 --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_first.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export NEW_VAR='new' +export UNSET_ME='overridden' diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh new file mode 100644 index 0000000000..9955a0e6d6 --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export PATH_LIST='/path/first:/path/second:/path/fourth' +unset EMPTY_PATH_LIST
\ No newline at end of file diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index ded1539e18..f3644cb0b7 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -24,16 +24,24 @@ ############################################################################## import unittest import os + +from spack import spack_root +from llnl.util.filesystem import join_path from spack.environment import EnvironmentModifications +from spack.environment import SetEnv, UnsetEnv +from spack.environment import RemovePath, PrependPath, AppendPath class EnvironmentTest(unittest.TestCase): + def setUp(self): - os.environ.clear() os.environ['UNSET_ME'] = 'foo' os.environ['EMPTY_PATH_LIST'] = '' os.environ['PATH_LIST'] = '/path/second:/path/third' - os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' # NOQA: ignore=E501 + + def tearDown(self): + pass def test_set(self): env = EnvironmentModifications() @@ -74,9 +82,18 @@ class EnvironmentTest(unittest.TestCase): env.remove_path('REMOVE_PATH_LIST', '/duplicate/') env.apply_modifications() - self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) - self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) - self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) + self.assertEqual( + '/path/first:/path/second:/path/third:/path/last', + os.environ['PATH_LIST'] + ) + self.assertEqual( + '/path/first:/path/middle:/path/last', + os.environ['EMPTY_PATH_LIST'] + ) + self.assertEqual( + '/path/first:/path/middle:/path/last', + os.environ['NEWLY_CREATED_PATH_LIST'] + ) self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) def test_extra_arguments(self): @@ -95,3 +112,41 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): assert x is y + + def test_source_files(self): + datadir = join_path(spack_root, 'lib', 'spack', + 'spack', 'test', 'data') + files = [ + join_path(datadir, 'sourceme_first.sh'), + join_path(datadir, 'sourceme_second.sh') + ] + env = EnvironmentModifications.from_sourcing_files(*files) + modifications = env.group_by_name() + + self.assertEqual(len(modifications), 4) + # Set new variables + self.assertEqual(len(modifications['NEW_VAR']), 1) + self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv)) + self.assertEqual(modifications['NEW_VAR'][0].value, 'new') + # Unset variables + self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1) + self.assertTrue(isinstance( + modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) + # Modified variables + self.assertEqual(len(modifications['UNSET_ME']), 1) + self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) + self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden') + + self.assertEqual(len(modifications['PATH_LIST']), 3) + self.assertTrue( + isinstance(modifications['PATH_LIST'][0], RemovePath) + ) + self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third') + self.assertTrue( + isinstance(modifications['PATH_LIST'][1], AppendPath) + ) + self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') + self.assertTrue( + isinstance(modifications['PATH_LIST'][2], PrependPath) + ) + self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 595667bf35..a56bd8ebdc 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,20 +34,127 @@ from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec +platform = spack.architecture.sys_type() + +linux_os_name = 'debian' +linux_os_version = '6' + +if platform.name == 'linux': + linux_os = platform.operating_system("default_os") + linux_os_name = linux_os.name + linux_os_version = linux_os.version + mock_compiler_config = """\ compilers: - all: - clang@3.3: +- compiler: + spec: clang@3.3 + operating_system: {0}{1} + paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None - gcc@4.5.0: + modules: 'None' +- compiler: + spec: gcc@4.5.0 + operating_system: {0}{1} + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' +- compiler: + spec: clang@3.3 + operating_system: CNL10 + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' +- compiler: + spec: clang@3.3 + operating_system: SuSE11 + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' +- compiler: + spec: clang@3.3 + operating_system: redhat6 + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' +- compiler: + spec: clang@3.3 + operating_system: yosemite + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' +- compiler: + paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran -""" + operating_system: CNL10 + spec: gcc@4.5.0 + modules: 'None' +- compiler: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: SuSE11 + spec: gcc@4.5.0 + modules: 'None' +- compiler: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: redhat6 + spec: gcc@4.5.0 + modules: 'None' +- compiler: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: yosemite + spec: gcc@4.5.0 + modules: 'None' +- compiler: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: elcapitan + spec: gcc@4.5.0 + modules: 'None' +- compiler: + spec: clang@3.3 + operating_system: elcapitan + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' +""".format(linux_os_name, linux_os_version) mock_packages_config = """\ packages: @@ -60,6 +167,10 @@ packages: paths: externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc + externalmodule: + buildable: False + modules: + externalmodule@1.0%gcc@4.5.0: external-module """ class MockPackagesTest(unittest.TestCase): diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 56e294de26..582e067860 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -73,7 +73,7 @@ configuration_alter_environment = { 'all': { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, - '=x86-linux': { + 'platform=test target=x86_64': { 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} } @@ -116,6 +116,7 @@ class TclTests(MockPackagesTest): def get_modulefile_content(self, spec): spec.concretize() + print spec, '&&&&&' generator = spack.modules.TclModule(spec) generator.write() content = FILE_REGISTRY[generator.file_name].split('\n') @@ -123,27 +124,28 @@ class TclTests(MockPackagesTest): def test_simple_case(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpich@3.0.4=x86-linux') + spec = spack.spec.Spec('mpich@3.0.4') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpileaks=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) spack.modules.CONFIGURATION = configuration_autoload_all - spec = spack.spec.Spec('mpileaks=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'module load ' in x]), 5) def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment - spec = spack.spec.Spec('mpileaks=x86-linux') + spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -152,8 +154,9 @@ class TclTests(MockPackagesTest): len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) - spec = spack.spec.Spec('libdwarf=x64-linux') + spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -164,14 +167,14 @@ class TclTests(MockPackagesTest): def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist - spec = spack.spec.Spec('mpileaks=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts - spec = spack.spec.Spec('mpileaks=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual( len([x for x in content if x.startswith('conflict')]), 2) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index f653ca3477..034e6b3923 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,9 +25,13 @@ """ Test for multi_method dispatch. """ +import unittest import spack from spack.multimethod import * +from spack.version import * +from spack.spec import Spec +from spack.multimethod import when from spack.test.mock_packages_test import * from spack.version import * @@ -88,21 +92,18 @@ class MultiMethodTest(MockPackagesTest): self.assertEqual(pkg.has_a_default(), 'default') - def test_architecture_match(self): - pkg = spack.repo.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') - - pkg = spack.repo.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') - - pkg = spack.repo.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') - - pkg = spack.repo.get('multimethod=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') + def test_target_match(self): + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + pkg = spack.repo.get('multimethod target='+target.name) + self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.repo.get('multimethod=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) + pkg = spack.repo.get('multimethod target='+targets[-1].name) + if len(targets) == 1: + self.assertEqual(pkg.different_by_target(), targets[-1].name) + else: + self.assertRaises(NoSuchMethodError, pkg.different_by_target) def test_dependency_match(self): diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py new file mode 100644 index 0000000000..ed5f6ff8ad --- /dev/null +++ b/lib/spack/spack/test/operating_system.py @@ -0,0 +1,55 @@ +""" Test checks if the operating_system class is created correctly and that +the functions are using the correct operating_system. Also checks whether +the operating_system correctly uses the compiler_strategy +""" + +import unittest +import os +import platform +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.darwin import Darwin +from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.mac_os import MacOs +from spack.operating_system.cnl import ComputeNodeLinux + +class TestOperatingSystem(unittest.TestCase): + + def setUp(self): + cray_xc = CrayXc() + linux = Linux() + darwin = Darwin() + self.cray_operating_sys = cray_xc.operating_system('front_os') + self.cray_default_os = cray_xc.operating_system('default_os') + self.cray_back_os = cray_xc.operating_system('back_os') + self.darwin_operating_sys = darwin.operating_system('default_os') + self.linux_operating_sys = linux.operating_system('default_os') + + def test_cray_front_end_operating_system(self): + self.assertIsInstance(self.cray_operating_sys, LinuxDistro) + + def test_cray_front_end_compiler_strategy(self): + self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + + def test_cray_back_end_operating_system(self): + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + + def test_cray_back_end_compiler_strategy(self): + self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") + + def test_linux_operating_system(self): + self.assertIsInstance(self.linux_operating_sys, LinuxDistro) + + def test_linux_compiler_strategy(self): + self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") + + + def test_cray_front_end_compiler_list(self): + """ Operating systems will now be in charge of finding compilers. + So, depending on which operating system you want to build for + or which operating system you are on, then you could detect + compilers in a certain way. Cray linux environment on the front + end is just a regular linux distro whereas the Cray linux compute + node is a stripped down version which modules are important + """ + self.assertEquals(True, False) diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index 90382dfc4a..b5ba0ecf35 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -42,6 +42,13 @@ class ConcretizeTest(MockPackagesTest): self.check_normalize('optional-dep-test+a', Spec('optional-dep-test+a', Spec('a'))) + self.check_normalize('optional-dep-test a=true', + Spec('optional-dep-test a=true', Spec('a'))) + + + self.check_normalize('optional-dep-test a=true', + Spec('optional-dep-test+a', Spec('a'))) + self.check_normalize('optional-dep-test@1.1', Spec('optional-dep-test@1.1', Spec('b'))) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 4645f98565..712f07ac4d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -29,8 +29,11 @@ You can find the dummy packages here:: spack/lib/spack/spack/test/mock_packages """ import spack +import spack.architecture import spack.package +from llnl.util.lang import list_modules + from spack.spec import Spec from spack.test.mock_packages_test import * @@ -239,8 +242,10 @@ class SpecDagTest(MockPackagesTest): def test_unsatisfiable_architecture(self): - self.set_pkg_dep('mpileaks', 'mpich=bgqos_0') - spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + platform = spack.architecture.sys_type() + + self.set_pkg_dep('mpileaks', 'mpich platform=test target=be') + spec = Spec('mpileaks ^mpich platform=test target=fe ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 60eb86d652..9876bfd5a8 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,6 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest +import spack.architecture from spack.spec import * from spack.test.mock_packages_test import * @@ -106,7 +108,8 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_namespaced_dep(self): - """Ensure spec from same or unspecified namespace satisfies namespace constraint.""" + """Ensure spec from same or unspecified namespace satisfies namespace + constraint.""" self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich') self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi') @@ -138,11 +141,16 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_architecture(self): - self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') - self.check_satisfies('foo=bgqos_0', '=bgqos_0') - - self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0') + platform = spack.architecture.sys_type() + self.check_satisfies( + 'foo platform=test target=frontend os=frontend', + 'platform=test target=frontend os=frontend') + self.check_satisfies( + 'foo platform=test target=backend os=backend', + 'platform=test target=backend', 'platform=test os=backend') + self.check_satisfies( + 'foo platform=test target=default_target os=default_os', + 'platform=test target=default_target os=default_os') def test_satisfies_dependencies(self): @@ -157,10 +165,14 @@ class SpecSematicsTest(MockPackagesTest): self.check_satisfies('mpileaks^mpich@2.0', '^mpich@1:3') self.check_unsatisfiable('mpileaks^mpich@1.2', '^mpich@2.0') - self.check_satisfies('mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') + self.check_satisfies( + 'mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') def test_satisfies_virtual_dependencies(self): @@ -190,12 +202,20 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_matching_variant(self): self.check_satisfies('mpich+foo', 'mpich+foo') self.check_satisfies('mpich~foo', 'mpich~foo') + self.check_satisfies('mpich foo=1', 'mpich foo=1') + + #confirm that synonymous syntax works correctly + self.check_satisfies('mpich+foo', 'mpich foo=True') + self.check_satisfies('mpich foo=true', 'mpich+foo') + self.check_satisfies('mpich~foo', 'mpich foo=FALSE') + self.check_satisfies('mpich foo=False', 'mpich~foo') def test_satisfies_unconstrained_variant(self): # only asked for mpich, no constraints. Either will do. self.check_satisfies('mpich+foo', 'mpich') self.check_satisfies('mpich~foo', 'mpich') + self.check_satisfies('mpich foo=1', 'mpich') def test_unsatisfiable_variants(self): @@ -204,16 +224,44 @@ class SpecSematicsTest(MockPackagesTest): # 'mpich' is not concrete: self.check_satisfies('mpich', 'mpich+foo', False) self.check_satisfies('mpich', 'mpich~foo', False) + self.check_satisfies('mpich', 'mpich foo=1', False) # 'mpich' is concrete: self.check_unsatisfiable('mpich', 'mpich+foo', True) self.check_unsatisfiable('mpich', 'mpich~foo', True) + self.check_unsatisfiable('mpich', 'mpich foo=1', True) def test_unsatisfiable_variant_mismatch(self): # No matchi in specs self.check_unsatisfiable('mpich~foo', 'mpich+foo') self.check_unsatisfiable('mpich+foo', 'mpich~foo') + self.check_unsatisfiable('mpich foo=1', 'mpich foo=2') + + + def test_satisfies_matching_compiler_flag(self): + self.check_satisfies('mpich cppflags="-O3"', 'mpich cppflags="-O3"') + self.check_satisfies('mpich cppflags="-O3 -Wall"', 'mpich cppflags="-O3 -Wall"') + + + def test_satisfies_unconstrained_compiler_flag(self): + # only asked for mpich, no constraints. Any will do. + self.check_satisfies('mpich cppflags="-O3"', 'mpich') + + + def test_unsatisfiable_compiler_flag(self): + # This case is different depending on whether the specs are concrete. + + # 'mpich' is not concrete: + self.check_satisfies('mpich', 'mpich cppflags="-O3"', False) + + # 'mpich' is concrete: + self.check_unsatisfiable('mpich', 'mpich cppflags="-O3"', True) + + + def test_unsatisfiable_compiler_flag_mismatch(self): + # No matchi in specs + self.check_unsatisfiable('mpich cppflags="-O3"', 'mpich cppflags="-O2"') def test_satisfies_virtual(self): @@ -301,18 +349,29 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain('libelf+debug+foo', 'libelf+debug', 'libelf+foo') self.check_constrain('libelf+debug+foo', 'libelf+debug', 'libelf+debug+foo') + self.check_constrain('libelf debug=2 foo=1', 'libelf debug=2', 'libelf foo=1') + self.check_constrain('libelf debug=2 foo=1', 'libelf debug=2', 'libelf debug=2 foo=1') + self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf~foo') self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') - def test_constrain_arch(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + def test_constrain_compiler_flags(self): + self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cppflags="-Wall"') + self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cflags="-O3" cppflags="-Wall"') + + def test_constrain_architecture(self): + self.check_constrain('libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os') + self.check_constrain('libelf target=default_target os=default_os', + 'libelf', + 'libelf target=default_target os=default_os') def test_constrain_compiler(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + self.check_constrain('libelf %gcc@4.4.7', 'libelf %gcc@4.4.7', 'libelf %gcc@4.4.7') + self.check_constrain('libelf %gcc@4.4.7', 'libelf', 'libelf %gcc@4.4.7') def test_invalid_constraint(self): @@ -321,9 +380,11 @@ class SpecSematicsTest(MockPackagesTest): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') + self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') - self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54') - + self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + self.check_invalid_constraint('libelf platform=test target=be os=be', + 'libelf target=fe os=fe') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -332,7 +393,12 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') - self.check_constrain_changed('libelf', '=bgqos_0') + self.check_constrain_changed('libelf', 'debug=2') + self.check_constrain_changed('libelf', 'cppflags="-O3"') + + platform = spack.architecture.sys_type() + self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) + self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) def test_constrain_not_changed(self): @@ -343,9 +409,12 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') - self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0') - self.check_constrain_not_changed('libelf^foo', 'libelf^foo') - self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') + self.check_constrain_not_changed('libelf debug=2', 'debug=2') + self.check_constrain_not_changed('libelf cppflags="-O3"', 'cppflags="-O3"') + + platform = spack.architecture.sys_type() + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf target='+default_target, 'target='+default_target) def test_constrain_dependency_changed(self): @@ -355,7 +424,10 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') - self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0') + + platform = spack.architecture.sys_type() + default_target = platform.target('default_target').name + self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) def test_constrain_dependency_not_changed(self): @@ -365,4 +437,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') - self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') + self.check_constrain_not_changed('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"') + + platform = spack.architecture.sys_type() + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf^foo target='+default_target, 'libelf^foo target='+default_target) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 928d111ea9..4a534d7b5c 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -58,7 +58,7 @@ class SpecSyntaxTest(unittest.TestCase): # ================================================================================ # Parse checks # ================================================================================ - def check_parse(self, expected, spec=None): + def check_parse(self, expected, spec=None, remove_arch=True): """Assert that the provided spec is able to be parsed. If this is called with one argument, it assumes that the string is canonical (i.e., no spaces and ~ instead of - for variants) and that it @@ -70,6 +70,7 @@ class SpecSyntaxTest(unittest.TestCase): if spec is None: spec = expected output = spack.spec.parse(spec) + parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) @@ -104,6 +105,8 @@ class SpecSyntaxTest(unittest.TestCase): def test_full_specs(self): self.check_parse("mvapich_foo^_openmpi@1.2:1.4,1.6%intel@12.1+debug~qt_4^stackwalker@8.1_1e") + self.check_parse("mvapich_foo^_openmpi@1.2:1.4,1.6%intel@12.1 debug=2~qt_4^stackwalker@8.1_1e") + self.check_parse('mvapich_foo^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags="-O3"+debug~qt_4^stackwalker@8.1_1e') def test_canonicalize(self): self.check_parse( @@ -128,7 +131,10 @@ class SpecSyntaxTest(unittest.TestCase): def test_duplicate_variant(self): self.assertRaises(DuplicateVariantError, self.check_parse, "x@1.2+debug+debug") - self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2+debug+debug") + self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2+debug debug=true") + self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2 debug=false debug=true") + self.assertRaises(DuplicateVariantError, self.check_parse, "x ^y@1.2 debug=false~debug") + def test_duplicate_depdendence(self): self.assertRaises(DuplicateDependencyError, self.check_parse, "x ^y ^y") diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index a026403e2e..4624f901c8 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -43,7 +43,6 @@ class VersionsTest(unittest.TestCase): self.assertFalse(a > b) self.assertFalse(a >= b) - def assert_ver_gt(self, a, b): a, b = ver(a), ver(b) self.assertTrue(a > b) @@ -53,7 +52,6 @@ class VersionsTest(unittest.TestCase): self.assertFalse(a < b) self.assertFalse(a <= b) - def assert_ver_eq(self, a, b): a, b = ver(a), ver(b) self.assertFalse(a > b) @@ -63,55 +61,43 @@ class VersionsTest(unittest.TestCase): self.assertFalse(a < b) self.assertTrue(a <= b) - def assert_in(self, needle, haystack): self.assertTrue(ver(needle) in ver(haystack)) - def assert_not_in(self, needle, haystack): self.assertFalse(ver(needle) in ver(haystack)) - def assert_canonical(self, canonical_list, version_list): self.assertEqual(ver(canonical_list), ver(version_list)) - def assert_overlaps(self, v1, v2): self.assertTrue(ver(v1).overlaps(ver(v2))) - def assert_no_overlap(self, v1, v2): self.assertFalse(ver(v1).overlaps(ver(v2))) - def assert_satisfies(self, v1, v2): self.assertTrue(ver(v1).satisfies(ver(v2))) - def assert_does_not_satisfy(self, v1, v2): self.assertFalse(ver(v1).satisfies(ver(v2))) - def check_intersection(self, expected, a, b): self.assertEqual(ver(expected), ver(a).intersection(ver(b))) - def check_union(self, expected, a, b): self.assertEqual(ver(expected), ver(a).union(ver(b))) - def test_two_segments(self): self.assert_ver_eq('1.0', '1.0') self.assert_ver_lt('1.0', '2.0') self.assert_ver_gt('2.0', '1.0') - def test_three_segments(self): self.assert_ver_eq('2.0.1', '2.0.1') self.assert_ver_lt('2.0', '2.0.1') self.assert_ver_gt('2.0.1', '2.0') - def test_alpha(self): # TODO: not sure whether I like this. 2.0.1a is *usually* # TODO: less than 2.0.1, but special-casing it makes version @@ -120,7 +106,6 @@ class VersionsTest(unittest.TestCase): self.assert_ver_gt('2.0.1a', '2.0.1') self.assert_ver_lt('2.0.1', '2.0.1a') - def test_patch(self): self.assert_ver_eq('5.5p1', '5.5p1') self.assert_ver_lt('5.5p1', '5.5p2') @@ -129,7 +114,6 @@ class VersionsTest(unittest.TestCase): self.assert_ver_lt('5.5p1', '5.5p10') self.assert_ver_gt('5.5p10', '5.5p1') - def test_num_alpha_with_no_separator(self): self.assert_ver_lt('10xyz', '10.1xyz') self.assert_ver_gt('10.1xyz', '10xyz') @@ -137,7 +121,6 @@ class VersionsTest(unittest.TestCase): self.assert_ver_lt('xyz10', 'xyz10.1') self.assert_ver_gt('xyz10.1', 'xyz10') - def test_alpha_with_dots(self): self.assert_ver_eq('xyz.4', 'xyz.4') self.assert_ver_lt('xyz.4', '8') @@ -145,30 +128,25 @@ class VersionsTest(unittest.TestCase): self.assert_ver_lt('xyz.4', '2') self.assert_ver_gt('2', 'xyz.4') - def test_nums_and_patch(self): self.assert_ver_lt('5.5p2', '5.6p1') self.assert_ver_gt('5.6p1', '5.5p2') self.assert_ver_lt('5.6p1', '6.5p1') self.assert_ver_gt('6.5p1', '5.6p1') - def test_rc_versions(self): self.assert_ver_gt('6.0.rc1', '6.0') self.assert_ver_lt('6.0', '6.0.rc1') - def test_alpha_beta(self): self.assert_ver_gt('10b2', '10a1') self.assert_ver_lt('10a2', '10b2') - def test_double_alpha(self): self.assert_ver_eq('1.0aa', '1.0aa') self.assert_ver_lt('1.0a', '1.0aa') self.assert_ver_gt('1.0aa', '1.0a') - def test_padded_numbers(self): self.assert_ver_eq('10.0001', '10.0001') self.assert_ver_eq('10.0001', '10.1') @@ -176,24 +154,20 @@ class VersionsTest(unittest.TestCase): self.assert_ver_lt('10.0001', '10.0039') self.assert_ver_gt('10.0039', '10.0001') - def test_close_numbers(self): self.assert_ver_lt('4.999.9', '5.0') self.assert_ver_gt('5.0', '4.999.9') - def test_date_stamps(self): self.assert_ver_eq('20101121', '20101121') self.assert_ver_lt('20101121', '20101122') self.assert_ver_gt('20101122', '20101121') - def test_underscores(self): self.assert_ver_eq('2_0', '2_0') self.assert_ver_eq('2.0', '2_0') self.assert_ver_eq('2_0', '2.0') - def test_rpm_oddities(self): self.assert_ver_eq('1b.fc17', '1b.fc17') self.assert_ver_lt('1b.fc17', '1.fc17') @@ -202,7 +176,6 @@ class VersionsTest(unittest.TestCase): self.assert_ver_gt('1g.fc17', '1.fc17') self.assert_ver_lt('1.fc17', '1g.fc17') - # Stuff below here is not taken from RPM's tests and is # unique to spack def test_version_ranges(self): @@ -214,7 +187,6 @@ class VersionsTest(unittest.TestCase): self.assert_ver_lt('1.2:1.4', '1.5:1.6') self.assert_ver_gt('1.5:1.6', '1.2:1.4') - def test_contains(self): self.assert_in('1.3', '1.2:1.4') self.assert_in('1.2.5', '1.2:1.4') @@ -233,7 +205,6 @@ class VersionsTest(unittest.TestCase): self.assert_in('1.4.1', '1.2.7:1.4') self.assert_not_in('1.4.1', '1.2.7:1.4.0') - def test_in_list(self): self.assert_in('1.2', ['1.5', '1.2', '1.3']) self.assert_in('1.2.5', ['1.5', '1.2:1.3']) @@ -245,7 +216,6 @@ class VersionsTest(unittest.TestCase): self.assert_not_in('1.2.5:1.5', ['1.5', '1.2:1.3']) self.assert_not_in('1.1:1.2.5', ['1.5', '1.2:1.3']) - def test_ranges_overlap(self): self.assert_overlaps('1.2', '1.2') self.assert_overlaps('1.2.1', '1.2.1') @@ -262,7 +232,6 @@ class VersionsTest(unittest.TestCase): self.assert_overlaps(':', '1.6:1.9') self.assert_overlaps('1.6:1.9', ':') - def test_overlap_with_containment(self): self.assert_in('1.6.5', '1.6') self.assert_in('1.6.5', ':1.6') @@ -273,7 +242,6 @@ class VersionsTest(unittest.TestCase): self.assert_not_in(':1.6', '1.6.5') self.assert_in('1.6.5', ':1.6') - def test_lists_overlap(self): self.assert_overlaps('1.2b:1.7,5', '1.6:1.9,1') self.assert_overlaps('1,2,3,4,5', '3,4,5,6,7') @@ -287,7 +255,6 @@ class VersionsTest(unittest.TestCase): self.assert_no_overlap('1,2,3,4,5', '6,7') self.assert_no_overlap('1,2,3,4,5', '6:7') - def test_canonicalize_list(self): self.assert_canonical(['1.2', '1.3', '1.4'], ['1.2', '1.3', '1.3', '1.4']) @@ -316,7 +283,6 @@ class VersionsTest(unittest.TestCase): self.assert_canonical([':'], [':,1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4']) - def test_intersection(self): self.check_intersection('2.5', '1.0:2.5', '2.5:3.0') @@ -325,12 +291,11 @@ class VersionsTest(unittest.TestCase): self.check_intersection('0:1', ':', '0:1') self.check_intersection(['1.0', '2.5:2.7'], - ['1.0:2.7'], ['2.5:3.0','1.0']) + ['1.0:2.7'], ['2.5:3.0', '1.0']) self.check_intersection(['2.5:2.7'], - ['1.1:2.7'], ['2.5:3.0','1.0']) + ['1.1:2.7'], ['2.5:3.0', '1.0']) self.check_intersection(['0:1'], [':'], ['0:1']) - def test_intersect_with_containment(self): self.check_intersection('1.6.5', '1.6.5', ':1.6') self.check_intersection('1.6.5', ':1.6', '1.6.5') @@ -338,7 +303,6 @@ class VersionsTest(unittest.TestCase): self.check_intersection('1.6:1.6.5', ':1.6.5', '1.6') self.check_intersection('1.6:1.6.5', '1.6', ':1.6.5') - def test_union_with_containment(self): self.check_union(':1.6', '1.6.5', ':1.6') self.check_union(':1.6', ':1.6', '1.6.5') @@ -346,8 +310,6 @@ class VersionsTest(unittest.TestCase): self.check_union(':1.6', ':1.6.5', '1.6') self.check_union(':1.6', '1.6', ':1.6.5') - - def test_union_with_containment(self): self.check_union(':', '1.0:', ':2.0') self.check_union('1:4', '1:3', '2:4') @@ -356,7 +318,6 @@ class VersionsTest(unittest.TestCase): # Tests successor/predecessor case. self.check_union('1:4', '1:2', '3:4') - def test_basic_version_satisfaction(self): self.assert_satisfies('4.7.3', '4.7.3') @@ -372,7 +333,6 @@ class VersionsTest(unittest.TestCase): self.assert_does_not_satisfy('4.8', '4.9') self.assert_does_not_satisfy('4', '4.9') - def test_basic_version_satisfaction_in_lists(self): self.assert_satisfies(['4.7.3'], ['4.7.3']) @@ -388,7 +348,6 @@ class VersionsTest(unittest.TestCase): self.assert_does_not_satisfy(['4.8'], ['4.9']) self.assert_does_not_satisfy(['4'], ['4.9']) - def test_version_range_satisfaction(self): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_satisfies('4.3.0', '4.3:4.7') @@ -400,7 +359,6 @@ class VersionsTest(unittest.TestCase): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_does_not_satisfy('4.8.0', '4.3:4.7') - def test_version_range_satisfaction_in_lists(self): self.assert_satisfies(['4.7b6'], ['4.3:4.7']) self.assert_satisfies(['4.3.0'], ['4.3:4.7']) @@ -423,3 +381,11 @@ class VersionsTest(unittest.TestCase): self.assert_satisfies('4.8.0', '4.2, 4.3:4.8') self.assert_satisfies('4.8.2', '4.2, 4.3:4.8') + + def test_formatted_strings(self): + versions = '1.2.3', '1_2_3', '1-2-3' + for item in versions: + v = Version(item) + self.assertEqual(v.dotted, '1.2.3') + self.assertEqual(v.dashed, '1-2-3') + self.assertEqual(v.underscored, '1_2_3') diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d2ccfde69b..14b56e8d6c 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -22,10 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -__all__ = ['Executable', 'which', 'ProcessError'] import os -import sys import re import subprocess import inspect @@ -34,9 +32,12 @@ import llnl.util.tty as tty import spack import spack.error +__all__ = ['Executable', 'which', 'ProcessError'] + class Executable(object): """Class representing a program that can be run on the command line.""" + def __init__(self, name): self.exe = name.split(' ') self.returncode = None @@ -44,16 +45,13 @@ class Executable(object): if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name) - def add_default_arg(self, arg): self.exe.append(arg) - @property def command(self): return ' '.join(self.exe) - def __call__(self, *args, **kwargs): """Run this executable in a subprocess. @@ -105,6 +103,8 @@ class Executable(object): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) + env = kwargs.get('env', None) + # TODO: This is deprecated. Remove in a future version. return_output = kwargs.pop("return_output", False) @@ -114,8 +114,8 @@ class Executable(object): else: output = kwargs.pop("output", None) - error = kwargs.pop("error", None) - input = kwargs.pop("input", None) + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) if input is str: raise ValueError("Cannot use `str` as input stream.") @@ -126,85 +126,91 @@ class Executable(object): return subprocess.PIPE, False else: return arg, False + ostream, close_ostream = streamify(output, 'w') - estream, close_estream = streamify(error, 'w') - istream, close_istream = streamify(input, 'r') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): - ignore_errors = (ignore_errors,) + ignore_errors = (ignore_errors, ) quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: - tty.warn("Quotes in command arguments can confuse scripts like configure.", - "The following arguments may cause problems when executed:", - str("\n".join([" "+arg for arg in quoted_args])), - "Quotes aren't needed because spack doesn't use a shell.", - "Consider removing them") + tty.warn( + "Quotes in command arguments can confuse scripts like" + " configure.", + "The following arguments may cause problems when executed:", + str("\n".join([" " + arg for arg in quoted_args])), + "Quotes aren't needed because spack doesn't use a shell.", + "Consider removing them") cmd = self.exe + list(args) - cmd_line = "'%s'" % "' '".join(map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) + cmd_line = "'%s'" % "' '".join( + map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: proc = subprocess.Popen( - cmd, stdin=istream, stderr=estream, stdout=ostream) + cmd, + stdin=istream, + stderr=estream, + stdout=ostream, + env=env) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): - raise ProcessError("Command exited with status %d:" - % proc.returncode, cmd_line) + raise ProcessError("Command exited with status %d:" % + proc.returncode, cmd_line) + if output is str or error is str: result = '' - if output is str: result += out - if error is str: result += err + if output is str: + result += out + if error is str: + result += err return result except OSError, e: raise ProcessError( - "%s: %s" % (self.exe[0], e.strerror), - "Command: " + cmd_line) + "%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line) except subprocess.CalledProcessError, e: if fail_on_error: raise ProcessError( - str(e), - "\nExit status %d when invoking command: %s" - % (proc.returncode, cmd_line)) + str(e), "\nExit status %d when invoking command: %s" % + (proc.returncode, cmd_line)) finally: - if close_ostream: output.close() - if close_estream: error.close() - if close_istream: input.close() - + if close_ostream: + output.close() + if close_estream: + error.close() + if close_istream: + input.close() def __eq__(self, other): return self.exe == other.exe - def __neq__(self, other): return not (self == other) - def __hash__(self): - return hash((type(self),) + tuple(self.exe)) - + return hash((type(self), ) + tuple(self.exe)) def __repr__(self): return "<exe: %s>" % self.exe - def __str__(self): return ' '.join(self.exe) - def which(name, **kwargs): """Finds an executable in the path like command-line which.""" - path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) + path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) required = kwargs.get('required', False) if not path: @@ -233,14 +239,16 @@ class ProcessError(spack.error.SpackError): @property def long_message(self): msg = self._long_message - if msg: msg += "\n\n" + if msg: + msg += "\n\n" if self.build_log: msg += "See build log for details:\n" msg += " %s" % self.build_log if self.package_context: - if msg: msg += "\n\n" + if msg: + msg += "\n\n" msg += '\n'.join(self.package_context) return msg @@ -267,7 +275,7 @@ def _get_package_context(): frame = f[0] # Find a frame with 'self' in the local variables. - if not 'self' in frame.f_locals: + if 'self' not in frame.f_locals: continue # Look only at a frame in a subclass of spack.Package @@ -280,9 +288,8 @@ def _get_package_context(): # Build a message showing where in install we failed. lines.append("%s:%d, in %s:" % ( - inspect.getfile(frame.f_code), - frame.f_lineno, - frame.f_code.co_name)) + inspect.getfile(frame.f_code), frame.f_lineno, frame.f_code.co_name + )) sourcelines, start = inspect.getsourcelines(frame) for i, line in enumerate(sourcelines): diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 20686d44b2..ad875f5ef5 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -32,5 +32,5 @@ currently variants are just flags. class Variant(object): """Represents a variant on a build. Can be either on or off.""" def __init__(self, default, description): - self.default = bool(default) + self.default = default self.description = str(description) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 247f6d2362..858d581472 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -43,16 +43,16 @@ be called on any of the types:: intersection concrete """ -import os -import sys import re from bisect import bisect_left from functools import wraps + from functools_backport import total_ordering # Valid version characters VALID_VERSION = r'[A-Za-z0-9_.-]' + def int_if_int(string): """Convert a string to int if possible. Otherwise, return a string.""" try: @@ -62,10 +62,11 @@ def int_if_int(string): def coerce_versions(a, b): - """Convert both a and b to the 'greatest' type between them, in this order: + """ + Convert both a and b to the 'greatest' type between them, in this order: Version < VersionRange < VersionList - This is used to simplify comparison operations below so that we're always - comparing things that are of the same type. + This is used to simplify comparison operations below so that we're always + comparing things that are of the same type. """ order = (Version, VersionRange, VersionList) ta, tb = type(a), type(b) @@ -105,6 +106,7 @@ def coerced(method): @total_ordering class Version(object): """Class to represent versions""" + def __init__(self, string): string = str(string) @@ -124,6 +126,17 @@ class Version(object): # last element of separators is '' self.separators = tuple(re.split(segment_regex, string)[1:-1]) + @property + def dotted(self): + return '.'.join(str(x) for x in self.version) + + @property + def underscored(self): + return '_'.join(str(x) for x in self.version) + + @property + def dashed(self): + return '-'.join(str(x) for x in self.version) def up_to(self, index): """Return a version string up to the specified component, exclusive. @@ -131,15 +144,12 @@ class Version(object): """ return '.'.join(str(x) for x in self[:index]) - def lowest(self): return self - def highest(self): return self - @coerced def satisfies(self, other): """A Version 'satisfies' another if it is at least as specific and has a @@ -147,11 +157,10 @@ class Version(object): gcc@4.7 so that when a user asks to build with gcc@4.7, we can find a suitable compiler. """ - nself = len(self.version) + nself = len(self.version) nother = len(other.version) return nother <= nself and self.version[:nother] == other.version - def wildcard(self): """Create a regex that will match variants of this version string.""" def a_or_n(seg): @@ -181,28 +190,22 @@ class Version(object): wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1) return wc - def __iter__(self): return iter(self.version) - def __getitem__(self, idx): return tuple(self.version[idx]) - def __repr__(self): return self.string - def __str__(self): return self.string - @property def concrete(self): return self - @coerced def __lt__(self, other): """Version comparison is designed for consistency with the way RPM @@ -235,28 +238,23 @@ class Version(object): # If the common prefix is equal, the one with more segments is bigger. return len(self.version) < len(other.version) - @coerced def __eq__(self, other): return (other is not None and type(other) == Version and self.version == other.version) - def __ne__(self, other): return not (self == other) - def __hash__(self): return hash(self.version) - @coerced def __contains__(self, other): if other is None: return False return other.version[:len(self.version)] == self.version - def is_predecessor(self, other): """True if the other version is the immediate predecessor of this one. That is, NO versions v exist such that: @@ -269,16 +267,13 @@ class Version(object): ol = other.version[-1] return type(sl) == int and type(ol) == int and (ol - sl == 1) - def is_successor(self, other): return other.is_predecessor(self) - @coerced def overlaps(self, other): return self in other or other in self - @coerced def union(self, other): if self == other or other in self: @@ -288,7 +283,6 @@ class Version(object): else: return VersionList([self, other]) - @coerced def intersection(self, other): if self == other: @@ -299,6 +293,7 @@ class Version(object): @total_ordering class VersionRange(object): + def __init__(self, start, end): if isinstance(start, basestring): start = Version(start) @@ -310,15 +305,12 @@ class VersionRange(object): if start and end and end < start: raise ValueError("Invalid Version range: %s" % self) - def lowest(self): return self.start - def highest(self): return self.end - @coerced def __lt__(self, other): """Sort VersionRanges lexicographically so that they are ordered first @@ -331,28 +323,24 @@ class VersionRange(object): s, o = self, other if s.start != o.start: - return s.start is None or (o.start is not None and s.start < o.start) + return s.start is None or (o.start is not None and s.start < o.start) # NOQA: ignore=E501 return (s.end != o.end and o.end is None or (s.end is not None and s.end < o.end)) - @coerced def __eq__(self, other): return (other is not None and type(other) == VersionRange and self.start == other.start and self.end == other.end) - def __ne__(self, other): return not (self == other) - @property def concrete(self): return self.start if self.start == self.end else None - @coerced def __contains__(self, other): if other is None: @@ -373,57 +361,55 @@ class VersionRange(object): other.end in self.end))) return in_upper - @coerced def satisfies(self, other): - """A VersionRange satisfies another if some version in this range - would satisfy some version in the other range. To do this it must - either: - a) Overlap with the other range - b) The start of this range satisfies the end of the other range. - - This is essentially the same as overlaps(), but overlaps assumes - that its arguments are specific. That is, 4.7 is interpreted as - 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied - by 4.7.3.5, etc. - - Rationale: - If a user asks for gcc@4.5:4.7, and a package is only compatible with - gcc@4.7.3:4.8, then that package should be able to build under the - constraints. Just using overlaps() would not work here. - - Note that we don't need to check whether the end of this range - would satisfy the start of the other range, because overlaps() - already covers that case. - - Note further that overlaps() is a symmetric operation, while - satisfies() is not. + """ + A VersionRange satisfies another if some version in this range + would satisfy some version in the other range. To do this it must + either: + a) Overlap with the other range + b) The start of this range satisfies the end of the other range. + + This is essentially the same as overlaps(), but overlaps assumes + that its arguments are specific. That is, 4.7 is interpreted as + 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied + by 4.7.3.5, etc. + + Rationale: + If a user asks for gcc@4.5:4.7, and a package is only compatible with + gcc@4.7.3:4.8, then that package should be able to build under the + constraints. Just using overlaps() would not work here. + + Note that we don't need to check whether the end of this range + would satisfy the start of the other range, because overlaps() + already covers that case. + + Note further that overlaps() is a symmetric operation, while + satisfies() is not. """ return (self.overlaps(other) or # if either self.start or other.end are None, then this can't # satisfy, or overlaps() would've taken care of it. self.start and other.end and self.start.satisfies(other.end)) - @coerced def overlaps(self, other): - return ((self.start == None or other.end is None or + return ((self.start is None or other.end is None or self.start <= other.end or other.end in self.start or self.start in other.end) and - (other.start is None or self.end == None or + (other.start is None or self.end is None or other.start <= self.end or other.start in self.end or self.end in other.start)) - @coerced def union(self, other): if not self.overlaps(other): if (self.end is not None and other.start is not None and - self.end.is_predecessor(other.start)): + self.end.is_predecessor(other.start)): return VersionRange(self.start, other.end) if (other.end is not None and self.start is not None and - other.end.is_predecessor(self.start)): + other.end.is_predecessor(self.start)): return VersionRange(other.start, self.end) return VersionList([self, other]) @@ -442,13 +428,12 @@ class VersionRange(object): else: end = self.end # TODO: See note in intersection() about < and in discrepancy. - if not other.end in self.end: + if other.end not in self.end: if end in other.end or other.end > self.end: end = other.end return VersionRange(start, end) - @coerced def intersection(self, other): if self.overlaps(other): @@ -470,7 +455,7 @@ class VersionRange(object): # 1.6 < 1.6.5 = True (lexicographic) # Should 1.6 NOT be less than 1.6.5? Hm. # Here we test (not end in other.end) first to avoid paradox. - if other.end is not None and not end in other.end: + if other.end is not None and end not in other.end: if other.end < end or other.end in end: end = other.end @@ -479,15 +464,12 @@ class VersionRange(object): else: return VersionList() - def __hash__(self): return hash((self.start, self.end)) - def __repr__(self): return self.__str__() - def __str__(self): out = "" if self.start: @@ -501,6 +483,7 @@ class VersionRange(object): @total_ordering class VersionList(object): """Sorted, non-redundant list of Versions and VersionRanges.""" + def __init__(self, vlist=None): self.versions = [] if vlist is not None: @@ -515,7 +498,6 @@ class VersionList(object): for v in vlist: self.add(ver(v)) - def add(self, version): if type(version) in (Version, VersionRange): # This normalizes single-value version ranges. @@ -524,9 +506,9 @@ class VersionList(object): i = bisect_left(self, version) - while i-1 >= 0 and version.overlaps(self[i-1]): - version = version.union(self[i-1]) - del self.versions[i-1] + while i - 1 >= 0 and version.overlaps(self[i - 1]): + version = version.union(self[i - 1]) + del self.versions[i - 1] i -= 1 while i < len(self) and version.overlaps(self[i]): @@ -542,7 +524,6 @@ class VersionList(object): else: raise TypeError("Can't add %s to VersionList" % type(version)) - @property def concrete(self): if len(self) == 1: @@ -550,11 +531,9 @@ class VersionList(object): else: return None - def copy(self): return VersionList(self) - def lowest(self): """Get the lowest version in the list.""" if not self: @@ -562,7 +541,6 @@ class VersionList(object): else: return self[0].lowest() - def highest(self): """Get the highest version in the list.""" if not self: @@ -570,7 +548,6 @@ class VersionList(object): else: return self[-1].highest() - @coerced def overlaps(self, other): if not other or not self: @@ -586,14 +563,12 @@ class VersionList(object): o += 1 return False - def to_dict(self): """Generate human-readable dict for YAML.""" if self.concrete: - return { 'version' : str(self[0]) } + return {'version': str(self[0])} else: - return { 'versions' : [str(v) for v in self] } - + return {'versions': [str(v) for v in self]} @staticmethod def from_dict(dictionary): @@ -605,7 +580,6 @@ class VersionList(object): else: raise ValueError("Dict must have 'version' or 'versions' in it.") - @coerced def satisfies(self, other, strict=False): """A VersionList satisfies another if some version in the list @@ -633,20 +607,17 @@ class VersionList(object): o += 1 return False - @coerced def update(self, other): for v in other.versions: self.add(v) - @coerced def union(self, other): result = self.copy() result.update(other) return result - @coerced def intersection(self, other): # TODO: make this faster. This is O(n^2). @@ -656,7 +627,6 @@ class VersionList(object): result.add(s.intersection(o)) return result - @coerced def intersect(self, other): """Intersect this spec's list with other. @@ -678,50 +648,40 @@ class VersionList(object): if i == 0: if version not in self[0]: return False - elif all(version not in v for v in self[i-1:]): + elif all(version not in v for v in self[i - 1:]): return False return True - def __getitem__(self, index): return self.versions[index] - def __iter__(self): return iter(self.versions) - def __reversed__(self): return reversed(self.versions) - def __len__(self): return len(self.versions) - @coerced def __eq__(self, other): return other is not None and self.versions == other.versions - def __ne__(self, other): return not (self == other) - @coerced def __lt__(self, other): return other is not None and self.versions < other.versions - def __hash__(self): return hash(tuple(self.versions)) - def __str__(self): return ",".join(str(v) for v in self.versions) - def __repr__(self): return str(self.versions) @@ -730,7 +690,7 @@ def _string_to_version(string): """Converts a string to a Version, VersionList, or VersionRange. This is private. Client code should use ver(). """ - string = string.replace(' ','') + string = string.replace(' ', '') if ',' in string: return VersionList(string.split(',')) @@ -738,7 +698,7 @@ def _string_to_version(string): elif ':' in string: s, e = string.split(':') start = Version(s) if s else None - end = Version(e) if e else None + end = Version(e) if e else None return VersionRange(start, end) else: diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index 91ad77c8fd..bb8333f023 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -67,10 +67,15 @@ class ProviderIndex(object): if type(spec) != spack.spec.Spec: spec = spack.spec.Spec(spec) + if not spec.name: + # Empty specs do not have a package + return + assert(not spec.virtual) pkg = spec.package for provided_spec, provider_spec in pkg.provided.iteritems(): + provider_spec.compiler_flags = spec.compiler_flags.copy()#We want satisfaction other than flags if provider_spec.satisfies(spec, deps=False): provided_name = provided_spec.name diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py new file mode 100644 index 0000000000..c2d084d6c3 --- /dev/null +++ b/lib/spack/spack/yaml_version_check.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Yaml Version Check is a module for ensuring that config file +formats are compatible with the current version of Spack.""" +import os.path +import os +import llnl.util.tty as tty +import spack.util.spack_yaml as syaml +import spack.config + + +def check_yaml_versions(): + check_compiler_yaml_version() + +def check_compiler_yaml_version(): + config_scopes = spack.config.config_scopes + for scope in config_scopes.values(): + file_name = os.path.join(scope.path, 'compilers.yaml') + data = None + if os.path.isfile(file_name): + with open(file_name) as f: + data = syaml.load(f) + + if data: + compilers = data['compilers'] + if len(compilers) > 0: + if (not isinstance(compilers, list)) or 'operating_system' not in compilers[0]['compiler']: + new_file = os.path.join(scope.path, '_old_compilers.yaml') + tty.warn('%s in out of date compilers format. ' + 'Moved to %s. Spack automatically generate ' + 'a compilers config file ' + % (file_name, new_file)) + os.rename(file_name, new_file) diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 index 722c7fcba6..57534eb6e2 100755 --- a/share/spack/qa/run-flake8 +++ b/share/spack/qa/run-flake8 @@ -20,10 +20,20 @@ fi # Check if changed files are flake8 conformant [framework] changed=$(git diff --name-only develop... | grep '.py$') -# Exempt url lines in changed packages from overlong line errors. +# Add approved style exemptions to the changed packages. for file in $changed; do if [[ $file = *package.py ]]; then - perl -i~ -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file; + cp "$file" "$file~" + + # Exempt lines with urls and descriptions from overlong line errors. + perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + + # Exempt '@when' decorated functions from redefinition errors. + perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file fi done diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml new file mode 100644 index 0000000000..5f8b38007b --- /dev/null +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -0,0 +1,40 @@ +compilers: + all: + clang@3.3: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: None + strategy: PATH + gcc@4.5.0: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + modules: None + strategy: PATH + gcc@5.2.0: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-gnu + - gcc/5.2.0 + strategy: MODULES + intel@15.0.1: + cc: cc + ccx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-intel + - intel/15.0.1 + strategy: MODULES + intel@15.1.2: + cc: /path/to/icc + cxx: /path/to/ic++ + f77: /path/to/ifort + fc: /path/to/ifort + strategy: PATH
\ No newline at end of file diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py new file mode 100644 index 0000000000..260dcbe851 --- /dev/null +++ b/var/spack/packages/adios/package.py @@ -0,0 +1,38 @@ +import os + +from spack import * +class Adios(Package): + """The Adaptable IO System (ADIOS) provides a simple, + flexible way for scientists to describe the + data in their code that may need to be written, + read, or processed outside of the running simulation + """ + + homepage = "http://www.olcf.ornl.gov/center-projects/adios/" + url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz" + + version('1.9.0', 'dbf5cb10e32add2f04c9b4052b7ffa76') + + # Lots of setting up here for this package + # module swap PrgEnv-intel PrgEnv-$COMP + # module load cray-netcdf/4.3.3.1 + # module load cray-hdf5/1.8.14 + # module load python/2.7.10 + depends_on('hdf5') + depends_on('mxml') + + def install(self, spec, prefix): + configure_args = ["--prefix=%s" % prefix, + "--with-mxml=%s" % spec['mxml'].prefix, + "--with-hdf5=%s" % spec['hdf5'].prefix, + "--with-netcdf=%s" % os.environ["NETCDF_DIR"], + "--with-infiniband=no", + "MPICC=cc","MPICXX=CC","MPIFC=ftn", + "CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"] + + if spec.satisfies('%gcc'): + configure_args.extend(["CC=gcc", "CXX=g++", "FC=gfortran"]) + + configure(*configure_args) + make() + make("install") diff --git a/var/spack/packages/mxml/package.py b/var/spack/packages/mxml/package.py new file mode 100644 index 0000000000..f79251d312 --- /dev/null +++ b/var/spack/packages/mxml/package.py @@ -0,0 +1,26 @@ +import os +from spack import * + +class Mxml(Package): + """Mini-XML is a small XML library that you can use to read and write XML + and XML-like data files in your application without requiring large + non-standard libraries + """ + + homepage = "http://www.msweet.org" + url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" + + version('2.9', 'e21cad0f7aacd18f942aa0568a8dee19') + version('2.8', 'd85ee6d30de053581242c4a86e79a5d2') + version('2.7', '76f2ae49bf0f5745d5cb5d9507774dc9') + version('2.6', '68977789ae64985dddbd1a1a1652642e') + version('2.5', 'f706377fba630b39fa02fd63642b17e5') + + # module swap PrgEnv-intel PrgEnv-$COMP (Can use whatever compiler you want to use) + # Case statement to change CC and CXX flags + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, "--disable-shared", 'CFLAGS=-static') + make() + make("install") + diff --git a/var/spack/repos/builtin.mock/packages/externalmodule/package.py b/var/spack/repos/builtin.mock/packages/externalmodule/package.py new file mode 100644 index 0000000000..f7b0da3fd9 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externalmodule/package.py @@ -0,0 +1,37 @@ + +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalmodule(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/module-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('externalprereq') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/multimethod/package.py b/var/spack/repos/builtin.mock/packages/multimethod/package.py index 2d15722470..649afa5945 100644 --- a/var/spack/repos/builtin.mock/packages/multimethod/package.py +++ b/var/spack/repos/builtin.mock/packages/multimethod/package.py @@ -22,8 +22,11 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import imp +from llnl.util.filesystem import join_path +from spack.util.naming import mod_to_class from spack import * - +import spack.architecture class Multimethod(Package): """This package is designed for use with Spack's multimethod test. @@ -101,25 +104,26 @@ class Multimethod(Package): # - # Make sure we can switch methods on different architectures + # Make sure we can switch methods on different target # - @when('=x86_64') - def different_by_architecture(self): - return 'x86_64' - - @when('=ppc64') - def different_by_architecture(self): - return 'ppc64' - - @when('=ppc32') - def different_by_architecture(self): - return 'ppc32' - - @when('=arm64') - def different_by_architecture(self): - return 'arm64' - - +# for platform_name in ['cray_xc', 'darwin', 'linux']: +# file_path = join_path(spack.platform_path, platform_name) +# platform_mod = imp.load_source('spack.platforms', file_path + '.py') +# cls = getattr(platform_mod, mod_to_class(platform_name)) + +# platform = cls() + platform = spack.architecture.sys_type() + targets = platform.targets.values() + if len(targets) > 1: + targets = targets[:-1] + + for target in targets: + @when('target='+target.name) + def different_by_target(self): + if isinstance(self.spec.architecture.target,basestring): + return self.spec.architecture.target + else: + return self.spec.architecture.target.name # # Make sure we can switch methods on different dependencies # diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 0177fe1a2b..001dde5329 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -22,30 +22,25 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import functools -import glob -import inspect import os -import re -from contextlib import closing -import spack -from llnl.util.lang import match_predicate from spack import * from spack.util.environment import * class R(Package): - """ - R is 'GNU S', a freely available language and environment for statistical computing and graphics which provides a - wide variety of statistical and graphical techniques: linear and nonlinear modelling, statistical tests, time series - analysis, classification, clustering, etc. Please consult the R project homepage for further information. - """ + """R is 'GNU S', a freely available language and environment for + statistical computing and graphics which provides a wide variety of + statistical and graphical techniques: linear and nonlinear modelling, + statistical tests, time series analysis, classification, clustering, etc. + Please consult the R project homepage for further information.""" + homepage = "https://www.r-project.org" url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" - + extendable = True + version('3.3.0', '5a7506c8813432d1621c9725e86baf7a') version('3.2.3', '1ba3dac113efab69e706902810cc2970') version('3.2.2', '57cef5c2e210a5454da1979562a10e5b') version('3.2.1', 'c2aac8b40f84e08e7f8c9068de9239a3') @@ -53,7 +48,8 @@ class R(Package): version('3.1.3', '53a85b884925aa6b5811dfc361d73fc4') version('3.1.2', '3af29ec06704cbd08d4ba8d69250ae74') - variant('external-lapack', default=False, description='Links to externally installed BLAS/LAPACK') + variant('external-lapack', default=False, + description='Links to externally installed BLAS/LAPACK') # Virtual dependencies depends_on('blas', when='+external-lapack') @@ -65,6 +61,7 @@ class R(Package): depends_on('icu') depends_on('glib') depends_on('zlib') + depends_on('bzip2') depends_on('libtiff') depends_on('jpeg') depends_on('cairo') @@ -72,18 +69,21 @@ class R(Package): depends_on('freetype') depends_on('tcl') depends_on('tk') + depends_on('curl') + depends_on('pcre') + depends_on('jdk') def install(self, spec, prefix): rlibdir = join_path(prefix, 'rlib') - options = ['--prefix=%s' % prefix, - '--libdir=%s' % rlibdir, - '--enable-R-shlib', - '--enable-BLAS-shlib', - '--enable-R-framework=no'] + configure_args = ['--prefix=%s' % prefix, + '--libdir=%s' % rlibdir, + '--enable-R-shlib', + '--enable-BLAS-shlib', + '--enable-R-framework=no'] if '+external-lapack' in spec: - options.extend(['--with-blas', '--with-lapack']) + configure_args.extend(['--with-blas', '--with-lapack']) - configure(*options) + configure(*configure_args) make() make('install') @@ -106,25 +106,24 @@ class R(Package): r_libs_path = ':'.join(r_libs_path) spack_env.set('R_LIBS', r_libs_path) - # For run time environment set only the path for extension_spec and prepend it to R_LIBS + # For run time environment set only the path for extension_spec and + # prepend it to R_LIBS if extension_spec.package.extends(self.spec): - run_env.prepend_path('R_LIBS', os.path.join(extension_spec.prefix, self.r_lib_dir)) - + run_env.prepend_path('R_LIBS', os.path.join( + extension_spec.prefix, self.r_lib_dir)) def setup_dependent_package(self, module, ext_spec): - """ - Called before R modules' install() methods. - - In most cases, extensions will only need to have one line:: - - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) - """ + """Called before R modules' install() methods. In most cases, + extensions will only need to have one line: + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % + self.stage.source_path)""" # R extension builds can have a global R executable function module.R = Executable(join_path(self.spec.prefix.bin, 'R')) # Add variable for library directry module.r_lib_dir = os.path.join(ext_spec.prefix, self.r_lib_dir) - # Make the site packages directory for extensions, if it does not exist already. + # Make the site packages directory for extensions, if it does not exist + # already. if ext_spec.package.is_extension: mkdirp(module.r_lib_dir) diff --git a/var/spack/repos/builtin/packages/armadillo/package.py b/var/spack/repos/builtin/packages/armadillo/package.py new file mode 100644 index 0000000000..b3e5994e30 --- /dev/null +++ b/var/spack/repos/builtin/packages/armadillo/package.py @@ -0,0 +1,67 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Armadillo(Package): + """Armadillo is a high quality linear algebra library (matrix maths) + for the C++ language, aiming towards a good balance between speed and + ease of use.""" + + homepage = "http://arma.sourceforge.net/" + url = "http://sourceforge.net/projects/arma/files/armadillo-7.200.1.tar.xz" + + version('7.200.1', 'ed86d6df0058979e107502e1fe3e469e') + + variant('hdf5', default=False, description='Include HDF5 support') + + depends_on('arpack') + depends_on('blas') + depends_on('lapack') + depends_on('superlu@5.2:') + depends_on('hdf5', when='+hdf5') + + def install(self, spec, prefix): + cmake_args = [ + # ARPACK support + '-DARPACK_LIBRARY={0}/libarpack.a'.format( + spec['arpack'].prefix.lib), + # BLAS support + '-DBLAS_LIBRARY={0}'.format(spec['blas'].blas_shared_lib), + # LAPACK support + '-DLAPACK_LIBRARY={0}'.format(spec['lapack'].lapack_shared_lib), + # SuperLU support + '-DSuperLU_INCLUDE_DIR={0}'.format(spec['superlu'].prefix.include), + '-DSuperLU_LIBRARY={0}/libsuperlu.a'.format( + spec['superlu'].prefix.lib64), + # HDF5 support + '-DDETECT_HDF5={0}'.format('ON' if '+hdf5' in spec else 'OFF') + ] + + cmake_args.extend(std_cmake_args) + cmake('.', *cmake_args) + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/arpack/package.py b/var/spack/repos/builtin/packages/arpack/package.py index 75158776fe..91b5f06a4a 100644 --- a/var/spack/repos/builtin/packages/arpack/package.py +++ b/var/spack/repos/builtin/packages/arpack/package.py @@ -24,12 +24,12 @@ ############################################################################## from spack import * import os -import shutil + class Arpack(Package): """A collection of Fortran77 subroutines designed to solve large scale - eigenvalue problems. - """ + eigenvalue problems.""" + homepage = "http://www.caam.rice.edu/software/ARPACK/" url = "http://www.caam.rice.edu/software/ARPACK/SRC/arpack96.tar.gz" @@ -39,27 +39,35 @@ class Arpack(Package): depends_on('lapack') def patch(self): - # Filter the cray makefile to make a spack one. - shutil.move('ARMAKES/ARmake.CRAY', 'ARmake.inc') makefile = FileFilter('ARmake.inc') - # Be sure to use Spack F77 wrapper - makefile.filter('^FC.*', 'FC = f77') - makefile.filter('^FFLAGS.*', 'FFLAGS = -O2 -g') + # Section 1: Paths and Libraries + + # Change the build directory + makefile.filter('^home.*', 'home = %s' % os.getcwd()) + + # Use external BLAS/LAPACK + makefile.filter('^BLASdir.*', + 'BLASdir = %s' % self.spec['blas'].prefix) + makefile.filter('^LAPACKdir.*', + 'LAPACKdir = %s' % self.spec['lapack'].prefix) + + # Do not include the platform in the library name + makefile.filter('^PLAT.*', 'PLAT = ') + makefile.filter('^ARPACKLIB.*', 'ARPACKLIB = $(home)/libarpack.a') - # Set up some variables. - makefile.filter('^PLAT.*', 'PLAT = ') - makefile.filter('^home.*', 'home = %s' % os.getcwd()) - makefile.filter('^BLASdir.*', 'BLASdir = %s' % self.spec['blas'].prefix) - makefile.filter('^LAPACKdir.*', 'LAPACKdir = %s' % self.spec['lapack'].prefix) + # Section 2: Compilers - # build the library in our own prefix. - makefile.filter('^ARPACKLIB.*', 'ARPACKLIB = %s/libarpack.a' % os.getcwd()) + # Be sure to use the Spack compiler wrapper + makefile.filter('^FC.*', 'FC = {0}'.format(os.environ['F77'])) + makefile.filter('^FFLAGS.*', 'FFLAGS = -O2 -g -fPIC') + if not which('ranlib'): + makefile.filter('^RANLIB.*', 'RANLIB = touch') def install(self, spec, prefix): with working_dir('SRC'): make('all') - mkdirp(prefix.lib) + mkdir(prefix.lib) install('libarpack.a', prefix.lib) diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index cd6f1d04f1..815c184577 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -41,7 +41,7 @@ class Astyle(Package): # we need to edit the makefile in place to set compiler: make_file = join_path(self.stage.source_path, 'build', 'gcc', 'Makefile') - filter_file(r'^CXX\s*=.*', 'CXX=%s'.format(spack_cxx), make_file) + filter_file(r'^CXX\s*=.*', 'CXX=%s' % spack_cxx, make_file) make('-f', make_file, diff --git a/var/spack/repos/builtin/packages/bertini/package.py b/var/spack/repos/builtin/packages/bertini/package.py new file mode 100644 index 0000000000..8d7da705e4 --- /dev/null +++ b/var/spack/repos/builtin/packages/bertini/package.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Bertini(Package): + """Bertini is a general-purpose solver, written in C, that was created + for research about polynomial continuation. It solves for the numerical + solution of systems of polynomial equations using homotopy continuation.""" + + homepage = "https://bertini.nd.edu/" + url = "https://bertini.nd.edu/BertiniSource_v1.5.tar.gz" + + version('1.5', 'e3f6cc6e7f9a0cf1d73185e8671af707') + + variant('mpi', default=True, description='Compile in parallel') + + depends_on('flex') + depends_on('bison') + depends_on('gmp') + depends_on('mpfr') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 9e4cc98ae6..5f305abb02 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -30,8 +30,9 @@ class Binutils(Package): url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + # 2.26 is incompatible with py-pillow build for some reason. version('2.26', '64146a0faa3b411ba774f47d41de239f') - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66', preferred=True) version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index b1b9c58b32..67294fb9a1 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -25,9 +25,8 @@ from spack import * import spack import sys - import os -import sys + class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -43,6 +42,7 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 + version('1.61.0', '6095876341956f65f9d35939ccea1a9f') version('1.60.0', '65a840e1a0b13a558ff19eeb2c4f0cbe') version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') @@ -74,23 +74,24 @@ class Boost(Package): version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') default_install_libs = set(['atomic', - 'chrono', - 'date_time', - 'filesystem', - 'graph', - 'iostreams', - 'locale', - 'log', - 'math', - 'program_options', - 'random', - 'regex', - 'serialization', - 'signals', - 'system', - 'test', - 'thread', - 'wave']) + 'chrono', + 'date_time', + 'filesystem', + 'graph', + 'iostreams', + 'locale', + 'log', + 'math', + 'program_options', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'test', + 'thread', + 'timer', + 'wave']) # mpi/python are not installed by default because they pull in many # dependencies and/or because there is a great deal of customization @@ -108,6 +109,7 @@ class Boost(Package): variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") variant('icu_support', default=False, description="Include ICU support (for regex/locale libraries)") + variant('graph', default=False, description="Build the Boost Graph library") depends_on('icu', when='+icu_support') depends_on('python', when='+python') @@ -119,15 +121,18 @@ class Boost(Package): patch('boost_11856.patch', when='@1.60.0%gcc@4.4.7') def url_for_version(self, version): - """Handle Boost's weird URLs, which write the version two different ways.""" + """ + Handle Boost's weird URLs, + which write the version two different ways. + """ parts = [str(p) for p in Version(version)] dots = ".".join(parts) underscores = "_".join(parts) - return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( - dots, underscores) + return "http://downloads.sourceforge.net/project/boost" \ + "/boost/%s/boost_%s.tar.bz2" % (dots, underscores) def determine_toolset(self, spec): - if spec.satisfies("=darwin-x86_64"): + if spec.satisfies("platform=darwin"): return 'darwin' toolsets = {'g++': 'gcc', @@ -148,20 +153,20 @@ class Boost(Package): if '+python' in spec: options.append('--with-python=%s' % - join_path(spec['python'].prefix.bin, 'python')) + join_path(spec['python'].prefix.bin, 'python')) with open('user-config.jam', 'w') as f: compiler_wrapper = join_path(spack.build_env_path, 'c++') f.write("using {0} : : {1} ;\n".format(boostToolsetId, - compiler_wrapper)) + compiler_wrapper)) if '+mpi' in spec: f.write('using mpi : %s ;\n' % - join_path(spec['mpi'].prefix.bin, 'mpicxx')) + join_path(spec['mpi'].prefix.bin, 'mpicxx')) if '+python' in spec: f.write('using python : %s : %s ;\n' % - (spec['python'].version, - join_path(spec['python'].prefix.bin, 'python'))) + (spec['python'].version, + join_path(spec['python'].prefix.bin, 'python'))) def determine_b2_options(self, spec, options): if '+debug' in spec: @@ -177,8 +182,7 @@ class Boost(Package): '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, '-s', 'ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, - '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, - ]) + '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) linkTypes = ['static'] if '+shared' in spec: @@ -190,7 +194,8 @@ class Boost(Package): if '+singlethreaded' in spec: threadingOpts.append('single') if not threadingOpts: - raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") + raise RuntimeError("""At least one of {singlethreaded, + multithreaded} must be enabled""") options.extend([ 'toolset=%s' % self.determine_toolset(spec), @@ -201,9 +206,9 @@ class Boost(Package): def install(self, spec, prefix): # On Darwin, Boost expects the Darwin libtool. However, one of the - # dependencies may have pulled in Spack's GNU libtool, and these two are - # not compatible. We thus create a symlink to Darwin's libtool and add - # it at the beginning of PATH. + # dependencies may have pulled in Spack's GNU libtool, and these two + # are not compatible. We thus create a symlink to Darwin's libtool + # and add it at the beginning of PATH. if sys.platform == 'darwin': newdir = os.path.abspath('darwin-libtool') mkdirp(newdir) @@ -216,7 +221,8 @@ class Boost(Package): withLibs.append(lib) if not withLibs: # if no libraries are specified for compilation, then you dont have - # to configure/build anything, just copy over to the prefix directory. + # to configure/build anything, just copy over to the prefix + # directory. src = join_path(self.stage.source_path, 'boost') mkdirp(join_path(prefix, 'include')) dst = join_path(prefix, 'include', 'boost') @@ -234,6 +240,9 @@ class Boost(Package): withLibs.remove('chrono') if not spec.satisfies('@1.43.0:'): withLibs.remove('random') + if '+graph' in spec and '+mpi' in spec: + withLibs.remove('graph') + withLibs.append('graph_parallel') # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' @@ -258,6 +267,7 @@ class Boost(Package): for threadingOpt in threadingOpts: b2('install', 'threading=%s' % threadingOpt, *b2_options) - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly + # on Darwin; correct this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py new file mode 100644 index 0000000000..dee332be14 --- /dev/null +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import sys + +from spack import * + +class CBlosc(Package): + """Blosc, an extremely fast, multi-threaded, meta-compressor library""" + homepage = "http://www.blosc.org" + url = "https://github.com/Blosc/c-blosc/archive/v1.9.2.tar.gz" + + version('1.9.2', 'dd2d83069d74b36b8093f1c6b49defc5') + version('1.9.1', '7d708d3daadfacf984a87b71b1734ce2') + version('1.9.0', 'e4c1dc8e2c468e5cfa2bf05eeee5357a') + version('1.8.1', 'd73d5be01359cf271e9386c90dcf5b05') + version('1.8.0', '5b92ecb287695ba20cc33d30bf221c4f') + + depends_on("cmake") + depends_on("snappy") + depends_on("zlib") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index 4b8fe0d8af..a424c73859 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -34,7 +34,7 @@ class Caliper(Package): homepage = "https://github.com/LLNL/Caliper" url = "" - version('master', git='ssh://git@github.com:LLNL/Caliper.git') + version('master', git='https://github.com/LLNL/Caliper.git') variant('mpi', default=False, description='Enable MPI function wrappers.') diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py new file mode 100644 index 0000000000..e13dd80e19 --- /dev/null +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -0,0 +1,28 @@ +from spack import * +import os + +from spack.pkg.builtin.intel import IntelInstaller + + +class Daal(IntelInstaller): + """Intel Data Analytics Acceleration Library. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/daal" + + version('2016.2.181', 'aad2aa70e5599ebfe6f85b29d8719d46', + url="file://%s/l_daal_2016.2.181.tgz" % os.getcwd()) + version('2016.3.210', 'ad747c0dd97dace4cad03cf2266cad28', + url="file://%s/l_daal_2016.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + daal_dir = os.path.join(self.intel_prefix, "daal") + for f in os.listdir(daal_dir): + os.symlink(os.path.join(daal_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 49dc971d3a..23ec74abed 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -25,18 +25,24 @@ from spack import * import sys + class Dealii(Package): - """C++ software library providing well-documented tools to build finite element codes for a broad variety of PDEs.""" + """C++ software library providing well-documented tools to build finite + element codes for a broad variety of PDEs.""" homepage = "https://www.dealii.org" - url = "https://github.com/dealii/dealii/releases/download/v8.4.0/dealii-8.4.0.tar.gz" + url = "https://github.com/dealii/dealii/releases/download/v8.4.1/dealii-8.4.1.tar.gz" + version('8.4.1', 'efbaf16f9ad59cfccad62302f36c3c1d') version('8.4.0', 'ac5dbf676096ff61e092ce98c80c2b00') + version('8.3.0', 'fc6cdcb16309ef4bea338a4f014de6fa') + version('8.2.1', '71c728dbec14f371297cd405776ccf08') + version('8.1.0', 'aa8fadc2ce5eb674f44f997461bf668d') version('dev', git='https://github.com/dealii/dealii.git') variant('mpi', default=True, description='Compile with MPI') variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') variant('doc', default=False, description='Compile with documentation') - variant('gsl' , default=True, description='Compile with GSL') + variant('gsl', default=True, description='Compile with GSL') variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') variant('metis', default=True, description='Compile with Metis') variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') @@ -47,38 +53,40 @@ class Dealii(Package): variant('trilinos', default=True, description='Compile with Trilinos (only with MPI)') # required dependencies, light version - depends_on ("blas") - # Boost 1.58 is blacklisted, see https://github.com/dealii/dealii/issues/1591 - # require at least 1.59 - depends_on ("boost@1.59.0:", when='~mpi') - depends_on ("boost@1.59.0:+mpi", when='+mpi') - depends_on ("bzip2") - depends_on ("cmake") - depends_on ("lapack") - depends_on ("muparser") - depends_on ("suite-sparse") - depends_on ("tbb") - depends_on ("zlib") + depends_on("blas") + # Boost 1.58 is blacklisted, see + # https://github.com/dealii/dealii/issues/1591 + # Require at least 1.59 + depends_on("boost@1.59.0:+thread+system+serialization+iostreams", when='~mpi') # NOQA: ignore=E501 + depends_on("boost@1.59.0:+mpi+thread+system+serialization+iostreams", when='+mpi') # NOQA: ignore=E501 + depends_on("bzip2") + depends_on("cmake") + depends_on("lapack") + depends_on("muparser") + depends_on("suite-sparse") + depends_on("tbb") + depends_on("zlib") # optional dependencies - depends_on ("mpi", when="+mpi") - depends_on ("arpack-ng+mpi", when='+arpack+mpi') - depends_on ("doxygen", when='+doc') - depends_on ("gsl", when='@8.5.0:+gsl') - depends_on ("gsl", when='@dev+gsl') - depends_on ("hdf5+mpi~cxx", when='+hdf5+mpi') #FIXME NetCDF declares dependency with ~cxx, why? - depends_on ("metis@5:", when='+metis') - depends_on ("netcdf+mpi", when="+netcdf+mpi") - depends_on ("netcdf-cxx", when='+netcdf+mpi') - depends_on ("oce", when='+oce') - depends_on ("p4est", when='+p4est+mpi') - depends_on ("petsc+mpi", when='+petsc+mpi') - depends_on ("slepc", when='+slepc+petsc+mpi') - depends_on ("trilinos", when='+trilinos+mpi') + depends_on("mpi", when="+mpi") + depends_on("arpack-ng+mpi", when='+arpack+mpi') + depends_on("doxygen+graphviz", when='+doc') + depends_on("graphviz", when='+doc') + depends_on("gsl", when='@8.5.0:+gsl') + depends_on("gsl", when='@dev+gsl') + depends_on("hdf5+mpi", when='+hdf5+mpi') + depends_on("metis@5:", when='+metis') + depends_on("netcdf+mpi", when="+netcdf+mpi") + depends_on("netcdf-cxx", when='+netcdf+mpi') + depends_on("oce", when='+oce') + depends_on("p4est", when='+p4est+mpi') + depends_on("petsc@:3.6.4+mpi", when='+petsc+mpi') # FIXME: update after 3.7 is supported upstream. # NOQA: ignore=E501 + depends_on("slepc@:3.6.3", when='+slepc+petsc+mpi') + depends_on("trilinos", when='+trilinos+mpi') # developer dependnecies - depends_on ("numdiff", when='@dev') - depends_on ("astyle@2.04", when='@dev') + depends_on("numdiff", when='@dev') + depends_on("astyle@2.04", when='@dev') def install(self, spec, prefix): options = [] @@ -96,17 +104,16 @@ class Dealii(Package): '-DDEAL_II_WITH_THREADS:BOOL=ON', '-DBOOST_DIR=%s' % spec['boost'].prefix, '-DBZIP2_DIR=%s' % spec['bzip2'].prefix, - # CMake's FindBlas/Lapack may pickup system's blas/lapack instead of Spack's. - # Be more specific to avoid this. - # Note that both lapack and blas are provided in -DLAPACK_XYZ variables + # CMake's FindBlas/Lapack may pickup system's blas/lapack instead + # of Spack's. Be more specific to avoid this. + # Note that both lapack and blas are provided in -DLAPACK_XYZ. '-DLAPACK_FOUND=true', - '-DLAPACK_INCLUDE_DIRS=%s;%s' % - (spec['lapack'].prefix.include, - spec['blas'].prefix.include), - '-DLAPACK_LIBRARIES=%s;%s' % - (join_path(spec['lapack'].prefix.lib,'liblapack.%s' % dsuf), # FIXME don't hardcode names - join_path(spec['blas'].prefix.lib,'libblas.%s' % dsuf)), # FIXME don't hardcode names - '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, + '-DLAPACK_INCLUDE_DIRS=%s;%s' % ( + spec['lapack'].prefix.include, spec['blas'].prefix.include), + '-DLAPACK_LIBRARIES=%s;%s' % ( + spec['lapack'].lapack_shared_lib, + spec['blas'].blas_shared_lib), + '-DMUPARSER_DIR=%s' % spec['muparser'].prefix, '-DUMFPACK_DIR=%s' % spec['suite-sparse'].prefix, '-DTBB_DIR=%s' % spec['tbb'].prefix, '-DZLIB_DIR=%s' % spec['zlib'].prefix @@ -116,33 +123,34 @@ class Dealii(Package): if '+mpi' in spec: options.extend([ '-DDEAL_II_WITH_MPI:BOOL=ON', - '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # FIXME: avoid hardcoding mpi wrappers names - '-DCMAKE_CXX_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), - '-DCMAKE_Fortran_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + '-DCMAKE_C_COMPILER=%s' % spec['mpi'].mpicc, + '-DCMAKE_CXX_COMPILER=%s' % spec['mpi'].mpicxx, + '-DCMAKE_Fortran_COMPILER=%s' % spec['mpi'].mpifc, ]) else: options.extend([ '-DDEAL_II_WITH_MPI:BOOL=OFF', ]) - # Optional dependencies for which librariy names are the same as CMake variables - for library in ('gsl','hdf5','p4est','petsc','slepc','trilinos','metis'): + # Optional dependencies for which librariy names are the same as CMake + # variables: + for library in ('gsl', 'hdf5', 'p4est', 'petsc', 'slepc', 'trilinos', 'metis'): # NOQA: ignore=E501 if library in spec: options.extend([ - '-D{library}_DIR={value}'.format(library=library.upper(), value=spec[library].prefix), - '-DDEAL_II_WITH_{library}:BOOL=ON'.format(library=library.upper()) + '-D%s_DIR=%s' % (library.upper(), spec[library].prefix), + '-DDEAL_II_WITH_%s:BOOL=ON' % library.upper() ]) else: options.extend([ - '-DDEAL_II_WITH_{library}:BOOL=OFF'.format(library=library.upper()) + '-DDEAL_II_WITH_%s:BOOL=OFF' % library.upper() ]) # doxygen options.extend([ - '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % ('ON' if '+doc' in spec else 'OFF'), + '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % + ('ON' if '+doc' in spec else 'OFF'), ]) - # arpack if '+arpack' in spec: options.extend([ @@ -159,11 +167,13 @@ class Dealii(Package): if '+netcdf' in spec: options.extend([ '-DNETCDF_FOUND=true', - '-DNETCDF_LIBRARIES=%s;%s' % - (join_path(spec['netcdf-cxx'].prefix.lib,'libnetcdf_c++.%s' % dsuf), - join_path(spec['netcdf'].prefix.lib,'libnetcdf.%s' % dsuf)), - '-DNETCDF_INCLUDE_DIRS=%s;%s' % - (spec['netcdf-cxx'].prefix.include, + '-DNETCDF_LIBRARIES=%s;%s' % ( + join_path(spec['netcdf-cxx'].prefix.lib, + 'libnetcdf_c++.%s' % dsuf), + join_path(spec['netcdf'].prefix.lib, + 'libnetcdf.%s' % dsuf)), + '-DNETCDF_INCLUDE_DIRS=%s;%s' % ( + spec['netcdf-cxx'].prefix.include, spec['netcdf'].prefix.include), ]) else: @@ -200,7 +210,7 @@ class Dealii(Package): with working_dir('examples/step-3'): cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) # An example which uses Metis + PETSc # FIXME: switch step-18 to MPI @@ -213,7 +223,7 @@ class Dealii(Package): if '^petsc' in spec and '^metis' in spec: cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run @@ -222,43 +232,58 @@ class Dealii(Package): print('========== Step-40 PETSc ============') print('=====================================') # list the number of cycles to speed up - filter_file(r'(const unsigned int n_cycles = 8;)', ('const unsigned int n_cycles = 2;'), 'step-40.cc') + filter_file(r'(const unsigned int n_cycles = 8;)', + ('const unsigned int n_cycles = 2;'), 'step-40.cc') cmake('.') if '^petsc' in spec: make('release') - make('run',parallel=False) + make('run', parallel=False) print('=====================================') print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos - filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') + # The below filter_file should be different for versions + # before and after 8.4.0 + if spec.satisfies('@8.4.0:'): + filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', + ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') + else: + filter_file(r'(#define USE_PETSC_LA.*)', + ('// #define USE_PETSC_LA'), 'step-40.cc') if '^trilinos+hypre' in spec: make('release') - make('run',parallel=False) + make('run', parallel=False) - print('=====================================') - print('=== Step-40 Trilinos SuperluDist ====') - print('=====================================') - # change to direct solvers - filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', (''), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', (''), 'step-40.cc') - filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', (''), 'step-40.cc') - filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') - filter_file(r'(preconditioner\);)', (''), 'step-40.cc') - if '^trilinos+superlu-dist' in spec: - make('release') - make('run',paralle=False) + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after + if spec.satisfies('@8.4.0:'): + print('=====================================') + print('=== Step-40 Trilinos SuperluDist ====') + print('=====================================') + # change to direct solvers + filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', + (''), 'step-40.cc') + filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', # NOQA: ignore=E501 + (''), 'step-40.cc') + filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', # NOQA: ignore=E501 + (''), 'step-40.cc') + filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(preconditioner\);)', (''), 'step-40.cc') + if '^trilinos+superlu-dist' in spec: + make('release') + make('run', paralle=False) - print('=====================================') - print('====== Step-40 Trilinos MUMPS =======') - print('=====================================') - # switch to Mumps - filter_file(r'(Amesos_Superludist)', ('Amesos_Mumps'), 'step-40.cc') - if '^trilinos+mumps' in spec: - make('release') - make('run',parallel=False) + print('=====================================') + print('====== Step-40 Trilinos MUMPS =======') + print('=====================================') + # switch to Mumps + filter_file(r'(Amesos_Superludist)', + ('Amesos_Mumps'), 'step-40.cc') + if '^trilinos+mumps' in spec: + make('release') + make('run', parallel=False) print('=====================================') print('============ Step-36 ================') @@ -267,7 +292,7 @@ class Dealii(Package): if 'slepc' in spec: cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) print('=====================================') print('============ Step-54 ================') @@ -276,7 +301,7 @@ class Dealii(Package): if 'oce' in spec: cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) def setup_environment(self, spack_env, env): env.set('DEAL_II_DIR', self.prefix) diff --git a/var/spack/repos/builtin/packages/doxygen/package.py b/var/spack/repos/builtin/packages/doxygen/package.py index 994a38ca87..b2e9582b5a 100644 --- a/var/spack/repos/builtin/packages/doxygen/package.py +++ b/var/spack/repos/builtin/packages/doxygen/package.py @@ -22,30 +22,31 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -#------------------------------------------------------------------------------ -# Author: Justin Too <justin@doubleotoo.com> -# Date: September 11, 2015 -#------------------------------------------------------------------------------ - from spack import * -import sys + class Doxygen(Package): """Doxygen is the de facto standard tool for generating documentation from annotated C++ sources, but it also supports other popular programming languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, - Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. - """ + Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some + extent D..""" + homepage = "http://www.stack.nl/~dimitri/doxygen/" url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" + version('1.8.11', 'f4697a444feaed739cfa2f0644abc19b') version('1.8.10', '79767ccd986f12a0f949015efb5f058f') + # graphviz appears to be a run-time optional dependency + variant('graphviz', default=True, description='Build with dot command support from Graphviz.') # NOQA: ignore=E501 + depends_on("cmake@2.8.12:") - # flex does not build on OSX, but it's provided there anyway - depends_on("flex", sys.platform != 'darwin') - depends_on("bison", sys.platform != 'darwin') + depends_on("flex") + depends_on("bison") + # optional dependencies + depends_on("graphviz", when="+graphviz") def install(self, spec, prefix): cmake('.', *std_cmake_args) diff --git a/var/spack/repos/builtin/packages/elk/package.py b/var/spack/repos/builtin/packages/elk/package.py index 9367dfdd78..b089e585dd 100644 --- a/var/spack/repos/builtin/packages/elk/package.py +++ b/var/spack/repos/builtin/packages/elk/package.py @@ -22,8 +22,9 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from spack import * import spack +from spack import * + class Elk(Package): '''An all-electron full-potential linearised augmented-plane wave @@ -35,79 +36,90 @@ class Elk(Package): version('3.3.17', 'f57f6230d14f3b3b558e5c71f62f0592') # Elk provides these libraries, but allows you to specify your own - variant('blas', default=True, description='Build with custom BLAS library') - variant('lapack', default=True, description='Build with custom LAPACK library') - variant('fft', default=True, description='Build with custom FFT library') + variant('blas', default=True, + description='Build with custom BLAS library') + variant('lapack', default=True, + description='Build with custom LAPACK library') + variant('fft', default=True, + description='Build with custom FFT library') # Elk does not provide these libraries, but allows you to use them - variant('mpi', default=True, description='Enable MPI parallelism') - variant('openmp', default=True, description='Enable OpenMP support') - variant('libxc', default=True, description='Link to Libxc functional library') + variant('mpi', default=True, + description='Enable MPI parallelism') + variant('openmp', default=True, + description='Enable OpenMP support') + variant('libxc', default=True, + description='Link to Libxc functional library') depends_on('blas', when='+blas') depends_on('lapack', when='+lapack') depends_on('fftw', when='+fft') - depends_on('mpi', when='+mpi') + depends_on('mpi@2:', when='+mpi') depends_on('libxc', when='+libxc') # Cannot be built in parallel parallel = False - def configure(self, spec): # Dictionary of configuration options config = { - 'MAKE': 'make', - 'F90': join_path(spack.build_env_path, 'f90'), - 'F77': join_path(spack.build_env_path, 'f77'), - 'AR': 'ar', - 'LIB_FFT': 'fftlib.a', - 'SRC_MPI': 'mpi_stub.f90', - 'SRC_OMP': 'omp_stub.f90', - 'SRC_libxc': 'libxcifc_stub.f90', - 'SRC_FFT': 'zfftifc.f90' + 'MAKE': 'make', + 'AR': 'ar' } # Compiler-specific flags flags = '' - if self.compiler.name == 'intel': - flags = '-O3 -ip -unroll -no-prec-div -openmp' + if self.compiler.name == 'intel': + flags = '-O3 -ip -unroll -no-prec-div' elif self.compiler.name == 'gcc': - flags = '-O3 -ffast-math -funroll-loops -fopenmp' + flags = '-O3 -ffast-math -funroll-loops' elif self.compiler.name == 'pgi': - flags = '-O3 -mp -lpthread' + flags = '-O3 -lpthread' elif self.compiler.name == 'g95': flags = '-O3 -fno-second-underscore' elif self.compiler.name == 'nag': flags = '-O4 -kind=byte -dusty -dcfuns' elif self.compiler.name == 'xl': - flags = '-O3 -qsmp=omp' + flags = '-O3' config['F90_OPTS'] = flags config['F77_OPTS'] = flags # BLAS/LAPACK support + # Note: BLAS/LAPACK must be compiled with OpenMP support + # if the +openmp variant is chosen blas = 'blas.a' lapack = 'lapack.a' - if '+blas' in spec: - blas = join_path(spec['blas'].prefix.lib, 'libblas.so') + if '+blas' in spec: + blas = spec['blas'].blas_shared_lib if '+lapack' in spec: - lapack = join_path(spec['lapack'].prefix.lib, 'liblapack.so') - config['LIB_LPK'] = ' '.join([lapack, blas]) # lapack must come before blas + lapack = spec['lapack'].lapack_shared_lib + # lapack must come before blas + config['LIB_LPK'] = ' '.join([lapack, blas]) # FFT support if '+fft' in spec: - config['LIB_FFT'] = join_path(spec['fftw'].prefix.lib, 'libfftw3.so') + config['LIB_FFT'] = join_path(spec['fftw'].prefix.lib, + 'libfftw3.so') config['SRC_FFT'] = 'zfftifc_fftw.f90' + else: + config['LIB_FFT'] = 'fftlib.a' + config['SRC_FFT'] = 'zfftifc.f90' # MPI support if '+mpi' in spec: - config.pop('SRC_MPI') - config['F90'] = join_path(spec['mpi'].prefix.bin, 'mpif90') - config['F77'] = join_path(spec['mpi'].prefix.bin, 'mpif77') + config['F90'] = spec['mpi'].mpifc + config['F77'] = spec['mpi'].mpif77 + else: + config['F90'] = join_path(spack.build_env_path, 'f90') + config['F77'] = join_path(spack.build_env_path, 'f77') + config['SRC_MPI'] = 'mpi_stub.f90' # OpenMP support if '+openmp' in spec: - config.pop('SRC_OMP') + config['F90_OPTS'] += ' ' + self.compiler.openmp_flag + config['F77_OPTS'] += ' ' + self.compiler.openmp_flag + else: + config['SRC_OMP'] = 'omp_stub.f90' # Libxc support if '+libxc' in spec: @@ -120,13 +132,14 @@ class Elk(Package): 'libxc.f90', 'libxcifc.f90' ]) + else: + config['SRC_libxc'] = 'libxcifc_stub.f90' # Write configuration options to include file with open('make.inc', 'w') as inc: for key in config: inc.write('{0} = {1}\n'.format(key, config[key])) - def install(self, spec, prefix): # Elk only provides an interactive setup script self.configure(spec) @@ -143,4 +156,3 @@ class Elk(Package): install_tree('examples', join_path(prefix, 'examples')) install_tree('species', join_path(prefix, 'species')) - diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ef6a3ccc7b..447964f286 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -26,20 +26,28 @@ from spack import * import os + class Espresso(Package): """ - QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials - modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. + QE is an integrated suite of Open-Source computer codes for + electronic-structure calculations and materials modeling at + the nanoscale. It is based on density-functional theory, plane + waves, and pseudopotentials. """ homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' + version( + '5.4.0', + '8bb78181b39bd084ae5cb7a512c1cfe7', + url='http://www.qe-forge.org/gf/download/frsrelease/211/968/espresso-5.4.0.tar.gz' + ) version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') - variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') + variant('mpi', default=True, description='Builds with mpi support') variant('openmp', default=False, description='Enables openMP support') variant('scalapack', default=True, description='Enables scalapack support') - variant('elpa', default=True, description='Use elpa as an eigenvalue solver') + variant('elpa', default=True, description='Uses elpa as an eigenvalue solver') depends_on('blas') depends_on('lapack') @@ -47,7 +55,12 @@ class Espresso(Package): depends_on('mpi', when='+mpi') depends_on('fftw~mpi', when='~mpi') depends_on('fftw+mpi', when='+mpi') - depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + # TODO : + mpi needed to avoid false dependencies installation + depends_on('scalapack', when='+scalapack+mpi') + + # Spurious problems running in parallel the Makefile + # generated by qe configure + parallel = False def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' @@ -87,10 +100,9 @@ class Espresso(Package): configure(*options) make('all') - if spec.architecture.startswith('darwin'): + if spec.satisfies('platform=darwin'): mkdirp(prefix.bin) for filename in glob("bin/*.x"): install(filename, prefix.bin) else: make('install') - diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py new file mode 100644 index 0000000000..0845237656 --- /dev/null +++ b/var/spack/repos/builtin/packages/fenics/package.py @@ -0,0 +1,176 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Fenics(Package): + """FEniCS is organized as a collection of interoperable components + that together form the FEniCS Project. These components include + the problem-solving environment DOLFIN, the form compiler FFC, the + finite element tabulator FIAT, the just-in-time compiler Instant, + the code generation interface UFC, the form language UFL and a + range of additional components.""" + + homepage = "http://fenicsproject.org/" + url = "https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-1.6.0.tar.gz" + + base_url = "https://bitbucket.org/fenics-project/{pkg}/downloads/{pkg}-{version}.tar.gz" # NOQA: ignore E501 + + variant('hdf5', default=True, description='Compile with HDF5') + variant('parmetis', default=True, description='Compile with ParMETIS') + variant('scotch', default=True, description='Compile with Scotch') + variant('petsc', default=True, description='Compile with PETSc') + variant('slepc', default=True, description='Compile with SLEPc') + variant('trilinos', default=True, description='Compile with Trilinos') + variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') + variant('vtk', default=False, description='Compile with VTK') + variant('qt', default=False, description='Compile with QT') + variant('mpi', default=True, description='Enables the distributed memory support') + variant('openmp', default=True, description='Enables the shared memory support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + # not part of spack list for now + # variant('petsc4py', default=True, description='Uses PETSc4py') + # variant('slepc4py', default=True, description='Uses SLEPc4py') + # variant('pastix', default=True, description='Compile with Pastix') + + extends('python') + + depends_on('py-numpy') + depends_on('py-ply') + depends_on('py-six') + depends_on('py-sphinx@1.0.1:', when='+doc') + depends_on('eigen@3.2.0:') + depends_on('boost') + depends_on('mpi', when='+mpi') + depends_on('hdf5', when='+hdf5') + depends_on('parmetis@4.0.2:^metis+real64', when='+parmetis') + depends_on('scotch~metis', when='+scotch~mpi') + depends_on('scotch+mpi~metis', when='+scotch+mpi') + depends_on('petsc@3.4:', when='+petsc') + depends_on('slepc@3.4:', when='+slepc') + depends_on('trilinos', when='+trilinos') + depends_on('vtk', when='+vtk') + depends_on('suite-sparse', when='+suite-sparse') + depends_on('qt', when='+qt') + + # This are the build dependencies + depends_on('py-setuptools') + depends_on('cmake@2.8.12:') + depends_on('swig@3.0.3:') + + releases = [ + { + 'version': '1.6.0', + 'md5': '35cb4baf7ab4152a40fb7310b34d5800', + 'resources': { + 'ffc': '358faa3e9da62a1b1a717070217b793e', + 'fiat': 'f4509d05c911fd93cea8d288a78a6c6f', + 'instant': '5f2522eb032a5bebbad6597b6fe0732a', + 'ufl': 'c40c5f04eaa847377ab2323122284016', + } + }, + { + 'version': '1.5.0', + 'md5': '9b589a3534299a5e6d22c13c5eb30bb8', + 'resources': { + 'ffc': '343f6d30e7e77d329a400fd8e73e0b63', + 'fiat': 'da3fa4dd8177bb251e7f68ec9c7cf6c5', + 'instant': 'b744023ded27ee9df4a8d8c6698c0d58', + 'ufl': '130d7829cf5a4bd5b52bf6d0955116fd', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=base_url.format(pkg='dolfin', version=release['version'])) + for name, md5 in release['resources'].items(): + resource(name=name, + url=base_url.format(pkg=name, **release), + md5=md5, + destination='depends', + when='@{version}'.format(**release), + placement=name) + + def cmake_is_on(self, option): + return 'ON' if option in self.spec else 'OFF' + + def install(self, spec, prefix): + for package in ['ufl', 'ffc', 'fiat', 'instant']: + with working_dir(join_path('depends', package)): + python('setup.py', 'install', '--prefix=%s' % prefix) + + cmake_args = [ + '-DCMAKE_BUILD_TYPE:STRING={0}'.format( + 'Debug' if '+debug' in spec else 'RelWithDebInfo'), + '-DBUILD_SHARED_LIBS:BOOL={0}'.format( + self.cmake_is_on('+shared')), + '-DDOLFIN_SKIP_BUILD_TESTS:BOOL=ON', + '-DDOLFIN_ENABLE_OPENMP:BOOL={0}'.format( + self.cmake_is_on('+openmp')), + '-DDOLFIN_ENABLE_CHOLMOD:BOOL={0}'.format( + self.cmake_is_on('suite-sparse')), + '-DDOLFIN_ENABLE_HDF5:BOOL={0}'.format( + self.cmake_is_on('hdf5')), + '-DDOLFIN_ENABLE_MPI:BOOL={0}'.format( + self.cmake_is_on('mpi')), + '-DDOLFIN_ENABLE_PARMETIS:BOOL={0}'.format( + self.cmake_is_on('parmetis')), + '-DDOLFIN_ENABLE_PASTIX:BOOL={0}'.format( + self.cmake_is_on('pastix')), + '-DDOLFIN_ENABLE_PETSC:BOOL={0}'.format( + self.cmake_is_on('petsc')), + '-DDOLFIN_ENABLE_PETSC4PY:BOOL={0}'.format( + self.cmake_is_on('py-petsc4py')), + '-DDOLFIN_ENABLE_PYTHON:BOOL={0}'.format( + self.cmake_is_on('python')), + '-DDOLFIN_ENABLE_QT:BOOL={0}'.format( + self.cmake_is_on('qt')), + '-DDOLFIN_ENABLE_SCOTCH:BOOL={0}'.format( + self.cmake_is_on('scotch')), + '-DDOLFIN_ENABLE_SLEPC:BOOL={0}'.format( + self.cmake_is_on('slepc')), + '-DDOLFIN_ENABLE_SLEPC4PY:BOOL={0}'.format( + self.cmake_is_on('py-slepc4py')), + '-DDOLFIN_ENABLE_SPHINX:BOOL={0}'.format( + self.cmake_is_on('py-sphinx')), + '-DDOLFIN_ENABLE_TRILINOS:BOOL={0}'.format( + self.cmake_is_on('trilinos')), + '-DDOLFIN_ENABLE_UMFPACK:BOOL={0}'.format( + self.cmake_is_on('suite-sparse')), + '-DDOLFIN_ENABLE_VTK:BOOL={0}'.format( + self.cmake_is_on('vtk')), + '-DDOLFIN_ENABLE_ZLIB:BOOL={0}'.format( + self.cmake_is_on('zlib')), + ] + + cmake_args.extend(std_cmake_args) + + with working_dir('build', create=True): + cmake('..', *cmake_args) + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/ferret/package.py b/var/spack/repos/builtin/packages/ferret/package.py new file mode 100644 index 0000000000..15ddfcee16 --- /dev/null +++ b/var/spack/repos/builtin/packages/ferret/package.py @@ -0,0 +1,103 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * +import os + + +class Ferret(Package): + """Ferret is an interactive computer visualization and analysis environment + designed to meet the needs of oceanographers and meteorologists + analyzing large and complex gridded data sets.""" + homepage = "http://ferret.noaa.gov/Ferret/" + url = "ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.tar.gz" + + version('6.96', '51722027c864369f41bab5751dfff8cc', + url="ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.tar.gz") + + depends_on("hdf5~mpi~fortran") + depends_on("netcdf~mpi") + depends_on("netcdf-fortran") + depends_on("readline") + depends_on("zlib") + + def patch(self): + hdf5_prefix = self.spec['hdf5'].prefix + netcdff_prefix = self.spec['netcdf-fortran'].prefix + readline_prefix = self.spec['readline'].prefix + libz_prefix = self.spec['zlib'].prefix + + filter_file(r'^BUILDTYPE.+', + 'BUILDTYPE = x86_64-linux', + 'FERRET/site_specific.mk') + filter_file(r'^INSTALL_FER_DIR.+', + 'INSTALL_FER_DIR = %s' % self.spec.prefix, + 'FERRET/site_specific.mk') + filter_file(r'^HDF5_DIR.+', + 'HDF5_DIR = %s' % hdf5_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^NETCDF4_DIR.+', + 'NETCDF4_DIR = %s' % netcdff_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^READLINE_DIR.+', + 'READLINE_DIR = %s' % readline_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^LIBZ_DIR.+', + 'LIBZ_DIR = %s' % libz_prefix, + 'FERRET/site_specific.mk') + filter_file(r'^JAVA_HOME.+', + ' ', + 'FERRET/site_specific.mk') + filter_file(r'-lm', + '-lgfortran -lm', + 'FERRET/platform_specific.mk.x86_64-linux') + + def install(self, spec, prefix): + hdf5_prefix = spec['hdf5'].prefix + netcdff_prefix = spec['netcdf-fortran'].prefix + netcdf_prefix = spec['netcdf'].prefix + libz_prefix = spec['zlib'].prefix + ln = which('ln') + ln('-sf', + hdf5_prefix + '/lib', + hdf5_prefix + '/lib64') + ln('-sf', + netcdff_prefix + '/lib', + netcdff_prefix + '/lib64') + ln('-sf', + netcdf_prefix + '/lib/libnetcdf.a', + netcdff_prefix + '/lib/libnetcdf.a') + ln('-sf', + netcdf_prefix + '/lib/libnetcdf.la', + netcdff_prefix + '/lib/libnetcdf.la') + ln('-sf', + libz_prefix + '/lib', + libz_prefix + '/lib64') + os.environ['LDFLAGS'] = '-lquadmath' + with working_dir('FERRET', create=False): + os.environ['LD_X11'] = '-L/usr/lib/X11 -lX11' + os.environ['HOSTTYPE'] = 'x86_64-linux' + make(parallel=False) + make("install") diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 926651010f..b778538606 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -24,15 +24,18 @@ ############################################################################## from spack import * + class Flex(Package): """Flex is a tool for generating scanners.""" homepage = "http://flex.sourceforge.net/" - url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" + url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" version('2.6.0', '5724bcffed4ebe39e9b55a9be80859ec') version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') + depends_on("bison") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 47dbeb2a99..224105ea0f 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -1,33 +1,9 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## from spack import * from contextlib import closing from glob import glob import sys -import os + class Gcc(Package): """The GNU Compiler Collection includes front ends for C, C++, @@ -50,10 +26,12 @@ class Gcc(Package): version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - variant('binutils', default=sys.platform != 'darwin', - description="Build via binutils") - variant('gold', default=sys.platform != 'darwin', - description="Build the gold linker plugin for ld-based LTO") + variant('binutils', + default=sys.platform != 'darwin', + description="Build via binutils") + variant('gold', + default=sys.platform != 'darwin', + description="Build the gold linker plugin for ld-based LTO") depends_on("mpfr") depends_on("gmp") @@ -63,16 +41,18 @@ class Gcc(Package): depends_on("binutils~libiberty+gold", when='+binutils +gold') # TODO: integrate these libraries. - #depends_on("ppl") - #depends_on("cloog") + # depends_on("ppl") + # depends_on("cloog") if sys.platform == 'darwin': patch('darwin/gcc-4.9.patch1', when='@4.9.3') patch('darwin/gcc-4.9.patch2', when='@4.9.3') + else: + provides('golang', when='@4.7.1:') def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', - string=True) + string=True) enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) @@ -80,62 +60,59 @@ class Gcc(Package): enabled_languages.add('go') # Generic options to compile GCC - options = ["--prefix=%s" % prefix, - "--libdir=%s/lib64" % prefix, + options = ["--prefix=%s" % prefix, "--libdir=%s/lib64" % prefix, "--disable-multilib", "--enable-languages=" + ','.join(enabled_languages), - "--with-mpc=%s" % spec['mpc'].prefix, - "--with-mpfr=%s" % spec['mpfr'].prefix, - "--with-gmp=%s" % spec['gmp'].prefix, - "--enable-lto", - "--with-quad"] + "--with-mpc=%s" % spec['mpc'].prefix, "--with-mpfr=%s" % + spec['mpfr'].prefix, "--with-gmp=%s" % spec['gmp'].prefix, + "--enable-lto", "--with-quad"] # Binutils if spec.satisfies('+binutils'): static_bootstrap_flags = "-static-libstdc++ -static-libgcc" - binutils_options = ["--with-sysroot=/", - "--with-stage1-ldflags=%s %s" % - (self.rpath_args, static_bootstrap_flags), - "--with-boot-ldflags=%s %s" % - (self.rpath_args, static_bootstrap_flags), - "--with-gnu-ld", - "--with-ld=%s/bin/ld" % spec['binutils'].prefix, - "--with-gnu-as", - "--with-as=%s/bin/as" % spec['binutils'].prefix] + binutils_options = [ + "--with-sysroot=/", "--with-stage1-ldflags=%s %s" % + (self.rpath_args, static_bootstrap_flags), + "--with-boot-ldflags=%s %s" % + (self.rpath_args, static_bootstrap_flags), "--with-gnu-ld", + "--with-ld=%s/bin/ld" % spec['binutils'].prefix, + "--with-gnu-as", + "--with-as=%s/bin/as" % spec['binutils'].prefix + ] options.extend(binutils_options) # Isl if 'isl' in spec: isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) - if sys.platform == 'darwin' : - darwin_options = [ "--with-build-config=bootstrap-debug" ] + if sys.platform == 'darwin': + darwin_options = ["--with-build-config=bootstrap-debug"] options.extend(darwin_options) build_dir = join_path(self.stage.path, 'spack-build') - configure = Executable( join_path(self.stage.source_path, 'configure') ) + configure = Executable(join_path(self.stage.source_path, 'configure')) with working_dir(build_dir, create=True): # Rest of install is straightforward. configure(*options) - if sys.platform == 'darwin' : make("bootstrap") - else: make() + if sys.platform == 'darwin': + make("bootstrap") + else: + make() make("install") self.write_rpath_specs() - @property def spec_dir(self): # e.g. lib64/gcc/x86_64-unknown-linux-gnu/4.9.2 spec_dir = glob("%s/lib64/gcc/*/*" % self.prefix) return spec_dir[0] if spec_dir else None - def write_rpath_specs(self): """Generate a spec file so the linker adds a rpath to the libs the compiler used to build the executable.""" if not self.spec_dir: tty.warn("Could not install specs for %s." % - self.spec.format('$_$@')) + self.spec.format('$_$@')) return gcc = Executable(join_path(self.prefix.bin, 'gcc')) @@ -146,5 +123,5 @@ class Gcc(Package): out.write(line + "\n") if line.startswith("*link:"): out.write("-rpath %s/lib:%s/lib64 \\\n" % - (self.prefix, self.prefix)) + (self.prefix, self.prefix)) set_install_permissions(specs_file) diff --git a/var/spack/repos/builtin/packages/ghostscript/package.py b/var/spack/repos/builtin/packages/ghostscript/package.py index ba787f858f..c22b90088e 100644 --- a/var/spack/repos/builtin/packages/ghostscript/package.py +++ b/var/spack/repos/builtin/packages/ghostscript/package.py @@ -24,18 +24,18 @@ ############################################################################## from spack import * + class Ghostscript(Package): """an interpreter for the PostScript language and for PDF. """ homepage = "http://ghostscript.com/" - url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" + url = "http://downloads.ghostscript.com/public/old-gs-releases/ghostscript-9.18.tar.gz" - version('9.16', '829319325bbdb83f5c81379a8f86f38f') + version('9.18', '33a47567d7a591c00a253caddd12a88a') parallel = False def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") + configure("--prefix=%s" % prefix, "--enable-shared") make() make("install") - diff --git a/var/spack/repos/builtin/packages/go-bootstrap/package.py b/var/spack/repos/builtin/packages/go-bootstrap/package.py new file mode 100644 index 0000000000..b0e2109fd3 --- /dev/null +++ b/var/spack/repos/builtin/packages/go-bootstrap/package.py @@ -0,0 +1,51 @@ +import os +import shutil +import glob +from spack import * + +# THIS PACKAGE SHOULD NOT EXIST +# it exists to make up for the inability to: +# * use an external go compiler +# * have go depend on itself +# * have a sensible way to find gccgo without a dep on gcc + + +class GoBootstrap(Package): + """Old C-bootstrapped go to bootstrap real go""" + homepage = "https://golang.org" + url = "https://go.googlesource.com/go" + + extendable = True + + # temporary fix until tags are pulled correctly + version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') + + variant('test', + default=True, + description="Run tests as part of build, a good idea but quite" + " time consuming") + + provides('golang@:1.4.2') + + depends_on('git') + + def install(self, spec, prefix): + bash = which('bash') + with working_dir('src'): + if '+test' in spec: + bash('all.bash') + else: + bash('make.bash') + + try: + os.makedirs(prefix) + except OSError: + pass + for f in glob.glob('*'): + if os.path.isdir(f): + shutil.copytree(f, os.path.join(prefix, f)) + else: + shutil.copy2(f, os.path.join(prefix, f)) + + def setup_environment(self, spack_env, run_env): + spack_env.set('GOROOT_FINAL', self.spec.prefix) diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py new file mode 100644 index 0000000000..13b83517d1 --- /dev/null +++ b/var/spack/repos/builtin/packages/go/package.py @@ -0,0 +1,80 @@ +import os +import shutil +import glob +import llnl.util.tty as tty +from spack import * + + +class Go(Package): + """The golang compiler and build environment""" + homepage = "https://golang.org" + url = "https://go.googlesource.com/go" + + extendable = True + + version('1.5.4', git='https://go.googlesource.com/go', tag='go1.5.4') + version('1.6.2', git='https://go.googlesource.com/go', tag='go1.6.2') + + variant('test', + default=True, + description="Run tests as part of build, a good idea but quite" + " time consuming") + + provides('golang') + + # to-do, make non-c self-hosting compilers feasible without backflips + # should be a dep on external go compiler + depends_on('go-bootstrap') + depends_on('git') + + def install(self, spec, prefix): + bash = which('bash') + with working_dir('src'): + if '+test' in spec: + bash('all.bash') + else: + bash('make.bash') + + try: + os.makedirs(prefix) + except OSError: + pass + for f in glob.glob('*'): + if os.path.isdir(f): + shutil.copytree(f, os.path.join(prefix, f)) + else: + shutil.copy2(f, os.path.join(prefix, f)) + + def setup_environment(self, spack_env, run_env): + spack_env.set('GOROOT_FINAL', self.spec.prefix) + spack_env.set('GOROOT_BOOTSTRAP', self.spec['go-bootstrap'].prefix) + + def setup_dependent_package(self, module, ext_spec): + """Called before go modules' install() methods. + + In most cases, extensions will only need to set GOPATH and use go:: + + env = os.environ + env['GOPATH'] = self.source_path + ':' + env['GOPATH'] + go('get', '<package>', env=env) + shutil.copytree('bin', os.path.join(prefix, '/bin')) + """ + # Add a go command/compiler for extensions + module.go = Executable(join_path(self.spec.prefix.bin, 'go')) + + def setup_dependent_environment(self, spack_env, run_env, ext_spec): + if os.environ.get('GOROOT', False): + tty.warn('GOROOT is set, this is not recommended') + + path_components = [] + # Set GOPATH to include paths of dependencies + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + path_components.append(d.prefix) + + # This *MUST* be first, this is where new code is installed + spack_env.set('GOPATH', ':'.join(path_components)) + + # Allow packages to find this when using module or dotkit + run_env.prepend_path('GOPATH', ':'.join( + [ext_spec.prefix] + path_components)) diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py index 203e7b7f3c..2f99015ba2 100644 --- a/var/spack/repos/builtin/packages/graphviz/package.py +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -23,6 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import sys + class Graphviz(Package): """Graph Visualization Software""" @@ -35,19 +37,27 @@ class Graphviz(Package): # related to missing Perl packages. If spack begins support for Perl in the # future, this package can be updated to depend_on('perl') and the # ncecessary devel packages. - variant('perl', default=False, description='Enable if you need the optional Perl language bindings.') + variant('perl', default=False, description='Enable if you need the optional Perl language bindings.') # NOQA: ignore=E501 parallel = False depends_on("swig") depends_on("python") depends_on("ghostscript") + depends_on("pkg-config") def install(self, spec, prefix): options = ['--prefix=%s' % prefix] - if not '+perl' in spec: + if '+perl' not in spec: options.append('--disable-perl') + # On OSX fix the compiler error: + # In file included from tkStubLib.c:15: + # /usr/include/tk.h:78:11: fatal error: 'X11/Xlib.h' file not found + # include <X11/Xlib.h> + if sys.platform == 'darwin': + options.append('CFLAGS=-I/opt/X11/include') + configure(*options) make() make("install") diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py new file mode 100644 index 0000000000..50f380083c --- /dev/null +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -0,0 +1,206 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import shutil +import sys + +from spack import * + +def _install_shlib(name, src, dst): + """Install a shared library from directory src to directory dst""" + if sys.platform == "darwin": + shlib0 = name + ".0.dylib" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib0), join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + else: + shlib000 = name + ".so.0.0.0" + shlib0 = name + ".so.0" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib000), join_path(dst, shlib000)) + os.symlink(shlib000, join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + +class Hdf5Blosc(Package): + """Blosc filter for HDF5""" + homepage = "https://github.com/Blosc/hdf5-blosc" + url = "https://github.com/Blosc/hdf5-blosc/archive/master.zip" + + version('master', '02c04acbf4bec66ec8a35bf157d1c9de') + + depends_on("c-blosc") + depends_on("hdf5") + depends_on("libtool") + + parallel = False + + def install(self, spec, prefix): + # The included cmake recipe doesn"t work for Darwin + # cmake(".", *std_cmake_args) + # + # make() + # make("install") + # if sys.platform == "darwin": + # fix_darwin_install_name(prefix.lib) + + libtool = Executable(join_path(spec["libtool"].prefix.bin, "libtool")) + if "+mpi" in spec["hdf5"]: + cc = "mpicc" + else: + cc = "cc" + shlibext = "so" if sys.platform!="darwin" else "dylib" + mkdirp(prefix.include) + mkdirp(prefix.lib) + + # Build and install filter + with working_dir("src"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_filter.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_filter.la", + "blosc_filter.lo", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + _install_shlib("libblosc_filter", ".libs", prefix.lib) + + # Build and install plugin + # The plugin requires at least HDF5 1.8.11: + if spec["hdf5"].satisfies("@1.8.11:"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_plugin.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_plugin.la", + "blosc_plugin.lo", + "-L%s" % prefix.lib, "-lblosc_filter", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + _install_shlib("libblosc_plugin", ".libs", prefix.lib) + + self.check_install(spec) + + def check_install(self, spec): + "Build and run a small program to test the installed HDF5 Blosc plugin" + print "Checking HDF5-Blosc plugin..." + checkdir = "spack-check" + with working_dir(checkdir, create=True): + source = r"""\ +#include <hdf5.h> +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> + +#define FILTER_BLOSC 32001 /* Blosc filter ID registered with the HDF group */ + +int main(int argc, char **argv) { + herr_t herr; + hid_t file = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(file >= 0); + hsize_t dims[3] = {10, 10, 10}; + hid_t space = H5Screate_simple(3, dims, NULL); + assert(space >= 0); + hid_t create_proplist = H5Pcreate(H5P_DATASET_CREATE); + assert(create_proplist >= 0); + herr = H5Pset_chunk(create_proplist, 3, dims); + assert(herr >= 0); + herr = H5Pset_filter(create_proplist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, + NULL); + assert(herr >= 0); + htri_t all_filters_avail = H5Pall_filters_avail(create_proplist); + assert(all_filters_avail > 0); + hid_t dataset = H5Dcreate(file, "dataset", H5T_NATIVE_DOUBLE, space, + H5P_DEFAULT, create_proplist, H5P_DEFAULT); + assert(dataset >= 0); + double data[10][10][10]; + for (int k=0; k<10; ++k) { + for (int j=0; j<10; ++j) { + for (int i=0; i<10; ++i) { + data[k][j][i] = 1.0 / (1.0 + i + j + k); + } + } + } + herr = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, + &data[0][0][0]); + assert(herr >= 0); + herr = H5Pclose(create_proplist); + assert(herr >= 0); + herr = H5Dclose(dataset); + assert(herr >= 0); + herr = H5Sclose(space); + assert(herr >= 0); + herr = H5Fclose(file); + assert(herr >= 0); + printf("Done.\n"); + return 0; +} +""" + expected = """\ +Done. +""" + with open("check.c", "w") as f: + f.write(source) + if "+mpi" in spec["hdf5"]: + cc = which("mpicc") + else: + cc = which("cc") + # TODO: Automate these path and library settings + cc("-c", "-I%s" % spec["hdf5"].prefix.include, "check.c") + cc("-o", "check", "check.o", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + try: + check = Executable("./check") + output = check(return_output=True) + except: + output = "" + success = output == expected + if not success: + print "Produced output does not match expected output." + print "Expected output:" + print "-"*80 + print expected + print "-"*80 + print "Produced output:" + print "-"*80 + print output + print "-"*80 + print "Environment:" + env = which("env") + env() + raise RuntimeError("HDF5 Blosc plugin check failed") + shutil.rmtree(checkdir) + + def setup_environment(self, spack_env, run_env): + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 21137ef356..e46f432be5 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -38,6 +38,7 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 + version('1.10.0-patch1', '9180ff0ef8dc2ef3f61bd37a7404f295') version('1.10.0', 'bdc935337ee8282579cd6bc4270ad199') version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618', preferred=True) version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py new file mode 100644 index 0000000000..efd5c8bb1d --- /dev/null +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -0,0 +1,119 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os +import platform + + +class Hpl(Package): + """HPL is a software package that solves a (random) dense linear system + in double precision (64 bits) arithmetic on distributed-memory computers. + It can thus be regarded as a portable as well as freely available + implementation of the High Performance Computing Linpack Benchmark.""" + + homepage = "http://www.netlib.org/benchmark/hpl/" + url = "http://www.netlib.org/benchmark/hpl/hpl-2.2.tar.gz" + + version('2.2', '0eb19e787c3dc8f4058db22c9e0c5320') + + variant('openmp', default=False, description='Enable OpenMP support') + + depends_on('mpi@1.1:') + depends_on('blas') + + parallel = False + + def configure(self, spec, arch): + # List of configuration options + # Order is important + config = [] + + # OpenMP support + if '+openmp' in spec: + config.append( + 'OMP_DEFS = {0}'.format(self.compiler.openmp_flag) + ) + + config.extend([ + # Shell + 'SHELL = /bin/sh', + 'CD = cd', + 'CP = cp', + 'LN_S = ln -fs', + 'MKDIR = mkdir -p', + 'RM = /bin/rm -f', + 'TOUCH = touch', + # Platform identifier + 'ARCH = {0}'.format(arch), + # HPL Directory Structure / HPL library + 'TOPdir = {0}'.format(os.getcwd()), + 'INCdir = $(TOPdir)/include', + 'BINdir = $(TOPdir)/bin/$(ARCH)', + 'LIBdir = $(TOPdir)/lib/$(ARCH)', + 'HPLlib = $(LIBdir)/libhpl.a', + # Message Passing library (MPI) + 'MPinc = -I{0}'.format(spec['mpi'].prefix.include), + 'MPlib = -L{0}'.format(spec['mpi'].prefix.lib), + # Linear Algebra library (BLAS or VSIPL) + 'LAinc = {0}'.format(spec['blas'].prefix.include), + 'LAlib = {0}'.format(spec['blas'].blas_shared_lib), + # F77 / C interface + 'F2CDEFS = -DAdd_ -DF77_INTEGER=int -DStringSunStyle', + # HPL includes / libraries / specifics + 'HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) ' + + '-I$(LAinc) -I$(MPinc)', + 'HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib)', + 'HPL_OPTS = -DHPL_DETAILED_TIMING -DHPL_PROGRESS_REPORT', + 'HPL_DEFS = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)', + # Compilers / linkers - Optimization flags + 'CC = {0}'.format(spec['mpi'].mpicc), + 'CCNOOPT = $(HPL_DEFS)', + 'CCFLAGS = $(HPL_DEFS) -O3', + 'LINKER = $(CC)', + 'LINKFLAGS = $(CCFLAGS) $(OMP_DEFS)', + 'ARCHIVER = ar', + 'ARFLAGS = r', + 'RANLIB = echo' + ]) + + # Write configuration options to include file + with open('Make.{0}'.format(arch), 'w') as makefile: + for var in config: + makefile.write('{0}\n'.format(var)) + + def install(self, spec, prefix): + # Arch used for file naming purposes only + arch = '{0}-{1}'.format(platform.system(), platform.processor()) + + # Generate Makefile include + self.configure(spec, arch) + + make('arch={0}'.format(arch)) + + # Manual installation + install_tree(join_path('bin', arch), prefix.bin) + install_tree(join_path('lib', arch), prefix.lib) + install_tree(join_path('include', arch), prefix.include) + install_tree('man', prefix.man) diff --git a/var/spack/repos/builtin/packages/hub/package.py b/var/spack/repos/builtin/packages/hub/package.py new file mode 100644 index 0000000000..ed8b742e42 --- /dev/null +++ b/var/spack/repos/builtin/packages/hub/package.py @@ -0,0 +1,24 @@ +from spack import * +import os + + +class Hub(Package): + """The github git wrapper""" + homepage = "https://github.com/github/hub" + url = "https://github.com/github/hub/archive/v2.2.3.tar.gz" + + version('head', git='https://github.com/github/hub') + version('2.2.3', '6675992ddd16d186eac7ba4484d57f5b') + version('2.2.2', '7edc8f5b5d3c7c392ee191dd999596fc') + version('2.2.1', '889a31ee9d10ae9cb333480d8dbe881f') + version('2.2.0', 'eddce830a079b8480f104aa7496f46fe') + version('1.12.4', '4f2ebb14834c9981b04e40b0d1754717') + + extends("go") + + def install(self, spec, prefix): + env = os.environ + env['GOPATH'] = self.stage.source_path + ':' + env['GOPATH'] + bash = which('bash') + bash(os.path.join('script', 'build'), '-o', os.path.join(prefix, 'bin', + 'hub')) diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py new file mode 100644 index 0000000000..493ca16417 --- /dev/null +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -0,0 +1,144 @@ +from spack import * +import os +import re + +from spack.pkg.builtin.intel import IntelInstaller, filter_pick, \ + get_all_components + + +class IntelParallelStudio(IntelInstaller): + """Intel Parallel Studio. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + + # TODO: can also try the online installer (will download files on demand) + version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" # NOQA: ignore=E501 + % os.getcwd()) + version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) # NOQA: ignore=E501 + version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) # NOQA: ignore=E501 + version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" # NOQA: ignore=E501 + % os.getcwd()) + version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 + version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 + + variant('rpath', default=True, description="Add rpath to .cfg files") + variant('newdtags', default=False, + description="Allow use of --enable-new-dtags in MPI wrappers") + variant('all', default=False, + description="Install all files with the requested edition") + variant('mpi', default=True, + description="Install the Intel MPI library and ITAC tool") + variant('mkl', default=True, description="Install the Intel MKL library") + variant('daal', + default=True, description="Install the Intel DAAL libraries") + variant('ipp', default=True, description="Install the Intel IPP libraries") + variant('tools', default=True, description="""Install the Intel Advisor,\ +VTune Amplifier, and Inspector tools""") + + provides('mpi', when='@cluster:+mpi') + provides('mkl', when='+mkl') + provides('daal', when='+daal') + provides('ipp', when='+ipp') + + def install(self, spec, prefix): + + base_components = "ALL" # when in doubt, install everything + mpi_components = "" + mkl_components = "" + daal_components = "" + ipp_components = "" + + if spec.satisfies('+all'): + base_components = "ALL" + else: + all_components = get_all_components() + regex = '(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + base_components = \ + filter_pick(all_components, re.compile(regex).search) + regex = '(icsxe|imb|mpi|itac|intel-tc|clck)' + mpi_components = \ + filter_pick(all_components, re.compile(regex).search) + mkl_components = \ + filter_pick(all_components, re.compile('(mkl)').search) + daal_components = \ + filter_pick(all_components, re.compile('(daal)').search) + ipp_components = \ + filter_pick(all_components, re.compile('(ipp)').search) + regex = '(gdb|vtune|inspector|advisor)' + tool_components = \ + filter_pick(all_components, re.compile(regex).search) + + components = base_components + if not spec.satisfies('+all'): + if spec.satisfies('+mpi') and 'cluster' in str(spec.version): + components += mpi_components + if spec.satisfies('+mkl'): + components += mkl_components + if spec.satisfies('+daal'): + components += daal_components + if spec.satisfies('+ipp'): + components += ipp_components + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or + spec.satisfies('@professional')): + components += tool_components + + self.intel_components = ';'.join(components) + IntelInstaller.install(self, spec, prefix) + + absbindir = os.path.dirname(os.path.realpath(os.path.join( + self.prefix.bin, "icc"))) + abslibdir = os.path.dirname(os.path.realpath(os.path.join + (self.prefix.lib, "intel64", "libimf.a"))) + + os.symlink(self.global_license_file, os.path.join(absbindir, + "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or + spec.satisfies('@professional')): + os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "inspector_xe/licenses", "license.lic")) + os.mkdir(os.path.join(self.prefix, "advisor_xe/licenses")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "advisor_xe/licenses", "license.lic")) + os.mkdir(os.path.join(self.prefix, "vtune_amplifier_xe/licenses")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) + + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and \ + spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join( + self.prefix, "itac_latest", "license.lic")) + if spec.satisfies('~newdtags'): + wrappers = ["mpif77", "mpif77", "mpif90", "mpif90", + "mpigcc", "mpigcc", "mpigxx", "mpigxx", + "mpiicc", "mpiicc", "mpiicpc", "mpiicpc", + "mpiifort", "mpiifort"] + wrapper_paths = [] + for root, dirs, files in os.walk(spec.prefix): + for name in files: + if name in wrappers: + wrapper_paths.append(os.path.join(spec.prefix, + root, name)) + for wrapper in wrapper_paths: + filter_file(r'-Xlinker --enable-new-dtags', r' ', + wrapper) + + if spec.satisfies('+rpath'): + for compiler_command in ["icc", "icpc", "ifort"]: + cfgfilename = os.path.join(absbindir, "%s.cfg" % + compiler_command) + with open(cfgfilename, "w") as f: + f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) + + os.symlink(os.path.join(self.prefix.man, "common", "man1"), + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py new file mode 100644 index 0000000000..ec3192380a --- /dev/null +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -0,0 +1,125 @@ +from spack import * +import os +import re + + +def filter_pick(input_list, regex_filter): + """Returns the items in input_list that are found in the regex_filter""" + return [l for l in input_list for m in (regex_filter(l),) if m] + + +def unfilter_pick(input_list, regex_filter): + """Returns the items in input_list that are not found in the + regex_filter""" + return [l for l in input_list for m in (regex_filter(l),) if not m] + + +def get_all_components(): + """Returns a list of all the components associated with the downloaded + Intel package""" + all_components = [] + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + for line in lines: + if line.find('<Abbr>') != -1: + component = line[line.find('<Abbr>') + 6:line.find('</Abbr>')] + all_components.append(component) + return all_components + + +class IntelInstaller(Package): + """Base package containing common methods for installing Intel software""" + + homepage = "https://software.intel.com/en-us" + intel_components = "ALL" + license_required = True + license_comment = '#' + license_files = ['Licenses/license.lic'] + license_vars = ['INTEL_LICENSE_FILE'] + license_url = \ + 'https://software.intel.com/en-us/articles/intel-license-manager-faq' + + @property + def global_license_file(self): + """Returns the path where a global license file should be stored.""" + if not self.license_files: + return + return join_path(self.global_license_dir, "intel", + os.path.basename(self.license_files[0])) + + def install(self, spec, prefix): + + # Remove the installation DB, otherwise it will try to install into + # location of other Intel builds + if os.path.exists(os.path.join(os.environ["HOME"], "intel", + "intel_sdp_products.db")): + os.remove(os.path.join(os.environ["HOME"], "intel", + "intel_sdp_products.db")) + + if not hasattr(self, "intel_prefix"): + self.intel_prefix = self.prefix + + silent_config_filename = 'silent.cfg' + with open(silent_config_filename, 'w') as f: + f.write(""" +ACCEPT_EULA=accept +PSET_MODE=install +CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes +PSET_INSTALL_DIR=%s +ACTIVATION_LICENSE_FILE=%s +ACTIVATION_TYPE=license_file +PHONEHOME_SEND_USAGE_DATA=no +CONTINUE_WITH_OPTIONAL_ERROR=yes +COMPONENTS=%s +""" % (self.intel_prefix, self.global_license_file, self.intel_components)) + + install_script = which("install.sh") + install_script('--silent', silent_config_filename) + + +class Intel(IntelInstaller): + """Intel Compilers. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + + # TODO: can also try the online installer (will download files on demand) + version('16.0.2', '1133fb831312eb519f7da897fec223fa', + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" # NOQA: ignore=E501 + % os.getcwd()) + version('16.0.3', '3208eeabee951fc27579177b593cefe9', + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" # NOQA: ignore=E501 + % os.getcwd()) + + variant('rpath', default=True, description="Add rpath to .cfg files") + + def install(self, spec, prefix): + components = [] + all_components = get_all_components() + regex = '(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + components = filter_pick(all_components, re.compile(regex).search) + + self.intel_components = ';'.join(components) + IntelInstaller.install(self, spec, prefix) + + absbindir = os.path.split(os.path.realpath(os.path.join( + self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join( + self.prefix.lib, "intel64", "libimf.a")))[0] + + # symlink or copy? + os.symlink(self.global_license_file, os.path.join(absbindir, + "license.lic")) + + if spec.satisfies('+rpath'): + for compiler_command in ["icc", "icpc", "ifort"]: + cfgfilename = os.path.join(absbindir, "%s.cfg" % + compiler_command) + with open(cfgfilename, "w") as f: + f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) + + os.symlink(os.path.join(self.prefix.man, "common", "man1"), + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/ipp/package.py b/var/spack/repos/builtin/packages/ipp/package.py new file mode 100644 index 0000000000..2bd931d5bd --- /dev/null +++ b/var/spack/repos/builtin/packages/ipp/package.py @@ -0,0 +1,26 @@ +from spack import * +import os + +from spack.pkg.builtin.intel import IntelInstaller + + +class Ipp(IntelInstaller): + """Intel Integrated Performance Primitives. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-ipp" + + version('9.0.3.210', '0e1520dd3de7f811a6ef6ebc7aa429a3', + url="file://%s/l_ipp_9.0.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + ipp_dir = os.path.join(self.intel_prefix, "ipp") + for f in os.listdir(ipp_dir): + os.symlink(os.path.join(ipp_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/launchmon/package.py b/var/spack/repos/builtin/packages/launchmon/package.py index f38bc38202..f256810005 100644 --- a/var/spack/repos/builtin/packages/launchmon/package.py +++ b/var/spack/repos/builtin/packages/launchmon/package.py @@ -24,29 +24,19 @@ ############################################################################## from spack import * + class Launchmon(Package): """Software infrastructure that enables HPC run-time tools to co-locate tool daemons with a parallel job.""" - homepage = "http://sourceforge.net/projects/launchmon" - url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" + homepage = "https://github.com/LLNL/LaunchMON" + url = "https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz" # NOQA: ignore=E501 - version('1.0.1', '2f12465803409fd07f91174a4389eb2b') - version('1.0.1-2', git='https://github.com/llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') + version('1.0.2', '8d6ba77a0ec2eff2fde2c5cc8fa7ff7a') depends_on('autoconf') depends_on('automake') depends_on('libtool') - - def patch(self): - # This patch makes libgcrypt compile correctly with newer gcc versions. - mf = FileFilter('tools/libgcrypt/tests/Makefile.in') - mf.filter(r'(basic_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - mf.filter(r'(tsexp_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - mf.filter(r'(keygen_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - mf.filter(r'(benchmark_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - - def install(self, spec, prefix): configure( "--prefix=" + prefix, diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py index 3f5a72116e..594271f655 100644 --- a/var/spack/repos/builtin/packages/libdwarf/package.py +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -23,11 +23,11 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os # Only build certain parts of dwarf because the other ones break. dwarf_dirs = ['libdwarf', 'dwarfdump2'] + class Libdwarf(Package): """The DWARF Debugging Information Format is of interest to programmers working on compilers and debuggers (and any one @@ -41,18 +41,17 @@ class Libdwarf(Package): MIPS/IRIX C compiler.""" homepage = "http://www.prevanders.net/dwarf.html" - url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" + url = "http://www.prevanders.net/libdwarf-20160507.tar.gz" list_url = homepage + version('20160507', 'ae32d6f9ece5daf05e2d4b14822ea811') version('20130729', '4cc5e48693f7b93b7aa0261e63c0e21d') version('20130207', '64b42692e947d5180e162e46c689dfbf') version('20130126', 'ded74a5e90edb5a12aac3c29d260c5db') - depends_on("libelf") parallel = False - def install(self, spec, prefix): # dwarf build does not set arguments for ar properly make.add_default_arg('ARFLAGS=rcs') @@ -69,7 +68,11 @@ class Libdwarf(Package): install('libdwarf.h', prefix.include) install('dwarf.h', prefix.include) - with working_dir('dwarfdump2'): + if spec.satisfies('@20130126:20130729'): + dwarfdump_dir = 'dwarfdump2' + else: + dwarfdump_dir = 'dwarfdump' + with working_dir(dwarfdump_dir): configure("--prefix=" + prefix) # This makefile has strings of copy commands that diff --git a/var/spack/repos/builtin/packages/libpciaccess/package.py b/var/spack/repos/builtin/packages/libpciaccess/package.py index 5d1e93eab7..91cef95cec 100644 --- a/var/spack/repos/builtin/packages/libpciaccess/package.py +++ b/var/spack/repos/builtin/packages/libpciaccess/package.py @@ -37,7 +37,7 @@ class Libpciaccess(Package): def install(self, spec, prefix): # libpciaccess does not support OS X - if spec.satisfies('=darwin-x86_64'): + if spec.satisfies('platform=darwin'): # create a dummy directory mkdir(prefix.lib) return diff --git a/var/spack/repos/builtin/packages/libpthread-stubs/package.py b/var/spack/repos/builtin/packages/libpthread-stubs/package.py new file mode 100644 index 0000000000..4bcca43c24 --- /dev/null +++ b/var/spack/repos/builtin/packages/libpthread-stubs/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class LibpthreadStubs(Package): + """The libpthread-stubs package provides weak aliases for pthread + functions not provided in libc or otherwise available by + default. """ + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/libpthread-stubs-0.1.tar.bz2" + + version('0.3', 'e8fa31b42e13f87e8f5a7a2b731db7ee') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxau/package.py b/var/spack/repos/builtin/packages/libxau/package.py new file mode 100644 index 0000000000..55816ecdbd --- /dev/null +++ b/var/spack/repos/builtin/packages/libxau/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libxau(Package): + """The libXau package contains a library implementing the X11 + Authorization Protocol. This is useful for restricting client + access to the display.""" + homepage = "http://xcb.freedesktop.org/" + url = "http://ftp.x.org/pub/individual/lib/libXau-1.0.8.tar.bz2" + + version('1.0.8', '685f8abbffa6d145c0f930f00703b21b') + + depends_on('xproto') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py index 0f39bb0f1d..746d4567e2 100644 --- a/var/spack/repos/builtin/packages/libxcb/package.py +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -35,18 +35,19 @@ class Libxcb(Package): version('1.11', '1698dd837d7e6e94d029dbe8b3a82deb') version('1.11.1', '118623c15a96b08622603a71d8789bf3') + depends_on("python") depends_on("xcb-proto") depends_on("pkg-config") - - # depends_on('pthread') # Ubuntu: apt-get install libpthread-stubs0-dev - # depends_on('xau') # Ubuntu: apt-get install libxau-dev + depends_on("libpthread-stubs") + depends_on('libxau') def patch(self): filter_file('typedef struct xcb_auth_info_t {', 'typedef struct {', 'src/xcb.h') def install(self, spec, prefix): + env['PKG_CONFIG_PATH'] = env['PKG_CONFIG_PATH'] + ':/usr/lib64/pkgconfig' configure("--prefix=%s" % prefix) make() diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index c090c131c6..7a6ea7401c 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, shutil +import os, glob class Llvm(Package): @@ -46,7 +46,9 @@ class Llvm(Package): variant('libcxx', default=True, description="Build the LLVM C++ standard library") variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers") variant('gold', default=True, description="Add support for LTO with the gold linker plugin") - + variant('shared_libs', default=False, description="Build all components as shared libraries, faster, less memory to build, less stable") + variant('link_dylib', default=False, description="Build and link the libLLVM shared library rather than static") + variant('all_targets', default=True, description="Build all supported targets, default targets <current arch>,NVPTX,AMDGPU,CppBackend") # Build dependency depends_on('cmake @2.8.12.2:') @@ -257,6 +259,28 @@ class Llvm(Package): if '+compiler-rt' not in spec: cmake_args.append('-DLLVM_EXTERNAL_COMPILER_RT_BUILD:Bool=OFF') + if '+shared_libs' in spec: + cmake_args.append('-DBUILD_SHARED_LIBS:Bool=ON') + + if '+link_dylib' in spec: + cmake_args.append('-DLLVM_LINK_LLVM_DYLIB:Bool=ON') + + if '+all_targets' not in spec: # all is default on cmake + targets = ['CppBackend', 'NVPTX', 'AMDGPU'] + if 'x86' in spec.architecture.target.lower(): + targets.append('X86') + elif 'arm' in spec.architecture.target.lower(): + targets.append('ARM') + elif 'aarch64' in spec.architecture.target.lower(): + targets.append('AArch64') + elif 'sparc' in spec.architecture.target.lower(): + targets.append('sparc') + elif ('ppc' in spec.architecture.target.lower() or + 'power' in spec.architecture.target.lower()): + targets.append('PowerPC') + + cmake_args.append('-DLLVM_TARGETS_TO_BUILD:Bool=' + ';'.join(targets)) + if '+clang' not in spec: if '+clang_extra' in spec: raise SpackException('The clang_extra variant requires the clang variant to be selected') @@ -267,7 +291,5 @@ class Llvm(Package): cmake(*cmake_args) make() make("install") - query_path = os.path.join('bin', 'clang-query') - # Manually install clang-query, because llvm doesn't... - if os.path.exists(query_path): - shutil.copy(query_path, os.path.join(prefix, 'bin')) + cp = which('cp') + cp('-a', 'bin/', prefix) diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 0a8b9b4577..7d75866d52 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os + class Lmod(Package): """ @@ -34,17 +34,25 @@ class Lmod(Package): variable. Modulefiles for Library packages provide environment variables that specify where the library and header files can be found. """ - homepage = "https://www.tacc.utexas.edu/research-development/tacc-projects/lmod" - url = "http://sourceforge.net/projects/lmod/files/Lmod-6.0.1.tar.bz2/download" + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' # NOQA: ignore=E501 + url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' + version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') + version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1') version('6.0.1', '91abf52fe5033bd419ffe2842ebe7af9') - depends_on("lua@5.2:") + depends_on('lua@5.2:') + depends_on('lua-luaposix') + depends_on('lua-luafilesystem') + + parallel = False + + def setup_environment(self, spack_env, run_env): + stage_lua_path = join_path( + self.stage.path, 'Lmod-{version}', 'src', '?.lua') + spack_env.append_path('LUA_PATH', stage_lua_path.format( + version=self.version), separator=';') def install(self, spec, prefix): - # Add our lua to PATH - os.environ['PATH'] = spec['lua'].prefix.bin + ';' + os.environ['PATH'] - configure('--prefix=%s' % prefix) - make() - make("install") + make('install') diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py new file mode 100644 index 0000000000..a61b9dd675 --- /dev/null +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class LuaLuafilesystem(Package): + """ + LuaFileSystem is a Lua library developed to complement the set of + functions related to file systems offered by the standard Lua distribution. + + LuaFileSystem offers a portable way to access the underlying directory + structure and file attributes. + + LuaFileSystem is free software and uses the same license as Lua 5.1 + """ + homepage = 'http://keplerproject.github.io/luafilesystem' + url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' + + version('1_6_3', 'd0552c7e5a082f5bb2865af63fb9dc95') + + extends('lua') + + def install(self, spec, prefix): + rockspec_fmt = join_path(self.stage.path, + 'luafilesystem-v_{version.underscored}', + 'rockspecs', + 'luafilesystem-{version.dotted}-1.rockspec') + luarocks('--tree=' + prefix, 'install', + rockspec_fmt.format(version=self.spec.version)) diff --git a/var/spack/repos/builtin/packages/lua-luaposix/package.py b/var/spack/repos/builtin/packages/lua-luaposix/package.py new file mode 100644 index 0000000000..9e96548f08 --- /dev/null +++ b/var/spack/repos/builtin/packages/lua-luaposix/package.py @@ -0,0 +1,16 @@ +from spack import * +import glob + + +class LuaLuaposix(Package): + """Lua posix bindings, including ncurses""" + homepage = "https://github.com/luaposix/luaposix/" + url = "https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz" + + version('33.4.0', 'b36ff049095f28752caeb0b46144516c') + + extends("lua") + + def install(self, spec, prefix): + rockspec = glob.glob('luaposix-*.rockspec') + luarocks('--tree=' + prefix, 'install', rockspec[0]) diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 9a73a22645..761932361b 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -25,10 +25,11 @@ from spack import * import os + class Lua(Package): """ The Lua programming language interpreter and library """ homepage = "http://www.lua.org" - url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" + url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" version('5.3.2', '33278c2ab5ee3c1a875be8d55c1ca2a1') version('5.3.1', '797adacada8d85761c079390ff1d9961') @@ -42,17 +43,118 @@ class Lua(Package): version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') + extendable = True + depends_on('ncurses') depends_on('readline') + resource( + name="luarocks", + url="https://keplerproject.github.io/luarocks/releases/" + "luarocks-2.3.0.tar.gz", + md5="a38126684cf42b7d0e7a3c7cf485defb", + destination="luarocks", + placement='luarocks') + def install(self, spec, prefix): - if spec.satisfies("=darwin-i686") or spec.satisfies("=darwin-x86_64"): + if spec.satisfies("platform=darwin"): target = 'macosx' else: target = 'linux' make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s -lncurses' % spec['ncurses'].prefix.lib, + 'MYLDFLAGS=-L%s -L%s ' % ( + spec['readline'].prefix.lib, + spec['ncurses'].prefix.lib), + 'MYLIBS=-lncurses', target) make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s -lncurses' % spec['ncurses'].prefix.lib, + 'MYLDFLAGS=-L%s -L%s ' % ( + spec['readline'].prefix.lib, + spec['ncurses'].prefix.lib), + 'MYLIBS=-lncurses', 'install') + + with working_dir(os.path.join('luarocks', 'luarocks')): + configure('--prefix=' + prefix, '--with-lua=' + prefix) + make('build') + make('install') + + def append_paths(self, paths, cpaths, path): + paths.append(os.path.join(path, '?.lua')) + paths.append(os.path.join(path, '?', 'init.lua')) + cpaths.append(os.path.join(path, '?.so')) + + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + lua_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + lua_paths.append(os.path.join(d.prefix, self.lua_lib_dir)) + lua_paths.append(os.path.join(d.prefix, self.lua_share_dir)) + + lua_patterns = [] + lua_cpatterns = [] + for p in lua_paths: + if os.path.isdir(p): + self.append_paths(lua_patterns, lua_cpatterns, p) + + # Always add this package's paths + for p in (os.path.join(self.spec.prefix, self.lua_lib_dir), + os.path.join(self.spec.prefix, self.lua_share_dir)): + self.append_paths(lua_patterns, lua_cpatterns, p) + + spack_env.set('LUA_PATH', ';'.join(lua_patterns), separator=';') + spack_env.set('LUA_CPATH', ';'.join(lua_cpatterns), separator=';') + + # Add LUA to PATH for dependent packages + spack_env.prepend_path('PATH', self.prefix.bin) + + # For run time environment set only the path for extension_spec and + # prepend it to LUAPATH + if extension_spec.package.extends(self.spec): + run_env.prepend_path('LUA_PATH', ';'.join(lua_patterns), + separator=';') + run_env.prepend_path('LUA_CPATH', ';'.join(lua_cpatterns), + separator=';') + + def setup_environment(self, spack_env, run_env): + run_env.prepend_path( + 'LUA_PATH', + os.path.join(self.spec.prefix, self.lua_share_dir, '?.lua'), + separator=';') + run_env.prepend_path( + 'LUA_PATH', os.path.join(self.spec.prefix, self.lua_share_dir, '?', + 'init.lua'), + separator=';') + run_env.prepend_path( + 'LUA_PATH', + os.path.join(self.spec.prefix, self.lua_lib_dir, '?.lua'), + separator=';') + run_env.prepend_path( + 'LUA_PATH', + os.path.join(self.spec.prefix, self.lua_lib_dir, '?', 'init.lua'), + separator=';') + run_env.prepend_path( + 'LUA_CPATH', + os.path.join(self.spec.prefix, self.lua_lib_dir, '?.so'), + separator=';') + + @property + def lua_lib_dir(self): + return os.path.join('lib', 'lua', '%d.%d' % self.version[:2]) + + @property + def lua_share_dir(self): + return os.path.join('share', 'lua', '%d.%d' % self.version[:2]) + + def setup_dependent_package(self, module, ext_spec): + """ + Called before lua modules's install() methods. + + In most cases, extensions will only need to have two lines:: + + luarocks('--tree=' + prefix, 'install', rock_spec_path) + """ + # Lua extension builds can have lua and luarocks executable functions + module.lua = Executable(join_path(self.spec.prefix.bin, 'lua')) + module.luarocks = Executable( + join_path(self.spec.prefix.bin, 'luarocks')) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 061179b78e..c4f2afaff2 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -24,55 +24,61 @@ ############################################################################## from spack import * -import glob, sys, os +import glob +import sys +import os + class Metis(Package): - """ - METIS is a set of serial programs for partitioning graphs, partitioning finite element meshes, and producing fill - reducing orderings for sparse matrices. The algorithms implemented in METIS are based on the multilevel - recursive-bisection, multilevel k-way, and multi-constraint partitioning schemes. - """ + """METIS is a set of serial programs for partitioning graphs, partitioning + finite element meshes, and producing fill reducing orderings for sparse + matrices. The algorithms implemented in METIS are based on the + multilevel recursive-bisection, multilevel k-way, and multi-constraint + partitioning schemes.""" - homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/metis/overview' - url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" + homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" + base_url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis" - version('5.1.0', '5465e67079419a69e0116de24fce58fe', - url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz') - version('4.0.3', '5efa35de80703c1b2c4d0de080fafbcf4e0d363a21149a1ad2f96e0144841a55', - url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz') + version('5.1.0', '5465e67079419a69e0116de24fce58fe') + version('5.0.2', 'acb521a4e8c2e6dd559a7f9abd0468c5') + version('4.0.3', 'd3848b454532ef18dc83e4fb160d1e10') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') variant('idx64', default=False, description='Use int64_t as default index type') - variant('double', default=False, description='Use double precision floating point types') + variant('real64', default=False, description='Use double precision floating point types') - depends_on('cmake @2.8:', when='@5:') # build-time dependency - depends_on('gdb', when='+gdb') + depends_on('cmake@2.8:', when='@5:') # build-time dependency patch('install_gklib_defs_rename.patch', when='@5:') + def url_for_version(self, version): + verdir = 'OLD/' if version < Version('4.0.3') else '' + return '%s/%smetis-%s.tar.gz' % (Metis.base_url, verdir, version) - @when('@4:4.0.3') + @when('@:4') def install(self, spec, prefix): - - if '+gdb' in spec: - raise InstallError('gdb support not implemented in METIS 4!') - if '+idx64' in spec: - raise InstallError('idx64 option not implemented in METIS 4!') - if '+double' in spec: - raise InstallError('double option not implemented for METIS 4!') + # Process library spec and options + unsupp_vars = [v for v in ('+gdb', '+idx64', '+real64') if v in spec] + if unsupp_vars: + msg = 'Given variants %s are unsupported by METIS 4!' % unsupp_vars + raise InstallError(msg) options = ['COPTIONS=-fPIC'] if '+debug' in spec: options.append('OPTFLAGS=-g -O0') make(*options) + # Compile and install library files + ccompile = Executable(self.compiler.cc) + mkdir(prefix.bin) - for x in ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', - 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk'): - install(x, prefix.bin) + binfiles = ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', + 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk') + for binfile in binfiles: + install(binfile, prefix.bin) mkdir(prefix.lib) install('libmetis.a', prefix.lib) @@ -82,106 +88,120 @@ class Metis(Package): install(h, prefix.include) mkdir(prefix.share) - for f in (join_path(*p) - for p in (('Programs', 'io.c'), - ('Test','mtest.c'), - ('Graphs','4elt.graph'), - ('Graphs', 'metis.mesh'), - ('Graphs', 'test.mgraph'))): - install(f, prefix.share) + sharefiles = (('Graphs', '4elt.graph'), ('Graphs', 'metis.mesh'), + ('Graphs', 'test.mgraph')) + for sharefile in tuple(join_path(*sf) for sf in sharefiles): + install(sharefile, prefix.share) if '+shared' in spec: + shared_flags = ['-fPIC', '-shared'] if sys.platform == 'darwin': - lib_dsuffix = 'dylib' - load_flag = '-Wl,-all_load' - no_load_flag = '' + shared_suffix = 'dylib' + shared_flags.extend(['-Wl,-all_load', 'libmetis.a']) else: - lib_dsuffix = 'so' - load_flag = '-Wl,-whole-archive' - no_load_flag = '-Wl,-no-whole-archive' + shared_suffix = 'so' + shared_flags.extend(['-Wl,-whole-archive', 'libmetis.a', + '-Wl,-no-whole-archive']) - os.system(spack_cc + ' -fPIC -shared ' + load_flag + - ' libmetis.a ' + no_load_flag + ' -o libmetis.' + - lib_dsuffix) - install('libmetis.' + lib_dsuffix, prefix.lib) + shared_out = '%s/libmetis.%s' % (prefix.lib, shared_suffix) + shared_flags.extend(['-o', shared_out]) - # Set up and run tests on installation - symlink(join_path(prefix.share, 'io.c'), 'io.c') - symlink(join_path(prefix.share, 'mtest.c'), 'mtest.c') - os.system(spack_cc + ' -I%s' % prefix.include + ' -c io.c') - os.system(spack_cc + ' -I%s' % prefix.include + - ' -L%s' % prefix.lib + ' -lmetis mtest.c io.o -o mtest') - _4eltgraph = join_path(prefix.share, '4elt.graph') - test_mgraph = join_path(prefix.share, 'test.mgraph') - metis_mesh = join_path(prefix.share, 'metis.mesh') - kmetis = join_path(prefix.bin, 'kmetis') - os.system('./mtest ' + _4eltgraph) - os.system(kmetis + ' ' + _4eltgraph + ' 40') - os.system(join_path(prefix.bin, 'onmetis') + ' ' + _4eltgraph) - os.system(join_path(prefix.bin, 'pmetis') + ' ' + test_mgraph + ' 2') - os.system(kmetis + ' ' + test_mgraph + ' 2') - os.system(kmetis + ' ' + test_mgraph + ' 5') - os.system(join_path(prefix.bin, 'partnmesh') + metis_mesh + ' 10') - os.system(join_path(prefix.bin, 'partdmesh') + metis_mesh + ' 10') - os.system(join_path(prefix.bin, 'mesh2dual') + metis_mesh) + ccompile(*shared_flags) + # Set up and run tests on installation + ccompile('-I%s' % prefix.include, '-L%s' % prefix.lib, + '-Wl,-rpath=%s' % (prefix.lib if '+shared' in spec else ''), + join_path('Programs', 'io.o'), join_path('Test', 'mtest.c'), + '-o', '%s/mtest' % prefix.bin, '-lmetis', '-lm') + + test_bin = lambda testname: join_path(prefix.bin, testname) + test_graph = lambda graphname: join_path(prefix.share, graphname) + + graph = test_graph('4elt.graph') + os.system('%s %s' % (test_bin('mtest'), graph)) + os.system('%s %s 40' % (test_bin('kmetis'), graph)) + os.system('%s %s' % (test_bin('onmetis'), graph)) + graph = test_graph('test.mgraph') + os.system('%s %s 2' % (test_bin('pmetis'), graph)) + os.system('%s %s 2' % (test_bin('kmetis'), graph)) + os.system('%s %s 5' % (test_bin('kmetis'), graph)) + graph = test_graph('metis.mesh') + os.system('%s %s 10' % (test_bin('partnmesh'), graph)) + os.system('%s %s 10' % (test_bin('partdmesh'), graph)) + os.system('%s %s' % (test_bin('mesh2dual'), graph)) + + # FIXME: The following code should replace the testing code in the + # block above since it causes installs to fail when one or more of the + # Metis tests fail, but it currently doesn't work because the 'mtest', + # 'onmetis', and 'partnmesh' tests return error codes that trigger + # false positives for failure. + """ + Executable(test_bin('mtest'))(test_graph('4elt.graph')) + Executable(test_bin('kmetis'))(test_graph('4elt.graph'), '40') + Executable(test_bin('onmetis'))(test_graph('4elt.graph')) + + Executable(test_bin('pmetis'))(test_graph('test.mgraph'), '2') + Executable(test_bin('kmetis'))(test_graph('test.mgraph'), '2') + Executable(test_bin('kmetis'))(test_graph('test.mgraph'), '5') + + Executable(test_bin('partnmesh'))(test_graph('metis.mesh'), '10') + Executable(test_bin('partdmesh'))(test_graph('metis.mesh'), '10') + Executable(test_bin('mesh2dual'))(test_graph('metis.mesh')) + """ @when('@5:') def install(self, spec, prefix): - options = [] options.extend(std_cmake_args) build_directory = join_path(self.stage.path, 'spack-build') source_directory = self.stage.source_path - options.append('-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=source_directory)) + options.append('-DGKLIB_PATH:PATH=%s/GKlib' % source_directory) options.append('-DCMAKE_INSTALL_NAME_DIR:PATH=%s/lib' % prefix) if '+shared' in spec: options.append('-DSHARED:BOOL=ON') - if '+debug' in spec: options.extend(['-DDEBUG:BOOL=ON', '-DCMAKE_BUILD_TYPE:STRING=Debug']) - if '+gdb' in spec: options.append('-DGDB:BOOL=ON') metis_header = join_path(source_directory, 'include', 'metis.h') - if '+idx64' in spec: filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) - - if '+double' in spec: + if '+real64' in spec: filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) # Make clang 7.3 happy. # Prevents "ld: section __DATA/__thread_bss extends beyond end of file" # See upstream LLVM issue https://llvm.org/bugs/show_bug.cgi?id=27059 - # Adopted from https://github.com/Homebrew/homebrew-science/blob/master/metis.rb + # and https://github.com/Homebrew/homebrew-science/blob/master/metis.rb if spec.satisfies('%clang@7.3.0'): - filter_file('#define MAX_JBUFS 128', '#define MAX_JBUFS 24', join_path(source_directory, 'GKlib', 'error.c')) + filter_file('#define MAX_JBUFS 128', '#define MAX_JBUFS 24', + join_path(source_directory, 'GKlib', 'error.c')) with working_dir(build_directory, create=True): cmake(source_directory, *options) make() - make("install") + make('install') + # now run some tests: - for f in ["4elt", "copter2", "mdual"]: - graph = join_path(source_directory,'graphs','%s.graph' % f) - Executable(join_path(prefix.bin,'graphchk'))(graph) - Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') - Executable(join_path(prefix.bin,'ndmetis'))(graph) + for f in ['4elt', 'copter2', 'mdual']: + graph = join_path(source_directory, 'graphs', '%s.graph' % f) + Executable(join_path(prefix.bin, 'graphchk'))(graph) + Executable(join_path(prefix.bin, 'gpmetis'))(graph, '2') + Executable(join_path(prefix.bin, 'ndmetis'))(graph) - graph = join_path(source_directory,'graphs','test.mgraph') - Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') - graph = join_path(source_directory,'graphs','metis.mesh') - Executable(join_path(prefix.bin,'mpmetis'))(graph,'2') + graph = join_path(source_directory, 'graphs', 'test.mgraph') + Executable(join_path(prefix.bin, 'gpmetis'))(graph, '2') + graph = join_path(source_directory, 'graphs', 'metis.mesh') + Executable(join_path(prefix.bin, 'mpmetis'))(graph, '2') # install GKlib headers, which will be needed for ParMETIS - GKlib_dist = join_path(prefix.include,'GKlib') + GKlib_dist = join_path(prefix.include, 'GKlib') mkdirp(GKlib_dist) - fs = glob.glob(join_path(source_directory,'GKlib',"*.h")) - for f in fs: - install(f, GKlib_dist) + hfiles = glob.glob(join_path(source_directory, 'GKlib', '*.h')) + for hfile in hfiles: + install(hfile, GKlib_dist) diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py new file mode 100644 index 0000000000..454e78d29c --- /dev/null +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -0,0 +1,28 @@ +from spack import * +import os + +from spack.pkg.builtin.intel import IntelInstaller + + +class Mkl(IntelInstaller): + """Intel Math Kernel Library. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-mkl" + + version('11.3.2.181', '536dbd82896d6facc16de8f961d17d65', + url="file://%s/l_mkl_11.3.2.181.tgz" % os.getcwd()) + version('11.3.3.210', 'f72546df27f5ebb0941b5d21fd804e34', + url="file://%s/l_mkl_11.3.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + mkl_dir = os.path.join(self.intel_prefix, "mkl") + for f in os.listdir(mkl_dir): + os.symlink(os.path.join(mkl_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 164d9db541..ea1844aea0 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -23,15 +23,14 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" homepage = "http://www.mpich.org" - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" - list_url = "http://www.mpich.org/static/downloads/" + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" + list_url = "http://www.mpich.org/static/downloads/" list_depth = 2 version('3.2', 'f414cfa77099cd1fa1a5ae4e22db508a') @@ -42,7 +41,8 @@ class Mpich(Package): version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') - variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('verbs', default=False, + description='Build support for OpenFabrics verbs.') variant('pmi', default=True, description='Build with PMI support') variant('hydra', default=True, description='Build the hydra process manager') @@ -57,9 +57,9 @@ class Mpich(Package): spack_env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dep_spec): - self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++') - self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') def install(self, spec, prefix): @@ -92,7 +92,6 @@ class Mpich(Package): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -102,18 +101,20 @@ class Mpich(Package): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') - - spack_cc = os.environ['CC'] - spack_cxx = os.environ['CXX'] - spack_f77 = os.environ['F77'] - spack_fc = os.environ['FC'] - - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + mpicc = join_path(bin, 'mpicc') + mpicxx = join_path(bin, 'mpicxx') + mpif77 = join_path(bin, 'mpif77') + mpif90 = join_path(bin, 'mpif90') + + # Substitute Spack compile wrappers for the real + # underlying compiler + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + filter_file(env['CC'], self.compiler.cc, mpicc, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, mpicxx, **kwargs) + filter_file(env['F77'], self.compiler.f77, mpif77, **kwargs) + filter_file(env['FC'], self.compiler.fc, mpif90, **kwargs) + + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + for wrapper in (mpicc, mpicxx, mpif77, mpif90): + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/mumps/mumps-shared.patch b/var/spack/repos/builtin/packages/mumps/mumps-shared.patch new file mode 100644 index 0000000000..592f48037c --- /dev/null +++ b/var/spack/repos/builtin/packages/mumps/mumps-shared.patch @@ -0,0 +1,119 @@ +diff -Naur MUMPS_5.0.1/libseq/Makefile MUMPS_5.0.1.new/libseq/Makefile +--- MUMPS_5.0.1/libseq/Makefile 2015-07-23 19:08:32.000000000 +0200 ++++ MUMPS_5.0.1.new/libseq/Makefile 2016-06-07 10:41:16.585179151 +0200 +@@ -8,11 +8,15 @@ + + include ../Makefile.inc + +-libmpiseq: libmpiseq$(PLAT)$(LIBEXT) ++libmpiseq: libmpiseq$(PLAT)$(LIBEXT) libmpiseq$(PLAT)$(SHLIBEXT) + + libmpiseq$(PLAT)$(LIBEXT): mpi.o mpic.o elapse.o + $(AR)$@ mpi.o mpic.o elapse.o + $(RANLIB) $@ ++ ++libmpiseq$(PLAT)$(SHLIBEXT): mpi.o mpic.o elapse.o ++ $(FC) $(LDFLAGS) $^ -o libmpiseq$(PLAT)$(SHLIBEXT) ++ + .f.o: + $(FC) $(OPTF) -c $*.f $(OUTF)$*.o + .c.o: +diff -Naur MUMPS_5.0.1/Makefile MUMPS_5.0.1.new/Makefile +--- MUMPS_5.0.1/Makefile 2015-07-23 19:08:29.000000000 +0200 ++++ MUMPS_5.0.1.new/Makefile 2016-06-07 10:50:21.863281217 +0200 +@@ -51,7 +51,7 @@ + dexamples: d + (cd examples ; $(MAKE) d) + +-requiredobj: Makefile.inc $(LIBSEQNEEDED) $(libdir)/libpord$(PLAT)$(LIBEXT) ++requiredobj: Makefile.inc $(LIBSEQNEEDED) $(libdir)/libpord$(PLAT)$(LIBEXT) $(libdir)/libpord$(PLAT)$(SHLIBEXT) + + # dummy MPI library (sequential version) + +@@ -62,16 +62,25 @@ + $(libdir)/libpord$(PLAT)$(LIBEXT): + if [ "$(LPORDDIR)" != "" ] ; then \ + cd $(LPORDDIR); \ +- $(MAKE) CC="$(CC)" CFLAGS="$(OPTC)" AR="$(AR)" RANLIB="$(RANLIB)" OUTC="$(OUTC)" LIBEXT=$(LIBEXT); \ ++ $(MAKE) CC="$(CC)" CFLAGS="$(OPTC)" AR="$(AR)" RANLIB="$(RANLIB)" LDFLAGS="$(LDFLAGS)" OUTC="$(OUTC)" LIBEXT=$(LIBEXT) PLAT=$(PLAT) SHLIBEXT=$(SHLIBEXT); \ + fi; + if [ "$(LPORDDIR)" != "" ] ; then \ + cp $(LPORDDIR)/libpord$(LIBEXT) $@; \ + fi; + ++$(libdir)/libpord$(PLAT)$(SHLIBEXT): ++ if [ "$(LPORDDIR)" != "" ] ; then \ ++ cd $(LPORDDIR); \ ++ $(MAKE) CC="$(CC)" CFLAGS="$(OPTC)" AR="$(AR)" RANLIB="$(RANLIB)" LDFLAGS="$(LDFLAGS)" OUTC="$(OUTC)" LIBEXT=$(LIBEXT) PLAT=$(PLAT) SHLIBEXT=$(SHLIBEXT) libpord$(PLAT)$(SHLIBEXT); \ ++ fi; ++ if [ "$(LPORDDIR)" != "" ] ; then \ ++ cp $(LPORDDIR)/libpord$(PLAT)$(SHLIBEXT) $@; \ ++ fi; ++ + clean: + (cd src; $(MAKE) clean) + (cd examples; $(MAKE) clean) +- (cd $(libdir); $(RM) *$(PLAT)$(LIBEXT)) ++ (cd $(libdir); $(RM) *$(PLAT)$(LIBEXT) *$(PLAT)$(SHLIBEXT)) + (cd libseq; $(MAKE) clean) + if [ "$(LPORDDIR)" != "" ] ; then \ + cd $(LPORDDIR); $(MAKE) realclean; \ +diff -Naur MUMPS_5.0.1/PORD/lib/Makefile MUMPS_5.0.1.new/PORD/lib/Makefile +--- MUMPS_5.0.1/PORD/lib/Makefile 2015-07-23 19:08:29.000000000 +0200 ++++ MUMPS_5.0.1.new/PORD/lib/Makefile 2016-06-07 10:49:48.889000958 +0200 +@@ -13,7 +13,7 @@ + + OBJS = graph.o gbipart.o gbisect.o ddcreate.o ddbisect.o nestdiss.o \ + multisector.o gelim.o bucket.o tree.o \ +- symbfac.o interface.o sort.o minpriority.o ++ symbfac.o interface.o sort.o minpriority.o + + # Note: numfac.c read.c mapping.c triangular.c matrix.c kernel.c + # were not direcly used by MUMPS and have been removed from the +@@ -24,12 +24,15 @@ + .c.o: + $(CC) $(COPTIONS) -c $*.c $(OUTC)$*.o + +-libpord$(LIBEXT):$(OBJS) ++libpord$(PLAT)$(LIBEXT):$(OBJS) + $(AR)$@ $(OBJS) + $(RANLIB) $@ + ++libpord$(PLAT)$(SHLIBEXT): $(OBJS) ++ $(CC) $(LDFLAGS) $(OBJS) -o libpord$(PLAT)$(SHLIBEXT) ++ + clean: + rm -f *.o + + realclean: +- rm -f *.o libpord.a ++ rm -f *.o libpord$(PLAT)$(SHLIBEXT) libpord$(PLAT)$(LIBEXT) +diff -Naur MUMPS_5.0.1/src/Makefile MUMPS_5.0.1.new/src/Makefile +--- MUMPS_5.0.1/src/Makefile 2015-07-23 19:08:29.000000000 +0200 ++++ MUMPS_5.0.1.new/src/Makefile 2016-06-07 10:40:52.534703722 +0200 +@@ -24,7 +24,10 @@ + include $(topdir)/Makefile.inc + + mumps_lib: $(libdir)/libmumps_common$(PLAT)$(LIBEXT) \ +- $(libdir)/lib$(ARITH)mumps$(PLAT)$(LIBEXT) ++ $(libdir)/libmumps_common$(PLAT)$(SHLIBEXT) \ ++ $(libdir)/lib$(ARITH)mumps$(PLAT)$(LIBEXT) \ ++ $(libdir)/lib$(ARITH)mumps$(PLAT)$(SHLIBEXT) ++ + + OBJS_COMMON_MOD = \ + ana_omp_m.o\ +@@ -162,6 +165,13 @@ + $(AR)$@ $? + $(RANLIB) $@ + ++$(libdir)/libmumps_common$(PLAT)$(SHLIBEXT): $(OBJS_COMMON_MOD) $(OBJS_COMMON_OTHER) ++ $(FC) $(LDFLAGS) $^ -L$(libdir) $(LORDERINGS) $(LIBS) $(LIBBLAS) $(LIBOTHERS) -o $(libdir)/libmumps_common$(PLAT)$(SHLIBEXT) ++ ++ ++$(libdir)/lib$(ARITH)mumps$(PLAT)$(SHLIBEXT): $(OBJS_MOD) $(OBJS_OTHER) ++ $(FC) $(LDFLAGS) $^ -L$(libdir) -lmumps_common$(PLAT) $(LORDERINGS) $(LIBS) $(LIBBLAS) $(LIBOTHERS) -o $(libdir)/lib$(ARITH)mumps$(PLAT)$(SHLIBEXT) ++ + # Dependencies between modules: + $(ARITH)mumps_load.o: $(ARITH)mumps_comm_buffer.o \ + $(ARITH)mumps_struc_def.o \ diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 92c45c9b95..8a90ef72d0 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -23,7 +23,10 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, sys, glob +import os +import sys +import subprocess + class Mumps(Package): """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" @@ -44,7 +47,6 @@ class Mumps(Package): variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') variant('shared', default=True, description='Build shared libraries') - depends_on('scotch + esmumps', when='~ptscotch+scotch') depends_on('scotch + esmumps + mpi', when='+ptscotch') depends_on('metis@5:', when='+metis') @@ -54,49 +56,64 @@ class Mumps(Package): depends_on('scalapack', when='+mpi') depends_on('mpi', when='+mpi') + patch('mumps-shared.patch', when='+shared') + # this function is not a patch function because in case scalapack # is needed it uses self.spec['scalapack'].fc_link set by the # setup_dependent_environment in scalapck. This happen after patch # end before install # def patch(self): def write_makefile_inc(self): - if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: - raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') + if (('+parmetis' in self.spec or + '+ptscotch' in self.spec)) and '+mpi' not in self.spec: + raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') # NOQA: E501 - makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] + makefile_conf = [ + "LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib + ] orderings = ['-Dpord'] if '+ptscotch' in self.spec or '+scotch' in self.spec: join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') - makefile_conf.extend( - ["ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, - "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, - join_lib, - join_lib.join(['esmumps', 'scotch', 'scotcherr']))]) + makefile_conf.extend([ + "ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, + "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, + join_lib, + join_lib.join(['esmumps', + 'scotch', + 'scotcherr'])) + ]) orderings.append('-Dscotch') if '+ptscotch' in self.spec: orderings.append('-Dptscotch') if '+parmetis' in self.spec and '+metis' in self.spec: - libname = 'parmetis' if '+parmetis' in self.spec else 'metis' - makefile_conf.extend( - ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include, - "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')]) + makefile_conf.extend([ + "IMETIS = -I%s" % self.spec['parmetis'].prefix.include, + "LMETIS = -L%s -l%s -L%s -l%s" % ( + self.spec['parmetis'].prefix.lib, 'parmetis', + self.spec['metis'].prefix.lib, 'metis') + ]) orderings.append('-Dparmetis') elif '+metis' in self.spec: - makefile_conf.extend( - ["IMETIS = -I%s" % self.spec['metis'].prefix.include, - "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')]) + makefile_conf.extend([ + "IMETIS = -I%s" % self.spec['metis'].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, + 'metis') + ]) orderings.append('-Dmetis') makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) - # when building shared libs need -fPIC, otherwise - # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1' can not be used when making a shared object; recompile with -fPIC + # when building shared libs need -fPIC, otherwise /usr/bin/ld: + # graph.o: relocation R_X86_64_32 against `.rodata.str1.1' can + # not be used when making a shared object; recompile with + # -fPIC fpic = '-fPIC' if '+shared' in self.spec else '' + # TODO: test this part, it needs a full blas, scalapack and # partitionning environment with 64bit integers if '+idx64' in self.spec: @@ -104,7 +121,7 @@ class Mumps(Package): # the fortran compilation flags most probably are # working only for intel and gnu compilers this is # perhaps something the compiler should provide - ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic,'-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), + ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic, '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), # NOQA: E501 'OPTL = %s -O ' % fpic, 'OPTC = %s -O -DINTSIZE64' % fpic]) else: @@ -113,49 +130,47 @@ class Mumps(Package): 'OPTL = %s -O ' % fpic, 'OPTC = %s -O ' % fpic]) - if '+mpi' in self.spec: makefile_conf.extend( - ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), - "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + ["CC = %s" % self.spec['mpi'].mpicc, + "FC = %s" % self.spec['mpi'].mpifc, "SCALAP = %s" % self.spec['scalapack'].fc_link, "MUMPS_TYPE = par"]) else: makefile_conf.extend( ["CC = cc", "FC = fc", - "FL = fc", "MUMPS_TYPE = seq"]) # TODO: change the value to the correct one according to the # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER - makefile_conf.append("CDEFS = -DAdd_") + makefile_conf.extend([ + 'CDEFS = -DAdd_', + 'FL = $(FC)', + ]) if '+shared' in self.spec: + makefile_conf.append('SHLIBEXT = .%s' % dso_suffix) if sys.platform == 'darwin': - # Building dylibs with mpif90 causes segfaults on 10.8 and 10.10. Use gfortran. (Homebrew) - makefile_conf.extend([ - 'LIBEXT=.dylib', - 'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'],prefix.lib), - 'RANLIB=echo' - ]) + makefile_conf.append( + 'LDFLAGS = -dynamiclib -Wl,-install_name -Wl,{0}/$(notdir $@) {1}{0} -undefined dynamic_lookup'.format(prefix.lib, self.compiler.fc_rpath_arg) # NOQA: E501 + ) else: - makefile_conf.extend([ - 'LIBEXT=.so', - 'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib, - 'RANLIB=echo' - ]) - else: - makefile_conf.extend([ - 'LIBEXT = .a', - 'AR = ar vr', - 'RANLIB = ranlib' - ]) - - - makefile_inc_template = join_path(os.path.dirname(self.module.__file__), - 'Makefile.inc') + makefile_conf.append( + 'LDFLAGS = -shared {0}{1}'.format( + self.compiler.fc_rpath_arg, + prefix.lib) + ) + + makefile_conf.extend([ + 'LIBEXT = .a', + 'AR = ar vr ', + 'RANLIB = ranlib' + ]) + + makefile_inc_template = \ + join_path(os.path.dirname(self.module.__file__), + 'Makefile.inc') with open(makefile_inc_template, "r") as fh: makefile_conf.extend(fh.read().split('\n')) @@ -164,46 +179,53 @@ class Mumps(Package): makefile_inc = '\n'.join(makefile_conf) fh.write(makefile_inc) - - def install(self, spec, prefix): make_libs = [] # the choice to compile ?examples is to have kind of a sanity # check on the libraries generated. if '+float' in spec: - make_libs.append('sexamples') + make_libs.append('s') if '+complex' in spec: - make_libs.append('cexamples') + make_libs.append('c') if '+double' in spec: - make_libs.append('dexamples') + make_libs.append('d') if '+complex' in spec: - make_libs.append('zexamples') + make_libs.append('z') self.write_makefile_inc() - # Build fails in parallel - make(*make_libs, parallel=False) + make('mumps_lib', parallel=False) + make(*make_libs) install_tree('lib', prefix.lib) install_tree('include', prefix.include) - if '~mpi' in spec: - lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' - lib_suffix = lib_dsuffix if '+shared' in spec else '.a' - install('libseq/libmpiseq%s' % lib_suffix, prefix.lib) - for f in glob.glob(join_path('libseq','*.h')): - install(f, prefix.include) - - # FIXME: extend the tests to mpirun -np 2 (or alike) when build with MPI - # FIXME: use something like numdiff to compare blessed output with the current - with working_dir('examples'): - if '+float' in spec: - os.system('./ssimpletest < input_simpletest_real') - if '+complex' in spec: - os.system('./csimpletest < input_simpletest_real') - if '+double' in spec: - os.system('./dsimpletest < input_simpletest_real') - if '+complex' in spec: - os.system('./zsimpletest < input_simpletest_cmplx') + if '~mpi' in spec: + install('libseq/libmpiseq.a', prefix.lib) + if '+shared' in spec: + install('libseq/libmpiseq.{0}'.format(dso_suffix), prefix.lib) + install('libseq/mpi.h', prefix.include) + install('libseq/mpif.h', prefix.include) + + # FIXME: extend the tests to mpirun -np 2 (or alike) when + # build with MPI + # FIXME: use something like numdiff to compare blessed output + # with the current + # TODO: test the installed mumps and not the one in stage + if '~mpi' in spec: + for t in make_libs: + make('{0}examples'.format(t)) + + with working_dir('examples'): + for t in make_libs: + input_file = 'input_simpletest_{0}'.format( + 'real' if t in ['s', 'd'] else 'cmplx') + with open(input_file) as input: + test = './{0}simpletest'.format(t) + ret = subprocess.call(test, + stdin=input) + if ret is not 0: + raise RuntimeError( + 'The test {0} did not pass'.format(test)) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index f4997bdfa1..34bace10d6 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os + class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" @@ -43,8 +43,9 @@ class Mvapich2(Package): variant('debug', default=False, description='Enables debug information and error messages at run-time') ########## - # TODO : Process managers should be grouped into the same variant, as soon as variant capabilities will be extended - # See https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so + # TODO : Process managers should be grouped into the same variant, + # as soon as variant capabilities will be extended See + # https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so SLURM = 'slurm' HYDRA = 'hydra' GFORKER = 'gforker' @@ -57,7 +58,8 @@ class Mvapich2(Package): ########## ########## - # TODO : Network types should be grouped into the same variant, as soon as variant capabilities will be extended + # TODO : Network types should be grouped into the same variant, as + # soon as variant capabilities will be extended PSM = 'psm' SOCK = 'sock' NEMESISIBTCP = 'nemesisibtcp' @@ -84,8 +86,8 @@ class Mvapich2(Package): @staticmethod def enabled(x): - """ - Given a variant name returns the string that means the variant is enabled + """Given a variant name returns the string that means the variant is + enabled :param x: variant name :return: @@ -93,8 +95,8 @@ class Mvapich2(Package): return '+' + x def set_build_type(self, spec, configure_args): - """ - Appends to configure_args the flags that depends only on the build type (i.e. release or debug) + """Appends to configure_args the flags that depends only on the build + type (i.e. release or debug) :param spec: spec :param configure_args: list of current configure arguments @@ -104,7 +106,8 @@ class Mvapich2(Package): "--disable-fast", "--enable-error-checking=runtime", "--enable-error-messages=all", - "--enable-g=dbg", "--enable-debuginfo" # Permits debugging with TotalView + # Permits debugging with TotalView + "--enable-g=dbg", "--enable-debuginfo" ] else: build_type_options = ["--enable-fast=all"] @@ -112,25 +115,41 @@ class Mvapich2(Package): configure_args.extend(build_type_options) def set_process_manager(self, spec, configure_args): - """ - Appends to configure_args the flags that will enable the appropriate process managers + """Appends to configure_args the flags that will enable the + appropriate process managers :param spec: spec :param configure_args: list of current configure arguments """ - # Check that slurm variant is not activated together with other pm variants - has_slurm_incompatible_variants = any(self.enabled(x) in spec for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) - if self.enabled(Mvapich2.SLURM) in spec and has_slurm_incompatible_variants: - raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) + # Check that slurm variant is not activated together with + # other pm variants + has_slurm_incompatible_variants = \ + any(self.enabled(x) in spec + for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) + + if self.enabled(Mvapich2.SLURM) in spec and \ + has_slurm_incompatible_variants: + raise RuntimeError(" %s : 'slurm' cannot be activated \ + together with other process managers" % self.name) process_manager_options = [] + # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 if self.enabled(Mvapich2.SLURM) in spec: - process_manager_options = [ - "--with-pm=slurm" - ] + if self.version > Version('2.0'): + process_manager_options = [ + "--with-pmi=pmi2", + "--with-pm=slurm" + ] + else: + process_manager_options = [ + "--with-pmi=slurm", + "--with-pm=no" + ] + elif has_slurm_incompatible_variants: pms = [] - # The variant name is equal to the process manager name in the configuration options + # The variant name is equal to the process manager name in + # the configuration options for x in Mvapich2.SLURM_INCOMPATIBLE_PMS: if self.enabled(x) in spec: pms.append(x) @@ -146,7 +165,9 @@ class Mvapich2(Package): if self.enabled(x) in spec: count += 1 if count > 1: - raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') + raise RuntimeError('network variants are mutually exclusive \ + (only one can be selected at a time)') + network_options = [] # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: @@ -164,6 +185,11 @@ class Mvapich2(Package): configure_args.extend(network_options) + def setup_environment(self, spack_env, run_env): + if self.enabled(Mvapich2.SLURM) in self.spec and \ + self.version > Version('2.0'): + run_env.set('SLURM_MPI_TYPE', 'pmi2') + def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) @@ -178,7 +204,8 @@ class Mvapich2(Package): self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') def install(self, spec, prefix): - # we'll set different configure flags depending on our environment + # we'll set different configure flags depending on our + # environment configure_args = [ "--prefix=%s" % prefix, "--enable-shared", @@ -208,7 +235,6 @@ class Mvapich2(Package): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -218,18 +244,20 @@ class Mvapich2(Package): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') - - spack_cc = os.environ['CC'] - spack_cxx = os.environ['CXX'] - spack_f77 = os.environ['F77'] - spack_fc = os.environ['FC'] - - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + mpicc = join_path(bin, 'mpicc') + mpicxx = join_path(bin, 'mpicxx') + mpif77 = join_path(bin, 'mpif77') + mpif90 = join_path(bin, 'mpif90') + + # Substitute Spack compile wrappers for the real + # underlying compiler + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + filter_file(env['CC'], self.compiler.cc, mpicc, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, mpicxx, **kwargs) + filter_file(env['F77'], self.compiler.f77, mpif77, **kwargs) + filter_file(env['FC'], self.compiler.fc, mpif90, **kwargs) + + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + for wrapper in (mpicc, mpicxx, mpif77, mpif90): + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index b67ea299a8..f8af76429b 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -24,16 +24,21 @@ ############################################################################## from spack import * + class NetcdfCxx4(Package): """C++ interface for NetCDF4""" homepage = "http://www.unidata.ucar.edu/software/netcdf" - url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" + url = "https://www.github.com/unidata/netcdf-cxx4/tarball/v4.3.0" - version('4.2', 'd019853802092cf686254aaba165fc81') + version('4.3.0', '0dde8b9763eecdafbd69d076e687337e') + version('4.2.1', 'd019853802092cf686254aaba165fc81') depends_on('netcdf') + depends_on("autoconf") def install(self, spec, prefix): + # Rebuild to prevent problems of inconsistency in git repo + which('autoreconf')('-ivf') configure('--prefix=%s' % prefix) make() make("install") diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 17c7ff82f4..72ff0ee6fc 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -23,6 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import sys + class Octave(Package): """GNU Octave is a high-level language, primarily intended for numerical @@ -34,7 +36,8 @@ class Octave(Package): homepage = "https://www.gnu.org/software/octave/" url = "ftp://ftp.gnu.org/gnu/octave/octave-4.0.0.tar.gz" - version('4.0.0' , 'a69f8320a4f20a8480c1b278b1adb799') + version('4.0.2', 'c2a5cacc6e4c52f924739cdf22c2c687') + version('4.0.0', 'a69f8320a4f20a8480c1b278b1adb799') # Variants variant('readline', default=True) @@ -62,33 +65,35 @@ class Octave(Package): # Required dependencies depends_on('blas') depends_on('lapack') + # Octave does not configure with sed from darwin: + depends_on('sed', sys.platform == 'darwin') depends_on('pcre') + depends_on('pkg-config') # Strongly recommended dependencies - depends_on('readline', when='+readline') + depends_on('readline', when='+readline') # Optional dependencies - depends_on('arpack', when='+arpack') - depends_on('curl', when='+curl') - depends_on('fftw', when='+fftw') - depends_on('fltk', when='+fltk') - depends_on('fontconfig', when='+fontconfig') - depends_on('freetype', when='+freetype') - depends_on('glpk', when='+glpk') - depends_on('gl2ps', when='+gl2ps') - depends_on('gnuplot', when='+gnuplot') - depends_on('ImageMagick', when='+magick') - depends_on('hdf5', when='+hdf5') - depends_on('jdk', when='+jdk') - depends_on('llvm', when='+llvm') - #depends_on('opengl', when='+opengl') # TODO: add package - depends_on('qhull', when='+qhull') - depends_on('qrupdate', when='+qrupdate') - #depends_on('qscintilla', when='+qscintilla) # TODO: add package - depends_on('qt', when='+qt') - depends_on('suite-sparse',when='+suitesparse') - depends_on('zlib', when='+zlib') - + depends_on('arpack', when='+arpack') + depends_on('curl', when='+curl') + depends_on('fftw', when='+fftw') + depends_on('fltk', when='+fltk') + depends_on('fontconfig', when='+fontconfig') + depends_on('freetype', when='+freetype') + depends_on('glpk', when='+glpk') + depends_on('gl2ps', when='+gl2ps') + depends_on('gnuplot', when='+gnuplot') + depends_on('ImageMagick', when='+magick') + depends_on('hdf5', when='+hdf5') + depends_on('jdk', when='+jdk') + depends_on('llvm', when='+llvm') + # depends_on('opengl', when='+opengl') # TODO: add package + depends_on('qhull', when='+qhull') + depends_on('qrupdate', when='+qrupdate') + # depends_on('qscintilla', when='+qscintilla) # TODO: add package + depends_on('qt', when='+qt') + depends_on('suite-sparse', when='+suitesparse') + depends_on('zlib', when='+zlib') def install(self, spec, prefix): config_args = [ @@ -154,7 +159,8 @@ class Octave(Package): config_args.append("--without-glpk") if '+magick' in spec: - config_args.append("--with-magick=%s" % spec['ImageMagick'].prefix.lib) + config_args.append("--with-magick=%s" + % spec['ImageMagick'].prefix.lib) if '+hdf5' in spec: config_args.extend([ @@ -187,7 +193,8 @@ class Octave(Package): if '+qrupdate' in spec: config_args.extend([ - "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, + "--with-qrupdate-includedir=%s" + % spec['qrupdate'].prefix.include, "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib ]) else: diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index ebfec4bded..d09ebd6739 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -48,11 +48,20 @@ class Openblas(Package): patch('make.patch') def install(self, spec, prefix): - # Openblas is picky about compilers. Configure fails with - # FC=/abs/path/to/f77, whereas FC=f77 works fine. - # To circumvent this, provide basename only: - make_defs = ['CC=%s' % os.path.basename(spack_cc), - 'FC=%s' % os.path.basename(spack_f77), + # As of 06/2016 there is no mechanism to specify that packages which + # depends on Blas/Lapack need C or/and Fortran symbols. For now + # require both. + if self.compiler.f77 is None: + raise InstallError('OpenBLAS requires both C and Fortran ', + 'compilers!') + + # Configure fails to pick up fortran from FC=/abs/path/to/f77, but + # works fine with FC=/abs/path/to/gfortran. + # When mixing compilers make sure that + # $SPACK_ROOT/lib/spack/env/<compiler> have symlinks with reasonable + # names and hack them inside lib/spack/spack/compilers/<compiler>.py + make_defs = ['CC=%s' % spack_cc, + 'FC=%s' % spack_f77, 'MAKE_NO_J=1'] make_targets = ['libs', 'netlib'] diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 163990bf15..588d862d96 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -24,9 +24,31 @@ ############################################################################## import os +import llnl.util.tty as tty + from spack import * +def _verbs_dir(): + """ + Try to find the directory where the OpenFabrics verbs package is + installed. Return None if not found. + """ + try: + # Try to locate Verbs by looking for a utility in the path + ibv_devices = which("ibv_devices") + # Run it (silently) to ensure it works + ibv_devices(output=str, error=str) + # Get path to executable + path = ibv_devices.exe[0] + # Remove executable name and "bin" directory + path = os.path.dirname(path) + path = os.path.dirname(path) + return path + except: + return None + + class Openmpi(Package): """Open MPI is a project combining technologies and resources from several other projects (FT-MPI, LA-MPI, LAM/MPI, and PACX-MPI) @@ -41,6 +63,7 @@ class Openmpi(Package): list_url = "http://www.open-mpi.org/software/ompi/" list_depth = 3 + version('1.10.3', 'e2fe4513200e2aaa1500b762342c674b') version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c') version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') @@ -52,19 +75,28 @@ class Openmpi(Package): patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') - variant('psm2', default=False, description='Build support for the Intel PSM2 library.') - variant('pmi', default=False, description='Build support for PMI-based launchers') - variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('psm2', default=False, + description='Build support for the Intel PSM2 library.') + variant('pmi', default=False, + description='Build support for PMI-based launchers') + variant('verbs', default=_verbs_dir() is not None, + description='Build support for OpenFabrics verbs.') variant('mxm', default=False, description='Build Mellanox Messaging support') - variant('thread_multiple', default=False, description='Enable MPI_THREAD_MULTIPLE support') + variant('thread_multiple', default=False, + description='Enable MPI_THREAD_MULTIPLE support') # TODO : variant support for alps, loadleveler is missing - variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') - variant('slurm', default=False, description='Build SLURM scheduler component') + variant('tm', default=False, + description='Build TM (Torque, PBSPro, and compatible) support') + variant('slurm', default=False, + description='Build SLURM scheduler component') variant('sqlite3', default=False, description='Build sqlite3 support') + variant('vt', default=True, + description='Build support for contributed package vt') + # TODO : support for CUDA is missing provides('mpi@:2.2', when='@1.6.5') @@ -74,8 +106,7 @@ class Openmpi(Package): depends_on('sqlite', when='+sqlite3') def url_for_version(self, version): - return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - + return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) # NOQA: ignore=E501 def setup_dependent_environment(self, spack_env, run_env, dependent_spec): spack_env.set('OMPI_CC', spack_cc) @@ -84,14 +115,30 @@ class Openmpi(Package): spack_env.set('OMPI_F77', spack_f77) def setup_dependent_package(self, module, dep_spec): - self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++') - self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') + def setup_environment(self, spack_env, run_env): + # As of 06/2016 there is no mechanism to specify that packages which + # depends on MPI need C or/and Fortran implementation. For now + # require both. + if (self.compiler.f77 is None) or (self.compiler.fc is None): + tty.warn('OpenMPI : FORTRAN compiler not found') + tty.warn('OpenMPI : FORTRAN bindings will be disabled') + spack_env.unset('FC') + spack_env.unset('F77') + # Setting an attribute here and using it in the 'install' + # method is needed to ensure tty.warn is actually displayed + # to user and not redirected to spack-build.out + self.config_extra = ['--enable-mpi-fortran=none', + '--disable-oshmem-fortran'] + @property def verbs(self): - # Up through version 1.6, this option was previously named --with-openib + # Up through version 1.6, this option was previously named + # --with-openib if self.spec.satisfies('@:1.6'): return 'openib' # In version 1.7, it was renamed to be --with-verbs @@ -103,6 +150,10 @@ class Openmpi(Package): "--with-hwloc=%s" % spec['hwloc'].prefix, "--enable-shared", "--enable-static"] + + if getattr(self, 'config_extra', None) is not None: + config_args.extend(self.config_extra) + # Variant based arguments config_args.extend([ # Schedulers @@ -111,22 +162,27 @@ class Openmpi(Package): # Fabrics '--with-psm' if '+psm' in spec else '--without-psm', '--with-psm2' if '+psm2' in spec else '--without-psm2', - ('--with-%s' % self.verbs) if '+verbs' in spec else ('--without-%s' % self.verbs), '--with-mxm' if '+mxm' in spec else '--without-mxm', # Other options - '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', + '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', # NOQA: ignore=E501 '--with-pmi' if '+pmi' in spec else '--without-pmi', - '--with-sqlite3' if '+sqlite3' in spec else '--without-sqlite3' + '--with-sqlite3' if '+sqlite3' in spec else '--without-sqlite3', + '--enable-vt' if '+vt' in spec else '--disable-vt' ]) + if '+verbs' in spec: + path = _verbs_dir() + if path is not None and path not in ('/usr', '/usr/local'): + config_args.append('--with-%s=%s' % (self.verbs, path)) + else: + config_args.append('--with-%s' % self.verbs) + else: + config_args.append('--without-%s' % self.verbs) # TODO: use variants for this, e.g. +lanl, +llnl, etc. # use this for LANL builds, but for LLNL builds, we need: # "--with-platform=contrib/platform/llnl/optimized" if self.version == ver("1.6.5") and '+lanl' in spec: - config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") - - if not self.compiler.f77 and not self.compiler.fc: - config_args.append("--enable-mpi-fortran=no") + config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") # NOQA: ignore=E501 configure(*config_args) make() @@ -143,40 +199,33 @@ class Openmpi(Package): be bound to whatever compiler they were built with. """ kwargs = {'ignore_absent': True, 'backup': False, 'string': False} - dir = os.path.join(self.prefix, 'share/openmpi/') - - cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', - 'ortecc-wrapper-data.txt', 'shmemcc-wrapper-data.txt'] - - cxx_wrappers = ['mpic++-vt-wrapper-data.txt', 'mpic++-wrapper-data.txt', - 'ortec++-wrapper-data.txt'] - - fc_wrappers = ['mpifort-vt-wrapper-data.txt', - 'mpifort-wrapper-data.txt', 'shmemfort-wrapper-data.txt'] - - for wrapper in cc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cc, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in cxx_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cxx, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in fc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - os.path.join(dir, wrapper), **kwargs) - - # These are symlinks in newer versions, so check that here - f77_wrappers = ['mpif77-vt-wrapper-data.txt', 'mpif77-wrapper-data.txt'] - f90_wrappers = ['mpif90-vt-wrapper-data.txt', 'mpif90-wrapper-data.txt'] - - for wrapper in f77_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.f77, - path, **kwargs) - for wrapper in f90_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - path, **kwargs) + wrapper_basepath = join_path(self.prefix, 'share', 'openmpi') + + wrappers = [ + ('mpicc-vt-wrapper-data.txt', self.compiler.cc), + ('mpicc-wrapper-data.txt', self.compiler.cc), + ('ortecc-wrapper-data.txt', self.compiler.cc), + ('shmemcc-wrapper-data.txt', self.compiler.cc), + ('mpic++-vt-wrapper-data.txt', self.compiler.cxx), + ('mpic++-wrapper-data.txt', self.compiler.cxx), + ('ortec++-wrapper-data.txt', self.compiler.cxx), + ('mpifort-vt-wrapper-data.txt', self.compiler.fc), + ('mpifort-wrapper-data.txt', self.compiler.fc), + ('shmemfort-wrapper-data.txt', self.compiler.fc), + ('mpif90-vt-wrapper-data.txt', self.compiler.fc), + ('mpif90-wrapper-data.txt', self.compiler.fc), + ('mpif77-vt-wrapper-data.txt', self.compiler.f77), + ('mpif77-wrapper-data.txt', self.compiler.f77) + ] + + for wrapper_name, compiler in wrappers: + wrapper = join_path(wrapper_basepath, wrapper_name) + if not os.path.islink(wrapper): + # Substitute Spack compile wrappers for the real + # underlying compiler + match = 'compiler=.*' + substitute = 'compiler={compiler}'.format(compiler=compiler) + filter_file(match, substitute, wrapper, **kwargs) + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 119cdd83c2..377bffe723 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -35,7 +35,7 @@ class Openssl(Package): Transport Layer Security (TLS v1) protocols as well as a full-strength general purpose cryptography library.""" homepage = "http://www.openssl.org" - url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" + url = "https://www.openssl.org/source/openssl-1.0.1h.tar.gz" version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') version('1.0.1r', '1abd905e079542ccae948af37e393d28') @@ -100,7 +100,7 @@ class Openssl(Package): # in the environment, then this will override what is set in the # Makefile, leading to build errors. env.pop('APPS', None) - if spec.satisfies("=darwin-x86_64") or spec.satisfies("=ppc64"): + if spec.satisfies("target=x86_64") or spec.satisfies("target=ppc64"): # This needs to be done for all 64-bit architectures (except Linux, # where it happens automatically?) env['KERNEL_BITS'] = '64' diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index 2dead4a76a..9b36f273e4 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -26,33 +26,36 @@ from spack import * import sys + class Parmetis(Package): - """ - ParMETIS is an MPI-based parallel library that implements a variety of algorithms for partitioning unstructured - graphs, meshes, and for computing fill-reducing orderings of sparse matrices. - """ + """ParMETIS is an MPI-based parallel library that implements a variety of + algorithms for partitioning unstructured graphs, meshes, and for + computing fill-reducing orderings of sparse matrices.""" + homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview' - url = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz' + base_url = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis' version('4.0.3', 'f69c479586bf6bb7aff6a9bc0c739628') + version('4.0.2', '0912a953da5bb9b5e5e10542298ffdce') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') - depends_on('cmake @2.8:') # build dependency + depends_on('cmake@2.8:') # build dependency depends_on('mpi') - - patch('enable_external_metis.patch') depends_on('metis@5:') + patch('enable_external_metis.patch') # bug fixes from PETSc developers - # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ + # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ # NOQA: ignore=E501 patch('pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch') - # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ + # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ # NOQA: ignore=E501 patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') - depends_on('gdb', when='+gdb') + def url_for_version(self, version): + verdir = 'OLD/' if version < Version('3.2.0') else '' + return '%s/%sparmetis-%s.tar.gz' % (Parmetis.base_url, verdir, version) def install(self, spec, prefix): options = [] @@ -60,30 +63,27 @@ class Parmetis(Package): build_directory = join_path(self.stage.path, 'spack-build') source_directory = self.stage.source_path - metis_source = join_path(source_directory, 'metis') - # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case - # FIXME : they use a non-standard name - options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=spec['metis'].prefix.include), - '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), - '-DCMAKE_C_COMPILER:STRING=mpicc', - '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) + options.extend([ + '-DGKLIB_PATH:PATH=%s/GKlib' % spec['metis'].prefix.include, + '-DMETIS_PATH:PATH=%s' % spec['metis'].prefix, + '-DCMAKE_C_COMPILER:STRING=%s' % spec['mpi'].mpicc, + '-DCMAKE_CXX_COMPILER:STRING=%s' % spec['mpi'].mpicxx + ]) if '+shared' in spec: options.append('-DSHARED:BOOL=ON') - if '+debug' in spec: options.extend(['-DDEBUG:BOOL=ON', '-DCMAKE_BUILD_TYPE:STRING=Debug']) - if '+gdb' in spec: options.append('-DGDB:BOOL=ON') with working_dir(build_directory, create=True): cmake(source_directory, *options) make() - make("install") + make('install') - # The shared library is not installed correctly on Darwin; correct this + # The shared library is not installed correctly on Darwin; fix this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/pcre/intel.patch b/var/spack/repos/builtin/packages/pcre/intel.patch new file mode 100644 index 0000000000..f160f55e1b --- /dev/null +++ b/var/spack/repos/builtin/packages/pcre/intel.patch @@ -0,0 +1,12 @@ +diff -up pcre-8.38/pcrecpp.cc.intel pcre-8.38/pcrecpp.cc +--- pcre-8.38/pcrecpp.cc.intel 2014-09-15 07:48:59.000000000 -0600 ++++ pcre-8.38/pcrecpp.cc 2016-06-08 16:16:56.702721214 -0600 +@@ -66,7 +66,7 @@ Arg RE::no_arg((void*)NULL); + // inclusive test if we ever needed it. (Note that not only the + // __attribute__ syntax, but also __USER_LABEL_PREFIX__, are + // gnu-specific.) +-#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) ++#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) && !defined(__INTEL_COMPILER) + # define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) + # define ULP_AS_STRING_INTERNAL(x) #x + # define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index 7a9f3b911d..a43729a1ac 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -24,17 +24,29 @@ ############################################################################## from spack import * + class Pcre(Package): """The PCRE package contains Perl Compatible Regular Expression libraries. These are useful for implementing regular expression pattern matching using the same syntax and semantics as Perl 5.""" + homepage = "http://www.pcre.org""" url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2" version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') version('8.38', '00aabbfe56d5a48b270f999b508c5ad2') + patch("intel.patch") + + variant('utf', default=True, + description='Enable support for UTF-8/16/32, ' + 'incompatible with EBCDIC.') + def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = ['--prefix=%s' % prefix] + if '+utf' in spec: + configure_args.append('--enable-utf') + + configure(*configure_args) make() make("install") diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 6456a1aabf..8dd1d8d2b9 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -28,13 +28,16 @@ from spack import * class Petsc(Package): """ - PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications - modeled by partial differential equations. + PETSc is a suite of data structures and routines for the scalable + (parallel) solution of scientific applications modeled by partial + differential equations. """ homepage = "http://www.mcs.anl.gov/petsc/index.html" url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + version('3.7.2', '50da49867ce7a49e7a0c1b37f4ec7b34') + version('3.6.4', '7632da2375a3df35b8891c9526dbdde7') version('3.6.3', '91dd3522de5a5ef039ff8f50800db606') version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') @@ -69,10 +72,12 @@ class Petsc(Package): depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') # Hypre does not support complex numbers. - # Also PETSc prefer to build it without internal superlu, likely due to conflict in headers - # see https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py + # Also PETSc prefer to build it without internal superlu, likely due to + # conflict in headers see + # https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py # NOQA: ignore=E501 depends_on('hypre~internal-superlu', when='+hypre+mpi~complex') - depends_on('superlu-dist', when='+superlu-dist+mpi') + depends_on('superlu-dist@:4.3', when='@:3.6.4+superlu-dist+mpi') + depends_on('superlu-dist@5.0.0:', when='@3.7:+superlu-dist+mpi') depends_on('mumps+mpi', when='+mumps+mpi') depends_on('scalapack', when='+mumps+mpi') @@ -80,17 +85,17 @@ class Petsc(Package): if '~mpi' in self.spec: compiler_opts = [ '--with-cc=%s' % os.environ['CC'], - '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), - '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), + '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), # NOQA: ignore=E501 + '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), # NOQA: ignore=E501 '--with-mpi=0' ] - error_message_fmt = '\t{library} support requires "+mpi" to be activated' + error_message_fmt = '\t{library} support requires "+mpi" to be activated' # NOQA: ignore=E501 - # If mpi is disabled (~mpi), it's an error to have any of these enabled. - # This generates a list of any such errors. + # If mpi is disabled (~mpi), it's an error to have any of these + # enabled. This generates a list of any such errors. errors = [error_message_fmt.format(library=x) - for x in ('hdf5', 'hypre', 'parmetis','mumps','superlu-dist') - if ('+'+x) in self.spec] + for x in ('hdf5', 'hypre', 'parmetis', 'mumps', 'superlu-dist') # NOQA: ignore=E501 + if ('+' + x) in self.spec] if errors: errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) @@ -105,26 +110,31 @@ class Petsc(Package): options = ['--with-ssl=0'] options.extend(self.mpi_dependent_options()) options.extend([ - '--with-precision=%s' % ('double' if '+double' in spec else 'single'), - '--with-scalar-type=%s' % ('complex' if '+complex' in spec else 'real'), + '--with-precision=%s' % ('double' if '+double' in spec else 'single'), # NOQA: ignore=E501 + '--with-scalar-type=%s' % ('complex' if '+complex' in spec else 'real'), # NOQA: ignore=E501 '--with-shared-libraries=%s' % ('1' if '+shared' in spec else '0'), '--with-debugging=%s' % ('1' if '+debug' in spec else '0'), '--with-blas-lapack-dir=%s' % spec['lapack'].prefix ]) # Activates library support if needed - for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis','mumps','scalapack'): + for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis', + 'mumps', 'scalapack'): options.append( - '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) + '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) # NOQA: ignore=E501 ) if library in spec: options.append( - '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) + '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) # NOQA: ignore=E501 ) - # PETSc does not pick up SuperluDist from the dir as they look for superlu_dist_4.1.a + # PETSc does not pick up SuperluDist from the dir as they look for + # superlu_dist_4.1.a if 'superlu-dist' in spec: options.extend([ - '--with-superlu_dist-include=%s' % spec['superlu-dist'].prefix.include, - '--with-superlu_dist-lib=%s' % join_path(spec['superlu-dist'].prefix.lib, 'libsuperlu_dist.a'), + '--with-superlu_dist-include=%s' % + spec['superlu-dist'].prefix.include, + '--with-superlu_dist-lib=%s' % + join_path(spec['superlu-dist'].prefix.lib, + 'libsuperlu_dist.a'), '--with-superlu_dist=1' ]) else: diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py new file mode 100644 index 0000000000..fc77ba7412 --- /dev/null +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -0,0 +1,115 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + + +class Psi4(Package): + """Psi4 is an open-source suite of ab initio quantum chemistry + programs designed for efficient, high-accuracy simulations of + a variety of molecular properties.""" + + homepage = "http://www.psicode.org/" + url = "https://github.com/psi4/psi4/archive/0.5.tar.gz" + + version('0.5', '53041b8a9be3958384171d0d22f9fdd0') + + # Required dependencies + depends_on('blas') + depends_on('lapack') + depends_on('boost+chrono+filesystem+python+regex+serialization+system+timer+thread') + depends_on('python') + depends_on('cmake') + depends_on('py-numpy') + + # Optional dependencies + # TODO: add packages for these + # depends_on('perl') + # depends_on('erd') + # depends_on('pcm-solver') + # depends_on('chemps2') + + def install(self, spec, prefix): + cmake_args = [ + '-DBLAS_TYPE={0}'.format(spec['blas'].name.upper()), + '-DBLAS_LIBRARIES={0}'.format(spec['blas'].blas_shared_lib), + '-DLAPACK_TYPE={0}'.format(spec['lapack'].name.upper()), + '-DLAPACK_LIBRARIES={0}'.format(spec['lapack'].lapack_shared_lib), + '-DBOOST_INCLUDEDIR={0}'.format(spec['boost'].prefix.include), + '-DBOOST_LIBRARYDIR={0}'.format(spec['boost'].prefix.lib), + '-DENABLE_CHEMPS2=OFF' + ] + + cmake_args.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *cmake_args) + + make() + make('install') + + self.filter_compilers(spec, prefix) + + def filter_compilers(self, spec, prefix): + """Run after install to tell the configuration files to + use the compilers that Spack built the package with. + + If this isn't done, they'll have PLUGIN_CXX set to + Spack's generic cxx. We want it to be bound to + whatever compiler it was built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + + cc_files = ['bin/psi4-config'] + cxx_files = ['bin/psi4-config', 'include/psi4/psiconfig.h'] + template = 'share/psi4/plugin/Makefile.template' + + for filename in cc_files: + filter_file(os.environ['CC'], self.compiler.cc, + os.path.join(prefix, filename), **kwargs) + + for filename in cxx_files: + filter_file(os.environ['CXX'], self.compiler.cxx, + os.path.join(prefix, filename), **kwargs) + + # The binary still keeps track of the compiler used to install Psi4 + # and uses it when creating a plugin template + filter_file('@PLUGIN_CXX@', self.compiler.cxx, + os.path.join(prefix, template), **kwargs) + + # The binary links to the build include directory instead of the + # installation include directory: + # https://github.com/psi4/psi4/issues/410 + filter_file('@PLUGIN_INCLUDES@', '-I{0}'.format( + ' -I'.join([ + os.path.join(spec['psi4'].prefix.include, 'psi4'), + os.path.join(spec['boost'].prefix.include, 'boost'), + os.path.join(spec['python'].prefix.include, 'python{0}'.format( + spec['python'].version.up_to(2))), + spec['lapack'].prefix.include, + spec['blas'].prefix.include, + '/usr/include' + ]) + ), os.path.join(prefix, template), **kwargs) diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index 86875bbcae..a9962777dc 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -24,29 +24,38 @@ ############################################################################## from spack import * + class PyAstropy(Package): - """ - The Astropy Project is a community effort to develop a single core + """The Astropy Project is a community effort to develop a single core package for Astronomy in Python and foster interoperability between - Python astronomy packages. - """ + Python astronomy packages.""" + homepage = 'http://www.astropy.org/' + url = 'https://pypi.python.org/packages/source/a/astropy/astropy-1.1.2.tar.gz' + version('1.1.2', 'cbe32023b5b1177d1e2498a0d00cda51') version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d') - def url_for_version(self, version): - return 'https://pypi.python.org/packages/source/a/astropy/astropy-{0}.tar.gz'.format(version) - + # Required dependencies extends('python') + depends_on('py-numpy') - depends_on('cfitsio') - depends_on('expat') + # Optional dependencies depends_on('py-h5py') - depends_on('py-numpy') + depends_on('py-beautifulsoup4') + depends_on('py-pyyaml') depends_on('py-scipy') + depends_on('libxml2') + depends_on('py-matplotlib') + depends_on('py-pytz') + depends_on('py-scikit-image') + depends_on('py-pandas') + + # System dependencies + depends_on('cfitsio') + depends_on('expat') def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', - '--use-system-expat') - python('setup.py', 'install', '--prefix=' + prefix) - + '--use-system-expat') + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-autopep8/package.py b/var/spack/repos/builtin/packages/py-autopep8/package.py new file mode 100644 index 0000000000..f2fd3cd683 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-autopep8/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyAutopep8(Package): + """Automatic pep8 formatter""" + homepage = "https://github.com/hhatto/autopep8" + url = "https://github.com/hhatto/autopep8/archive/ver1.2.2.tar.gz" + + version('1.2.2', 'def3d023fc9dfd1b7113602e965ad8e1') + + extends('python') + depends_on('py-setuptools') + depends_on('py-pep8') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py b/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py new file mode 100644 index 0000000000..64368fe70d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyBeautifulsoup4(Package): + """Beautiful Soup is a Python library for pulling data out of HTML and + XML files. It works with your favorite parser to provide idiomatic ways + of navigating, searching, and modifying the parse tree.""" + + homepage = "https://www.crummy.com/software/BeautifulSoup" + url = "https://pypi.python.org/packages/source/b/beautifulsoup4/beautifulsoup4-4.4.1.tar.gz" + + version('4.4.1', '8fbd9a7cac0704645fa20d1419036815') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-emcee/package.py b/var/spack/repos/builtin/packages/py-emcee/package.py new file mode 100644 index 0000000000..d57ef4bd76 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-emcee/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyEmcee(Package): + """emcee is an MIT licensed pure-Python implementation of Goodman & Weare's + Affine Invariant Markov chain Monte Carlo (MCMC) Ensemble sampler.""" + + homepage = "http://dan.iel.fm/emcee/current/" + url = "https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz" + + version('2.1.0', 'c6b6fad05c824d40671d4a4fc58dfff7') + + extends('python') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index de72bac44e..0180b658f5 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -23,21 +23,42 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import re + class PyH5py(Package): - """The h5py package provides both a high- and low-level interface to the HDF5 library from Python.""" + """The h5py package provides both a high- and low-level interface to the + HDF5 library from Python.""" + homepage = "https://pypi.python.org/pypi/h5py" url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz" - version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') + version('2.6.0', 'ec476211bd1de3f5ac150544189b0bf4') version('2.5.0', '6e4301b5ad5da0d51b0a1e5ac19e3b74') + version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') + + variant('mpi', default=False, description='Build with MPI support') - extends('python', ignore=lambda f: re.match(r'bin/cy*', f)) - depends_on('hdf5') - depends_on('py-numpy') - depends_on('py-cython') + extends('python') + + # Build dependencies + depends_on('py-cython@0.19:') + depends_on('pkg-config') + depends_on('hdf5@1.8.4:+mpi', when='+mpi') + depends_on('hdf5@1.8.4:~mpi', when='~mpi') + depends_on('mpi', when='+mpi') + + # Build and runtime dependencies + depends_on('py-numpy@1.6.1:') + + # Runtime dependencies + depends_on('py-six') def install(self, spec, prefix): - python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'configure', + '--hdf5={0}'.format(spec['hdf5'].prefix)) + + if '+mpi' in spec: + env['CC'] = spec['mpi'].mpicc + python('setup.py', 'configure', '--mpi') + + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py new file mode 100644 index 0000000000..47751a702d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyIminuit(Package): + """Interactive IPython-Friendly Minimizer based on SEAL Minuit2.""" + + homepage = "https://pypi.python.org/pypi/iminuit" + url = "https://pypi.python.org/packages/source/i/iminuit/iminuit-1.2.tar.gz" + + version('1.2', '4701ec472cae42015e26251703e6e984') + + # Required dependencies + extends('python') + depends_on('py-setuptools') + + # Optional dependencies + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-cython') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-nestle/package.py b/var/spack/repos/builtin/packages/py-nestle/package.py new file mode 100644 index 0000000000..16506e3eca --- /dev/null +++ b/var/spack/repos/builtin/packages/py-nestle/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyNestle(Package): + """Nested sampling algorithms for evaluating Bayesian evidence.""" + + homepage = "http://kbarbary.github.io/nestle/" + url = "https://pypi.python.org/packages/source/n/nestle/nestle-0.1.1.tar.gz" + + version('0.1.1', '4875c0f9a0a8e263c1d7f5fa6ce604c5') + + # Required dependencies + extends('python') + depends_on('py-numpy') + + # Optional dependencies + depends_on('py-scipy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-pep8/package.py b/var/spack/repos/builtin/packages/py-pep8/package.py new file mode 100644 index 0000000000..987783b392 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pep8/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyPep8(Package): + """python pep8 format checker""" + homepage = "https://github.com/PyCQA/pycodestyle" + url = "https://github.com/PyCQA/pycodestyle/archive/1.7.0.tar.gz" + + version('1.7.0', '31070a3a6391928893cbf5fa523eb8d9') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-ply/package.py b/var/spack/repos/builtin/packages/py-ply/package.py new file mode 100644 index 0000000000..47cd3b5dc8 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ply/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyPly(Package): + """PLY is nothing more than a straightforward lex/yacc implementation.""" + homepage = "http://www.dabeaz.com/ply" + url = "http://www.dabeaz.com/ply/ply-3.8.tar.gz" + + version('3.8', '94726411496c52c87c2b9429b12d5c50') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py new file mode 100644 index 0000000000..82ae2a2e69 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PySncosmo(Package): + """SNCosmo is a Python library for high-level supernova cosmology + analysis.""" + + homepage = "http://sncosmo.readthedocs.io/" + url = "https://pypi.python.org/packages/source/s/sncosmo/sncosmo-1.2.0.tar.gz" + + version('1.2.0', '028e6d1dc84ab1c17d2f3b6378b2cb1e') + + # Required dependencies + # py-sncosmo binaries are duplicates of those from py-astropy + extends('python', ignore=r'bin/.*') + depends_on('py-numpy') + depends_on('py-scipy') + depends_on('py-astropy') + + # Recommended dependencies + depends_on('py-matplotlib') + depends_on('py-iminuit') + depends_on('py-emcee') + depends_on('py-nestle') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-wcsaxes/package.py b/var/spack/repos/builtin/packages/py-wcsaxes/package.py new file mode 100644 index 0000000000..b0adbe3658 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-wcsaxes/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyWcsaxes(Package): + """WCSAxes is a framework for making plots of Astronomical data + in Matplotlib.""" + + homepage = "http://wcsaxes.readthedocs.io/en/latest/index.html" + url = "https://github.com/astrofrog/wcsaxes/archive/v0.8.tar.gz" + + version('0.8', 'de1c60fdae4c330bf5ddb9f1ab5ab920') + + extends('python', ignore=r'bin/pbr') + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-astropy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 86c12498e1..b6a62bf742 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -22,9 +22,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import functools -import glob -import inspect import os import re from contextlib import closing @@ -37,11 +34,10 @@ from spack.util.environment import * class Python(Package): """The Python programming language.""" + homepage = "http://www.python.org" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" - extendable = True - version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36') version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True) @@ -49,6 +45,8 @@ class Python(Package): version('2.7.9', '5eebcaa0030dc4061156d3429657fb83') version('2.7.8', 'd4bca0159acb0b44a781292b5231936f') + extendable = True + depends_on("openssl") depends_on("bzip2") depends_on("readline") @@ -64,39 +62,63 @@ class Python(Package): # Rest of install is pretty standard except setup.py needs to # be able to read the CPPFLAGS and LDFLAGS as it scans for the # library and headers to build - configure_args= [ - "--prefix=%s" % prefix, - "--with-threads", - "--enable-shared", - "CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % ( - spec['openssl'].prefix, spec['bzip2'].prefix, - spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix), - "LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % ( - spec['openssl'].prefix, spec['bzip2'].prefix, - spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix) - ] + cppflags = ' -I'.join([ + spec['openssl'].prefix.include, spec['bzip2'].prefix.include, + spec['readline'].prefix.include, spec['ncurses'].prefix.include, + spec['sqlite'].prefix.include, spec['zlib'].prefix.include + ]) + + ldflags = ' -L'.join([ + spec['openssl'].prefix.lib, spec['bzip2'].prefix.lib, + spec['readline'].prefix.lib, spec['ncurses'].prefix.lib, + spec['sqlite'].prefix.lib, spec['zlib'].prefix.lib + ]) + + config_args = [ + "--prefix={0}".format(prefix), + "--with-threads", + "--enable-shared", + "CPPFLAGS=-I{0}".format(cppflags), + "LDFLAGS=-L{0}".format(ldflags) + ] + if spec.satisfies('@3:'): - configure_args.append('--without-ensurepip') - configure(*configure_args) + config_args.append('--without-ensurepip') + + configure(*config_args) + make() make("install") - # Modify compiler paths in configuration files. This is necessary for - # building site packages outside of spack - filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', - (r'\4\5'), - join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + self.filter_compilers(spec, prefix) + + def filter_compilers(self, spec, prefix): + """Run after install to tell the configuration files and Makefiles + to use the compilers that Spack built the package with. + + If this isn't done, they'll have CC and CXX set to Spack's generic + cc and c++. We want them to be bound to whatever compiler + they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} - python3_version = '' + dirname = join_path(prefix.lib, + 'python{0}'.format(self.version.up_to(2))) + + config = 'config' if spec.satisfies('@3:'): - python3_version = '-%d.%dm' % self.version[:2] - makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') - filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', - (r'\4\5'), - makefile_filepath) + config = 'config-{0}m'.format(self.version.up_to(2)) + + files = [ + '_sysconfigdata.py', + join_path(config, 'Makefile') + ] + for filename in files: + filter_file(env['CC'], self.compiler.cc, + join_path(dirname, filename), **kwargs) + filter_file(env['CXX'], self.compiler.cxx, + join_path(dirname, filename), **kwargs) # ======================================================================== # Set up environment to make install easy for python extensions. @@ -104,57 +126,59 @@ class Python(Package): @property def python_lib_dir(self): - return os.path.join('lib', 'python%d.%d' % self.version[:2]) - + return join_path('lib', 'python{0}'.format(self.version.up_to(2))) @property def python_include_dir(self): - return os.path.join('include', 'python%d.%d' % self.version[:2]) - + return join_path('include', 'python{0}'.format(self.version.up_to(2))) @property def site_packages_dir(self): - return os.path.join(self.python_lib_dir, 'site-packages') - + return join_path(self.python_lib_dir, 'site-packages') def setup_dependent_environment(self, spack_env, run_env, extension_spec): - # TODO: do this only for actual extensions. + """Set PYTHONPATH to include site-packages dir for the + extension and any other python extensions it depends on.""" - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. python_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + python_paths.append(join_path(d.prefix, + self.site_packages_dir)) pythonpath = ':'.join(python_paths) spack_env.set('PYTHONPATH', pythonpath) - # For run time environment set only the path for extension_spec and prepend it to PYTHONPATH + # For run time environment set only the path for + # extension_spec and prepend it to PYTHONPATH if extension_spec.package.extends(self.spec): - run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) - + run_env.prepend_path('PYTHONPATH', join_path( + extension_spec.prefix, self.site_packages_dir)) def setup_dependent_package(self, module, ext_spec): - """ - Called before python modules' install() methods. + """Called before python modules' install() methods. In most cases, extensions will only need to have one line:: - python('setup.py', 'install', '--prefix=%s' % prefix) - """ + python('setup.py', 'install', '--prefix={0}'.format(prefix))""" + # Python extension builds can have a global python executable function - if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): - module.python = Executable(join_path(self.spec.prefix.bin, 'python3')) + if Version("3.0.0") <= self.version < Version("4.0.0"): + module.python = Executable(join_path(self.spec.prefix.bin, + 'python3')) else: - module.python = Executable(join_path(self.spec.prefix.bin, 'python')) + module.python = Executable(join_path(self.spec.prefix.bin, + 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) - module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) - module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) - - # Make the site packages directory for extensions, if it does not exist already. + module.python_lib_dir = join_path(ext_spec.prefix, + self.python_lib_dir) + module.python_include_dir = join_path(ext_spec.prefix, + self.python_include_dir) + module.site_packages_dir = join_path(ext_spec.prefix, + self.site_packages_dir) + + # Make the site packages directory for extensions if ext_spec.package.is_extension: mkdirp(module.site_packages_dir) @@ -167,25 +191,28 @@ class Python(Package): ignore_arg = args.get('ignore', lambda f: False) # Always ignore easy-install.pth, as it needs to be merged. - patterns = [r'easy-install\.pth$'] + patterns = [r'site-packages/easy-install\.pth$'] # Ignore pieces of setuptools installed by other packages. + # Must include directory name or it will remove all site*.py files. if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site[^/]*\.pyc?$') - patterns.append(r'setuptools\.pth') - patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'setuptools.*egg$') + patterns.extend([ + r'bin/easy_install[^/]*$', + r'site-packages/setuptools[^/]*\.egg$', + r'site-packages/setuptools\.pth$', + r'site-packages/site[^/]*\.pyc?$', + r'site-packages/__pycache__/site[^/]*\.pyc?$' + ]) if ext_pkg.name != 'py-numpy': patterns.append(r'bin/f2py$') return match_predicate(ignore_arg, patterns) - def write_easy_install_pth(self, exts): paths = [] for ext in sorted(exts.values()): - ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) - easy_pth = "%s/easy-install.pth" % ext_site_packages + ext_site_packages = join_path(ext.prefix, self.site_packages_dir) + easy_pth = join_path(ext_site_packages, "easy-install.pth") if not os.path.isfile(easy_pth): continue @@ -195,15 +222,18 @@ class Python(Package): line = line.rstrip() # Skip lines matching these criteria - if not line: continue - if re.search(r'^(import|#)', line): continue - if (ext.name != 'py-setuptools' and - re.search(r'setuptools.*egg$', line)): continue + if not line: + continue + if re.search(r'^(import|#)', line): + continue + if ((ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line))): + continue paths.append(line) - site_packages = os.path.join(self.prefix, self.site_packages_dir) - main_pth = "%s/easy-install.pth" % site_packages + site_packages = join_path(self.prefix, self.site_packages_dir) + main_pth = join_path(site_packages, "easy-install.pth") if not paths: if os.path.isfile(main_pth): @@ -211,15 +241,22 @@ class Python(Package): else: with closing(open(main_pth, 'w')) as f: - f.write("import sys; sys.__plen = len(sys.path)\n") + f.write(""" +import sys +sys.__plen = len(sys.path) +""") for path in paths: - f.write("%s\n" % path) - f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " - "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") - + f.write("{0}\n".format(path)) + f.write(""" +new = sys.path[sys.__plen:] +del sys.path[sys.__plen:] +p = getattr(sys, '__egginsert', 0) +sys.path[p:p] = new +sys.__egginsert = p + len(new) +""") def activate(self, ext_pkg, **args): - ignore=self.python_ignore(ext_pkg, args) + ignore = self.python_ignore(ext_pkg, args) args.update(ignore=ignore) super(Python, self).activate(ext_pkg, **args) @@ -228,12 +265,12 @@ class Python(Package): exts[ext_pkg.name] = ext_pkg.spec self.write_easy_install_pth(exts) - def deactivate(self, ext_pkg, **args): args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).deactivate(ext_pkg, **args) exts = spack.install_layout.extension_map(self.spec) - if ext_pkg.name in exts: # Make deactivate idempotent. + # Make deactivate idempotent + if ext_pkg.name in exts: del exts[ext_pkg.name] self.write_easy_install_pth(exts) diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index 0d3b6a3e57..5a6ce72a4b 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -24,15 +24,33 @@ ############################################################################## from spack import * + class RBiocgenerics(Package): """S4 generic functions needed by many Bioconductor packages.""" - homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' - url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" - - version('0.16.1', 'c2148ffd86fc6f1f819c7f68eb2c744f', expand=False) + homepage = 'https://bioconductor.org/packages/BiocGenerics/' + version('bioc-3.3', + git='https://github.com/Bioconductor-mirror/BiocGenerics.git', + branch='release-3.3') + version('bioc-3.2', + git='https://github.com/Bioconductor-mirror/BiocGenerics.git', + branch='release-3.2') extends('R') + def validate(self, spec): + """ + Checks that the version of R is appropriate for the Bioconductor + version. + """ + if spec.satisfies('@bioc-3.3'): + if not spec.satisfies('^R@3.3.0:3.3.9'): + raise InstallError('Must use R-3.3 for Bioconductor-3.3') + elif spec.satisfies('@bioc-3.2'): + if not spec.satisfies('^R@3.2.0:3.2.9'): + raise InstallError('Must use R-3.2 for Bioconductor-3.2') + def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + self.validate(spec) + R('CMD', 'INSTALL', '--library=%s' % + self.module.r_lib_dir, '%s' % self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-R6/package.py b/var/spack/repos/builtin/packages/r-R6/package.py new file mode 100644 index 0000000000..0b75888e79 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-R6/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RR6(Package): + """The R6 package allows the creation of classes with reference semantics, + similar to R's built-in reference classes. Compared to reference classes, + R6 classes are simpler and lighter-weight, and they are not built on S4 + classes so they do not require the methods package. These classes allow + public and private members, and they support inheritance, even when the + classes are defined in different packages.""" + + homepage = "https://github.com/wch/R6/" + url = "https://cran.r-project.org/src/contrib/R6_2.1.2.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/R6" + + version('2.1.2', 'b6afb9430e48707be87638675390e457') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py index 34a1eee79f..96f623a096 100644 --- a/var/spack/repos/builtin/packages/r-abind/package.py +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class RAbind(Package): """Combine multidimensional arrays into a single array. This is a generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and @@ -32,11 +33,12 @@ class RAbind(Package): homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/abind" - version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5', expand=False) + version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5') extends('R') def install(self, spec, prefix): - - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-curl/package.py b/var/spack/repos/builtin/packages/r-curl/package.py new file mode 100644 index 0000000000..c6e8f22a94 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-curl/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RCurl(Package): + """The curl() and curl_download() functions provide highly configurable + drop-in replacements for base url() and download.file() with better + performance, support for encryption (https, ftps), gzip compression, + authentication, and other libcurl goodies. The core of the package + implements a framework for performing fully customized requests where data + can be processed either in memory, on disk, or streaming via the callback + or connection interfaces. Some knowledge of libcurl is recommended; for a + more-user-friendly web client see the 'httr' package which builds on this + package with http specific tools and logic.""" + + homepage = "https://github.com/jeroenooms/curl" + url = "https://cran.r-project.org/src/contrib/curl_0.9.7.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/RCurl" + + version('0.9.7', 'a101f7de948cb828fef571c730f39217') + + extends('R') + + depends_on('curl') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-devtools/package.py b/var/spack/repos/builtin/packages/r-devtools/package.py new file mode 100644 index 0000000000..5f0b7b8779 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-devtools/package.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RDevtools(Package): + """Collection of package development tools.""" + + homepage = "https://github.com/hadley/devtools" + url = "https://cran.r-project.org/src/contrib/devtools_1.11.1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/devtools" + + version('1.11.1', '242672ee27d24dddcbdaac88c586b6c2') + + extends('R') + + depends_on('r-httr') + depends_on('r-memoise') + depends_on('r-whisker') + depends_on('r-digest') + depends_on('r-rstudioapi') + depends_on('r-jsonlite') + depends_on('r-git2r') + depends_on('r-withr') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-digest/package.py b/var/spack/repos/builtin/packages/r-digest/package.py new file mode 100644 index 0000000000..1d11afb139 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-digest/package.py @@ -0,0 +1,56 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RDigest(Package): + """Implementation of a function 'digest()' for the creation of hash digests + of arbitrary R objects (using the md5, sha-1, sha-256, crc32, xxhash and + murmurhash algorithms) permitting easy comparison of R language objects, as + well as a function 'hmac()' to create hash-based message authentication + code. The md5 algorithm by Ron Rivest is specified in RFC 1321, the sha-1 + and sha-256 algorithms are specified in FIPS-180-1 and FIPS-180-2, and the + crc32 algorithm is described in + ftp://ftp.rocksoft.com/cliens/rocksoft/papers/crc_v3.txt. For md5, sha-1, + sha-256 and aes, this package uses small standalone implementations that + were provided by Christophe Devine. For crc32, code from the zlib library + is used. For sha-512, an implementation by Aaron D. Gifford is used. For + xxhash, the implementation by Yann Collet is used. For murmurhash, an + implementation by Shane Day is used. Please note that this package is not + meant to be deployed for cryptographic purposes for which more + comprehensive (and widely tested) libraries such as OpenSSL should be + used.""" + + homepage = "http://dirk.eddelbuettel.com/code/digest.html" + url = "https://cran.r-project.org/src/contrib/digest_0.6.9.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/digest" + + version('0.6.9', '48048ce6c466bdb124716e45ba4a0e83') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py index fffae68019..a09b458eaf 100644 --- a/var/spack/repos/builtin/packages/r-filehash/package.py +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class RFilehash(Package): """Implements a simple key-value style database where character string keys are associated with data values that are stored on the disk. A simple @@ -37,10 +38,12 @@ class RFilehash(Package): homepage = 'https://cran.r-project.org/' url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/filehash" - version('2.3', '01fffafe09b148ccadc9814c103bdc2f', expand=False) + version('2.3', '01fffafe09b148ccadc9814c103bdc2f') extends('R') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-git2r/package.py b/var/spack/repos/builtin/packages/r-git2r/package.py new file mode 100644 index 0000000000..1f08379d6e --- /dev/null +++ b/var/spack/repos/builtin/packages/r-git2r/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RGit2r(Package): + """Interface to the 'libgit2' library, which is a pure C implementation of + the 'Git' core methods. Provides access to 'Git' repositories to extract + data and running some basic 'Git' commands.""" + + homepage = "https://github.com/ropensci/git2r" + url = "https://cran.r-project.org/src/contrib/git2r_0.15.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/git2r" + + version('0.15.0', '57658b3298f9b9aadc0dd77b4ef6a1e1') + + extends('R') + + depends_on('zlib') + depends_on('openssl') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-httr/package.py b/var/spack/repos/builtin/packages/r-httr/package.py new file mode 100644 index 0000000000..77ec34ab03 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-httr/package.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RHttr(Package): + """Useful tools for working with HTTP organised by HTTP verbs (GET(), + POST(), etc). Configuration functions make it easy to control additional + request components (authenticate(), add_headers() and so on).""" + + homepage = "https://github.com/hadley/httr" + url = "https://cran.r-project.org/src/contrib/httr_1.1.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/httr" + + version('1.1.0', '5ffbbc5c2529e49f00aaa521a2b35600') + + extends('R') + + depends_on('r-jsonlite') + depends_on('r-mime') + depends_on('r-curl') + depends_on('r-openssl') + depends_on('r-R6') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-jsonlite/package.py b/var/spack/repos/builtin/packages/r-jsonlite/package.py new file mode 100644 index 0000000000..6e231ed345 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-jsonlite/package.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RJsonlite(Package): + """A fast JSON parser and generator optimized for statistical data and the + web. Started out as a fork of 'RJSONIO', but has been completely rewritten + in recent versions. The package offers flexible, robust, high performance + tools for working with JSON in R and is particularly powerful for building + pipelines and interacting with a web API. The implementation is based on + the mapping described in the vignette (Ooms, 2014). In addition to + converting JSON data from/to R objects, 'jsonlite' contains functions to + stream, validate, and prettify JSON data. The unit tests included with the + package verify that all edge cases are encoded and decoded consistently for + use with dynamic data in systems and applications.""" + + homepage = "https://github.com/jeroenooms/jsonlite" + url = "https://cran.r-project.org/src/contrib/jsonlite_0.9.21.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/jsonlite" + + version('0.9.21', '4fc382747f88a79ff0718a0d06bed45d') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py index f86917ec0c..f6ba97740b 100644 --- a/var/spack/repos/builtin/packages/r-magic/package.py +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class RMagic(Package): """A collection of efficient, vectorized algorithms for the creation and investigation of magic squares and hypercubes, including a variety of @@ -32,12 +33,14 @@ class RMagic(Package): homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/magic" - version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029', expand=False) + version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029') extends('R') depends_on('r-abind') def install(self, spec, prefix): - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-memoise/package.py b/var/spack/repos/builtin/packages/r-memoise/package.py new file mode 100644 index 0000000000..6a0fb78650 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-memoise/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RMemoise(Package): + """Cache the results of a function so that when you call it again with the + same arguments it returns the pre-computed value.""" + + homepage = "https://github.com/hadley/memoise" + url = "https://cran.r-project.org/src/contrib/memoise_1.0.0.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/memoise" + + version('1.0.0', 'd31145292e2a88ae9a504cab1602e4ac') + + extends('R') + + depends_on('r-digest') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-mime/package.py b/var/spack/repos/builtin/packages/r-mime/package.py new file mode 100644 index 0000000000..fb079f44c5 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-mime/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RMime(Package): + """Guesses the MIME type from a filename extension using the data derived + from /etc/mime.types in UNIX-type systems.""" + + homepage = "https://github.com/yihui/mime" + url = "https://cran.r-project.org/src/contrib/mime_0.4.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/mime" + + version('0.4', '789cb33e41db2206c6fc7c3e9fbc2c02') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-openssl/package.py b/var/spack/repos/builtin/packages/r-openssl/package.py new file mode 100644 index 0000000000..3e77923d76 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-openssl/package.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class ROpenssl(Package): + """Bindings to OpenSSL libssl and libcrypto, plus custom SSH pubkey + parsers. Supports RSA, DSA and EC curves P-256, P-384 and P-521. + Cryptographic signatures can either be created and verified manually or via + x509 certificates. AES can be used in cbc, ctr or gcm mode for symmetric + encryption; RSA for asymmetric (public key) encryption or EC for Diffie + Hellman. High-level envelope functions combine RSA and AES for encrypting + arbitrary sized data. Other utilities include key generators, hash + functions (md5, sha1, sha256, etc), base64 encoder, a secure random number + generator, and 'bignum' math methods for manually performing crypto + calculations on large multibyte integers.""" + + homepage = "https://github.com/jeroenooms/openssl#readme" + url = "https://cran.r-project.org/src/contrib/openssl_0.9.4.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/openssl" + + version('0.9.4', '82a890e71ed0e74499878bedacfb8ccb') + + extends('R') + + depends_on('openssl') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-packrat/package.py b/var/spack/repos/builtin/packages/r-packrat/package.py new file mode 100644 index 0000000000..9520952bd7 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-packrat/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RPackrat(Package): + """Manage the R packages your project depends on in an isolated, portable, + and reproducible way.""" + + homepage = "https://github.com/rstudio/packrat/" + url = "https://cran.r-project.org/src/contrib/packrat_0.4.7-1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/packrat" + + version('0.4.7-1', '80c2413269b292ade163a70ba5053e84') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-rstudioapi/package.py b/var/spack/repos/builtin/packages/r-rstudioapi/package.py new file mode 100644 index 0000000000..0ef2d9b987 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-rstudioapi/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RRstudioapi(Package): + """Access the RStudio API (if available) and provide informative error + messages when it's not.""" + + homepage = "https://cran.r-project.org/web/packages/rstudioapi/index.html" + url = "https://cran.r-project.org/src/contrib/rstudioapi_0.5.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/rstudioapi" + + version('0.5', '6ce1191da74e7bcbf06b61339486b3ba') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-whisker/package.py b/var/spack/repos/builtin/packages/r-whisker/package.py new file mode 100644 index 0000000000..f338d150f1 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-whisker/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RWhisker(Package): + """logicless templating, reuse templates in many programming languages + including R""" + + homepage = "http://github.com/edwindj/whisker" + url = "https://cran.r-project.org/src/contrib/whisker_0.3-2.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/whisker" + + version('0.3-2', 'c4b9bf9a22e69ce003fe68663ab5e8e6') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/r-withr/package.py b/var/spack/repos/builtin/packages/r-withr/package.py new file mode 100644 index 0000000000..5ce7437c84 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-withr/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class RWithr(Package): + """A set of functions to run code 'with' safely and temporarily modified + global state. Many of these functions were originally a part of the + 'devtools' package, this provides a simple package with limited + dependencies to provide access to these functions.""" + + homepage = "http://github.com/jimhester/withr" + url = "https://cran.r-project.org/src/contrib/withr_1.0.1.tar.gz" + list_url = "https://cran.r-project.org/src/contrib/Archive/withr" + + version('1.0.1', 'ac38af2c6f74027c9592dd8f0acb7598') + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path) diff --git a/var/spack/repos/builtin/packages/rust-bindgen/package.py b/var/spack/repos/builtin/packages/rust-bindgen/package.py new file mode 100644 index 0000000000..854016d12a --- /dev/null +++ b/var/spack/repos/builtin/packages/rust-bindgen/package.py @@ -0,0 +1,18 @@ +from spack import * +import os + + +class RustBindgen(Package): + """The rust programming language toolchain""" + homepage = "http://www.rust-lang.org" + url = "https://github.com/crabtw/rust-bindgen" + + version('0.16', tag='0.16', git='https://github.com/crabtw/rust-bindgen') + + extends("rust") + depends_on("llvm") + + def install(self, spec, prefix): + env = dict(os.environ) + env['LIBCLANG_PATH'] = os.path.join(spec['llvm'].prefix, 'lib') + cargo('install', '--root', prefix, env=env) diff --git a/var/spack/repos/builtin/packages/rust/package.py b/var/spack/repos/builtin/packages/rust/package.py new file mode 100644 index 0000000000..65f81ce534 --- /dev/null +++ b/var/spack/repos/builtin/packages/rust/package.py @@ -0,0 +1,63 @@ +from spack import * +import os + + +def get_submodules(): + git = which('git') + git('submodule', 'update', '--init', '--recursive') + +class Rust(Package): + """The rust programming language toolchain""" + homepage = "http://www.rust-lang.org" + url = "https://github.com/rust-lang/rust" + + version('1.8.0', tag='1.8.0', git="https://github.com/rust-lang/rust") + + resource(name='cargo', + git="https://github.com/rust-lang/cargo.git", + tag='0.10.0', + destination='cargo') + + extendable = True + + # Rust + depends_on("llvm") + depends_on("curl") + depends_on("git") + depends_on("cmake") + depends_on("python@:2.8") + + # Cargo + depends_on("openssl") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--llvm-root=' + spec['llvm'].prefix) + + make() + make("install") + + # Install cargo, rust package manager + with working_dir(os.path.join('cargo', 'cargo')): + get_submodules() + configure('--prefix=' + prefix, + '--local-rust-root=' + prefix) + + make() + make("install") + + def setup_dependent_package(self, module, ext_spec): + """ + Called before python modules' install() methods. + + In most cases, extensions will only need to have one or two lines:: + + cargo('build') + cargo('install', '--root', prefix) + + or + + cargo('install', '--root', prefix) + """ + # Rust extension builds can have a global cargo executable function + module.cargo = Executable(join_path(self.spec.prefix.bin, 'cargo')) diff --git a/var/spack/repos/builtin/packages/scotch/Makefile.esmumps b/var/spack/repos/builtin/packages/scotch/Makefile.esmumps deleted file mode 100644 index 4bfc760197..0000000000 --- a/var/spack/repos/builtin/packages/scotch/Makefile.esmumps +++ /dev/null @@ -1,5 +0,0 @@ -esmumps : scotch - (cd esmumps ; $(MAKE) scotch && $(MAKE) install) - -ptesmumps : ptscotch - (cd esmumps ; $(MAKE) ptscotch && $(MAKE) ptinstall) diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py index e82c3acd42..3c2b4993ac 100644 --- a/var/spack/repos/builtin/packages/scotch/package.py +++ b/var/spack/repos/builtin/packages/scotch/package.py @@ -22,15 +22,16 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os from spack import * -import os, re + class Scotch(Package): """Scotch is a software package for graph and mesh/hypergraph partitioning, graph clustering, and sparse matrix ordering.""" homepage = "http://www.labri.fr/perso/pelegrin/scotch/" - url = "http://gforge.inria.fr/frs/download.php/latestfile/298/scotch_6.0.3.tar.gz" + url = "http://gforge.inria.fr/frs/download.php/latestfile/298/scotch_6.0.3.tar.gz" base_url = "http://gforge.inria.fr/frs/download.php/latestfile/298" list_url = "http://gforge.inria.fr/frs/?group_id=248" @@ -38,10 +39,11 @@ class Scotch(Package): version('6.0.0', 'c50d6187462ba801f9a82133ee666e8e') version('5.1.10b', 'f587201d6cf5cf63527182fbfba70753') - variant('mpi', default=False, description='Activate the compilation of PT-Scotch') + variant('mpi', default=False, description='Activate the compilation of parallel libraries') variant('compression', default=True, description='Activate the posibility to use compressed files') - variant('esmumps', default=False, description='Activate the compilation of the lib esmumps needed by mumps') - variant('shared', default=True, description='Build shared libraries') + variant('esmumps', default=False, description='Activate the compilation of esmumps needed by mumps') + variant('shared', default=True, description='Build a shared version of the library') + variant('metis', default=True, description='Build metis and parmetis wrapper libraries') depends_on('flex') depends_on('bison') @@ -51,43 +53,23 @@ class Scotch(Package): # NOTE: Versions of Scotch up to version 6.0.0 don't include support for # building with 'esmumps' in their default packages. In order to enable # support for this feature, we must grab the 'esmumps' enabled archives - # from the Scotch hosting site. These alternative archives include a strict + # from the Scotch hosting site. These alternative archives include a # superset of the behavior in their default counterparts, so we choose to # always grab these versions for older Scotch versions for simplicity. - @when('@:6.0.0') - def url_for_version(self, version): - return '%s/scotch_%s_esmumps.tar.gz' % (Scotch.base_url, version) - - @when('@6.0.1:') def url_for_version(self, version): return super(Scotch, self).url_for_version(version) - # NOTE: Several of the 'esmumps' enabled Scotch releases up to version 6.0.0 - # have broken build scripts that don't properly build 'esmumps' as a separate - # target, so we need a patch procedure to remove 'esmumps' from existing targets - # and to add it as a standalone target. @when('@:6.0.0') - def patch(self): - makefile_path = os.path.join('src', 'Makefile') - with open(makefile_path, 'r') as makefile: - esmumps_enabled = any(re.search(r'^esmumps(\s*):(.*)$', line) for line in makefile.readlines()) - - if not esmumps_enabled: - mff = FileFilter(makefile_path) - mff.filter(r'^.*((esmumps)|(ptesmumps)).*(install).*$', '') - - makefile_esmumps_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Makefile.esmumps') - with open(makefile_path, 'a') as makefile: - makefile.write('\ninclude %s\n' % makefile_esmumps_path) + def url_for_version(self, version): + return '%s/scotch_%s_esmumps.tar.gz' % (Scotch.base_url, version) - @when('@6.0.1:') def patch(self): - pass + self.configure() - # NOTE: Configuration of Scotch is achieved by writing a 'Makefile.inc' file - # that contains all of the configuration variables and their desired values - # for the installation. This function writes this file based on the given - # installation variants. + # NOTE: Configuration of Scotch is achieved by writing a 'Makefile.inc' + # file that contains all of the configuration variables and their desired + # values for the installation. This function writes this file based on + # the given installation variants. def configure(self): makefile_inc = [] cflags = [ @@ -96,41 +78,41 @@ class Scotch(Package): '-DSCOTCH_DETERMINISTIC', '-DSCOTCH_RENAME', '-DIDXSIZE64' - ] - - ## Library Build Type ## + ] + # Library Build Type # if '+shared' in self.spec: makefile_inc.extend([ + # todo change for Darwin systems 'LIB = .so', 'CLIBFLAGS = -shared -fPIC', 'RANLIB = echo', - 'AR = $(CC)', + 'AR = $(CC)', 'ARFLAGS = -shared $(LDFLAGS) -o' - ]) + ]) cflags.append('-fPIC') else: makefile_inc.extend([ 'LIB = .a', 'CLIBFLAGS = ', 'RANLIB = ranlib', - 'AR = ar', + 'AR = ar', 'ARFLAGS = -ruv ' - ]) + ]) - ## Compiler-Specific Options ## + # Compiler-Specific Options # if self.compiler.name == 'gcc': cflags.append('-Drestrict=__restrict') elif self.compiler.name == 'intel': cflags.append('-restrict') + mpicc_path = self.spec['mpi'].mpicc if '+mpi' in self.spec else 'mpicc' makefile_inc.append('CCS = $(CC)') - makefile_inc.append('CCP = %s' % - (self.spec['mpi'].mpicc if '+mpi' in self.spec else 'mpicc')) + makefile_inc.append('CCP = %s' % mpicc_path) makefile_inc.append('CCD = $(CCS)') - ## Extra Features ## + # Extra Features # ldflags = [] @@ -143,8 +125,10 @@ class Scotch(Package): makefile_inc.append('LDFLAGS = %s' % ' '.join(ldflags)) - ## General Features ## + # General Features # + flex_path = os.path.join(self.spec['flex'].prefix.bin, 'flex') + bison_path = os.path.join(self.spec['bison'].prefix.bin, 'bison') makefile_inc.extend([ 'EXE =', 'OBJ = .o', @@ -155,30 +139,59 @@ class Scotch(Package): 'MV = mv', 'CP = cp', 'CFLAGS = %s' % ' '.join(cflags), - 'LEX = %s -Pscotchyy -olex.yy.c' % os.path.join(self.spec['flex'].prefix.bin , 'flex'), - 'YACC = %s -pscotchyy -y -b y' % os.path.join(self.spec['bison'].prefix.bin, 'bison'), + 'LEX = %s -Pscotchyy -olex.yy.c' % flex_path, + 'YACC = %s -pscotchyy -y -b y' % bison_path, 'prefix = %s' % self.prefix - ]) + ]) with working_dir('src'): with open('Makefile.inc', 'w') as fh: fh.write('\n'.join(makefile_inc)) def install(self, spec, prefix): - self.configure() - targets = ['scotch'] if '+mpi' in self.spec: targets.append('ptscotch') - if '+esmumps' in self.spec: - targets.append('esmumps') - if '+mpi' in self.spec: - targets.append('ptesmumps') + if self.spec.version >= Version('6.0.0'): + if '+esmumps' in self.spec: + targets.append('esmumps') + if '+mpi' in self.spec: + targets.append('ptesmumps') with working_dir('src'): for target in targets: - make(target, parallel=(target!='ptesmumps')) + # It seams that building ptesmumps in parallel fails, for + # version prior to 6.0.0 there is no separated targets force + # ptesmumps, this library is built by the ptscotch target. This + # should explain the test for the can_make_parallel variable + can_make_parallel = \ + not (target == 'ptesmumps' or + (self.spec.version < Version('6.0.0') and + target == 'ptscotch')) + make(target, parallel=can_make_parallel) + + # todo change this to take into account darwin systems + lib_ext = '.so' if '+shared' in self.spec else '.a' + # It seams easier to remove metis wrappers from the folder that will be + # installed than to tweak the Makefiles + if '+metis' not in self.spec: + with working_dir('lib'): + lib_ext = '.so' if '+shared' in self.spec else '.a' + force_remove('libscotchmetis{0}'.format(lib_ext)) + force_remove('libptscotchparmetis{0}'.format(lib_ext)) + + with working_dir('include'): + force_remove('metis.h') + force_remove('parmetis.h') + + if '~esmumps' in self.spec and self.spec.version < Version('6.0.0'): + with working_dir('lib'): + force_remove('libesmumps{0}'.format(lib_ext)) + force_remove('libptesmumps{0}'.format(lib_ext)) + + with working_dir('include'): + force_remove('esmumps.h') install_tree('bin', prefix.bin) install_tree('lib', prefix.lib) diff --git a/var/spack/repos/builtin/packages/sed/package.py b/var/spack/repos/builtin/packages/sed/package.py new file mode 100644 index 0000000000..f2a240e1b3 --- /dev/null +++ b/var/spack/repos/builtin/packages/sed/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Sed(Package): + """GNU implementation of the famous stream editor.""" + homepage = "http://www.gnu.org/software/sed/" + url = "http://ftpmirror.gnu.org/sed/sed-4.2.2.tar.bz2" + + version('4.2.2', '7ffe1c7cdc3233e1e0c4b502df253974') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 3b1d08889c..817db68241 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -24,8 +24,10 @@ ############################################################################## from spack import * + class Serf(Package): - """Apache Serf - a high performance C-based HTTP client library built upon the Apache Portable Runtime (APR) library""" + """Apache Serf - a high performance C-based HTTP client library + built upon the Apache Portable Runtime (APR) library""" homepage = 'https://serf.apache.org/' url = 'https://archive.apache.org/dist/serf/serf-1.3.8.tar.bz2' @@ -36,6 +38,7 @@ class Serf(Package): depends_on('scons') depends_on('expat') depends_on('openssl') + depends_on('zlib') def install(self, spec, prefix): scons = which("scons") @@ -44,8 +47,10 @@ class Serf(Package): options.append('APR=%s' % spec['apr'].prefix) options.append('APU=%s' % spec['apr-util'].prefix) options.append('OPENSSL=%s' % spec['openssl'].prefix) - options.append('LINKFLAGS=-L%s/lib' % spec['expat'].prefix) - options.append('CPPFLAGS=-I%s/include' % spec['expat'].prefix) + options.append('LINKFLAGS=-L%s/lib -L%s/lib' % + (spec['expat'].prefix, spec['zlib'].prefix)) + options.append('CPPFLAGS=-I%s/include -I%s/include' % + (spec['expat'].prefix, spec['zlib'].prefix)) scons(*options) scons('install') diff --git a/var/spack/repos/builtin/packages/slepc/package.py b/var/spack/repos/builtin/packages/slepc/package.py index c148a579ec..d3739bf6a6 100644 --- a/var/spack/repos/builtin/packages/slepc/package.py +++ b/var/spack/repos/builtin/packages/slepc/package.py @@ -34,13 +34,16 @@ class Slepc(Package): homepage = "http://www.grycap.upv.es/slepc" url = "http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz" + version('3.7.1', '670216f263e3074b21e0623c01bc0f562fdc0bffcd7bd42dd5d8edbe73a532c2') + version('3.6.3', '384939d009546db37bc05ed81260c8b5ba451093bf891391d32eb7109ccff876') version('3.6.2', '2ab4311bed26ccf7771818665991b2ea3a9b15f97e29fd13911ab1293e8e65df') - variant('arpack', default=False, description='Enables Arpack wrappers') + variant('arpack', default=True, description='Enables Arpack wrappers') - depends_on('petsc') - depends_on('arpack-ng~mpi',when='+arpack^petsc~mpi') - depends_on('arpack-ng+mpi',when='+arpack^petsc+mpi') + depends_on('petsc@3.7:', when='@3.7.1:') + depends_on('petsc@3.6.3:3.6.4', when='@3.6.2:3.6.3') + depends_on('arpack-ng~mpi', when='+arpack^petsc~mpi') + depends_on('arpack-ng+mpi', when='+arpack^petsc+mpi') def install(self, spec, prefix): # set SLEPC_DIR for installation @@ -64,7 +67,7 @@ class Slepc(Package): configure('--prefix=%s' % prefix, *options) make('MAKE_NP=%s' % make_jobs, parallel=False) - #FIXME: + # FIXME: # make('test') make('install') diff --git a/var/spack/repos/builtin/packages/stream/package.py b/var/spack/repos/builtin/packages/stream/package.py new file mode 100644 index 0000000000..8b3f32af8a --- /dev/null +++ b/var/spack/repos/builtin/packages/stream/package.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Stream(Package): + """The STREAM benchmark is a simple synthetic benchmark program that + measures sustainable memory bandwidth (in MB/s) and the corresponding + computation rate for simple vector kernels.""" + + homepage = "https://www.cs.virginia.edu/stream/ref.html" + + version('5.10', git='https://github.com/jeffhammond/STREAM.git') + + variant('openmp', default=False, description='Build with OpenMP support') + + def patch(self): + makefile = FileFilter('Makefile') + + # Use the Spack compiler wrappers + makefile.filter('CC = .*', 'CC = cc') + makefile.filter('FC = .*', 'FC = f77') + + cflags = '-O2' + fflags = '-O2' + if '+openmp' in self.spec: + cflags += ' ' + self.compiler.openmp_flag + fflags += ' ' + self.compiler.openmp_flag + + # Set the appropriate flags for this compiler + makefile.filter('CFLAGS = .*', 'CFLAGS = {0}'.format(cflags)) + makefile.filter('FFLAGS = .*', 'FFLAGS = {0}'.format(fflags)) + + def install(self, spec, prefix): + make() + + # Manual installation + mkdir(prefix.bin) + install('stream_c.exe', prefix.bin) + install('stream_f.exe', prefix.bin) diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index 6c06b5497c..e51d7224d9 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -25,50 +25,54 @@ from spack import * import glob + class SuperluDist(Package): - """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines.""" + """A general purpose library for the direct solution of large, sparse, + nonsymmetric systems of linear equations on high performance machines.""" homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/" url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz" version('5.0.0', '2b53baf1b0ddbd9fcf724992577f0670') - # default to version 4.3 since petsc and trilinos are not tested with 5.0. - version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae', preferred=True) + version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae') version('4.2', 'ae9fafae161f775fbac6eba11e530a65') version('4.1', '4edee38cc29f687bd0c8eb361096a455') version('4.0', 'c0b98b611df227ae050bc1635c6940e0') - depends_on ('mpi') - depends_on ('blas') - depends_on ('lapack') - depends_on ('parmetis') - depends_on ('metis@5:') + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + depends_on('parmetis') + depends_on('metis@5:') def install(self, spec, prefix): makefile_inc = [] makefile_inc.extend([ 'PLAT = _mac_x', - 'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix + 'DSuperLUroot = %s' % self.stage.source_path, 'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a', 'BLASDEF = -DUSE_VENDOR_BLAS', - 'BLASLIB = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names + 'BLASLIB = %s %s' % + (to_link_flags(spec['lapack'].lapack_shared_lib), + to_link_flags(spec['blas'].blas_shared_lib)), 'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib, 'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib, 'FLIBS =', - 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', + 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', # NOQA: ignore=E501 'ARCH = ar', 'ARCHFLAGS = cr', 'RANLIB = true', - 'CC = mpicc', # FIXME avoid hardcoding MPI compiler names - 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include), + 'CC = %s' % spec['mpi'].mpicc, + 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' % + (spec['parmetis'].prefix.include, + spec['metis'].prefix.include), 'NOOPTS = -fPIC -std=c99', - 'FORTRAN = mpif77', + 'FORTRAN = %s' % spec['mpi'].mpif77, 'F90FLAGS = -O2', - 'LOADER = mpif77', + 'LOADER = %s' % spec['mpi'].mpif77, 'LOADOPTS =', 'CDEFS = -DAdd_' - ]) + ]) - #with working_dir('src'): with open('make.inc', 'w') as fh: fh.write('\n'.join(makefile_inc)) @@ -83,9 +87,10 @@ class SuperluDist(Package): mkdirp(headers_location) mkdirp(prefix.lib) - headers = glob.glob(join_path(self.stage.source_path, 'SRC','*.h')) + headers = glob.glob(join_path(self.stage.source_path, 'SRC', '*.h')) for h in headers: - install(h,headers_location) + install(h, headers_location) - superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a') - install(superludist_lib,self.prefix.lib) + superludist_lib = join_path(self.stage.source_path, + 'lib/libsuperlu_dist.a') + install(superludist_lib, self.prefix.lib) diff --git a/var/spack/repos/builtin/packages/superlu/package.py b/var/spack/repos/builtin/packages/superlu/package.py new file mode 100644 index 0000000000..c634c1d1ba --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu/package.py @@ -0,0 +1,54 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Superlu(Package): + """SuperLU is a general purpose library for the direct solution of large, + sparse, nonsymmetric systems of linear equations on high performance + machines. SuperLU is designed for sequential machines.""" + + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/#superlu" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_5.2.1.tar.gz" + + version('5.2.1', '3a1a9bff20cb06b7d97c46d337504447') + + depends_on('blas') + + def install(self, spec, prefix): + cmake_args = [ + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', + # BLAS support + '-Denable_blaslib=OFF', + '-DBLAS_blas_LIBRARY={0}'.format(spec['blas'].blas_shared_lib) + ] + + cmake_args.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *cmake_args) + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/tbb/package.py b/var/spack/repos/builtin/packages/tbb/package.py index 6c3ceb1e76..c88b170816 100644 --- a/var/spack/repos/builtin/packages/tbb/package.py +++ b/var/spack/repos/builtin/packages/tbb/package.py @@ -23,9 +23,9 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os import glob + class Tbb(Package): """Widely used C++ template library for task parallelism. Intel Threading Building Blocks (Intel TBB) lets you easily write parallel @@ -35,35 +35,39 @@ class Tbb(Package): homepage = "http://www.threadingbuildingblocks.org/" # Only version-specific URL's work for TBB - version('4.4.3', '80707e277f69d9b20eeebdd7a5f5331137868ce1', url='https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160128oss_src_0.tgz') + version('4.4.4', 'd4cee5e4ca75cab5181834877738619c56afeb71', url='https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz') # NOQA: ignore=E501 + version('4.4.3', '80707e277f69d9b20eeebdd7a5f5331137868ce1', url='https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160128oss_src_0.tgz') # NOQA: ignore=E501 - def coerce_to_spack(self,tbb_build_subdir): - for compiler in ["icc","gcc","clang"]: - fs = glob.glob(join_path(tbb_build_subdir,"*.%s.inc" % compiler )) - for f in fs: - lines = open(f).readlines() - of = open(f,"w") - for l in lines: - if l.strip().startswith("CPLUS ="): + def coerce_to_spack(self, tbb_build_subdir): + for compiler in ["icc", "gcc", "clang"]: + fs = glob.glob(join_path(tbb_build_subdir, + "*.%s.inc" % compiler)) + for f in fs: + lines = open(f).readlines() + of = open(f, "w") + for l in lines: + if l.strip().startswith("CPLUS ="): of.write("# coerced to spack\n") of.write("CPLUS = $(CXX)\n") - elif l.strip().startswith("CPLUS ="): + elif l.strip().startswith("CPLUS ="): of.write("# coerced to spack\n") of.write("CONLY = $(CC)\n") - else: - of.write(l); + else: + of.write(l) def install(self, spec, prefix): - # - # we need to follow TBB's compiler selection logic to get the proper build + link flags - # but we still need to use spack's compiler wrappers + if spec.satisfies('%gcc@6.1:') and spec.satisfies('@:4.4.3'): + raise InstallError('Only TBB 4.4.4 and above build with GCC 6.1!') + + # We need to follow TBB's compiler selection logic to get the proper + # build + link flags but we still need to use spack's compiler wrappers # to accomplish this, we do two things: # - # * Look at the spack spec to determine which compiler we should pass to tbb's Makefile + # * Look at the spack spec to determine which compiler we should pass + # to tbb's Makefile; # # * patch tbb's build system to use the compiler wrappers (CC, CXX) for - # icc, gcc, clang - # (see coerce_to_spack()) + # icc, gcc, clang (see coerce_to_spack()); # self.coerce_to_spack("build") @@ -74,7 +78,6 @@ class Tbb(Package): else: tbb_compiler = "gcc" - mkdirp(prefix) mkdirp(prefix.lib) @@ -82,10 +85,10 @@ class Tbb(Package): # tbb does not have a configure script or make install target # we simply call make, and try to put the pieces together # - make("compiler=%s" %(tbb_compiler)) + make("compiler=%s" % (tbb_compiler)) # install headers to {prefix}/include - install_tree('include',prefix.include) + install_tree('include', prefix.include) # install libs to {prefix}/lib tbb_lib_names = ["libtbb", @@ -94,10 +97,10 @@ class Tbb(Package): for lib_name in tbb_lib_names: # install release libs - fs = glob.glob(join_path("build","*release",lib_name + ".*")) + fs = glob.glob(join_path("build", "*release", lib_name + ".*")) for f in fs: install(f, prefix.lib) # install debug libs if they exist - fs = glob.glob(join_path("build","*debug",lib_name + "_debug.*")) + fs = glob.glob(join_path("build", "*debug", lib_name + "_debug.*")) for f in fs: install(f, prefix.lib) diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py index 5e87ed7fba..c301a5b4e5 100644 --- a/var/spack/repos/builtin/packages/tetgen/package.py +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -24,16 +24,19 @@ ############################################################################## from spack import * + class Tetgen(Package): - """TetGen is a program and library that can be used to generate tetrahedral - meshes for given 3D polyhedral domains. TetGen generates exact constrained - Delaunay tetrahedralizations, boundary conforming Delaunay meshes, and - Voronoi paritions.""" + """TetGen is a program and library that can be used to generate + tetrahedral meshes for given 3D polyhedral domains. TetGen + generates exact constrained Delaunay tetrahedralizations, + boundary conforming Delaunay meshes, and Voronoi paritions. + """ homepage = "http://www.tetgen.org" url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" version('1.4.3', 'd6a4bcdde2ac804f7ec66c29dcb63c18') + version('1.5.0', '3b9fd9cdec121e52527b0308f7aad5c1', url='http://www.tetgen.org/1.5/src/tetgen1.5.0.tar.gz') # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). diff --git a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py new file mode 100644 index 0000000000..9c9a66cdef --- /dev/null +++ b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py @@ -0,0 +1,21 @@ +from spack import * +import os +import shutil + + +class ThePlatinumSearcher(Package): + """Fast parallel recursive grep alternative""" + homepage = "https://github.com/monochromegane/the_platinum_searcher" + url = "https://github.com/monochromegane/the_platinum_searcher" + + package = 'github.com/monochromegane/the_platinum_searcher/...' + + version('head', go=package) + + extends("go") + + def install(self, spec, prefix): + env = os.environ + env['GOPATH'] = self.stage.source_path + ':' + env['GOPATH'] + go('install', self.package, env=env) + shutil.copytree('bin', os.path.join(prefix, 'bin')) diff --git a/var/spack/repos/builtin/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py index c46425c0d3..573ee38a79 100644 --- a/var/spack/repos/builtin/packages/tmux/package.py +++ b/var/spack/repos/builtin/packages/tmux/package.py @@ -24,26 +24,33 @@ ############################################################################## from spack import * + class Tmux(Package): """tmux is a terminal multiplexer. What is a terminal multiplexer? It lets - you switch easily between several programs in one terminal, detach them (they - keep running in the background) and reattach them to a different terminal. And - do a lot more. + you switch easily between several programs in one terminal, detach them + (they keep running in the background) and reattach them to a different + terminal. And do a lot more. """ homepage = "http://tmux.github.io" - url = "https://github.com/tmux/tmux/releases/download/2.1/tmux-2.1.tar.gz" + url = "https://github.com/tmux/tmux/releases/download/2.2/tmux-2.2.tar.gz" version('1.9a', 'b07601711f96f1d260b390513b509a2d') version('2.1', '74a2855695bccb51b6e301383ad4818c') + version('2.2', 'bd95ee7205e489c62c616bb7af040099') depends_on('libevent') depends_on('ncurses') def install(self, spec, prefix): + pkg_config_path = ':'.join([ + spec['libevent'].prefix, + spec['ncurses'].prefix + ]) + configure( "--prefix=%s" % prefix, - "PKG_CONFIG_PATH=%s:%s" % (spec['libevent'].prefix, spec['ncurses'].prefix)) + "PKG_CONFIG_PATH=%s" % pkg_config_path) make() make("install") diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 1eaec86405..3a53ac5c01 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -23,23 +23,30 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, sys, glob +import os +import sys -# Trilinos is complicated to build, as an inspiration a couple of links to other repositories which build it: +# Trilinos is complicated to build, as an inspiration a couple of links to +# other repositories which build it: # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 # https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package # https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh -# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb -# and some relevant documentation/examples: +# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb and some +# relevant documentation/examples: # https://github.com/trilinos/Trilinos/issues/175 + + class Trilinos(Package): - """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented - software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. + """The Trilinos Project is an effort to develop algorithms and enabling + technologies within an object-oriented software framework for the solution + of large-scale, complex multi-physics engineering and scientific problems. A unique design feature of Trilinos is its focus on packages. """ homepage = "https://trilinos.org/" url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + version('12.6.3', '960f5f4d3f7c3da818e5a5fb4684559eff7e0c25f959ef576561b8a52f0e4d1e') + version('12.6.2', '0c076090508170ddee5efeed317745027f9418319720dc40a072e478775279f9') version('12.6.1', 'adcf2d3aab74cdda98f88fee19cd1442604199b0515ee3da4d80cbe8f37d00e4') version('12.4.2', '7c830f7f0f68b8ad324690603baf404e') version('12.2.1', '6161926ea247863c690e927687f83be9') @@ -54,49 +61,52 @@ class Trilinos(Package): variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') - # not everyone has py-numpy activated, keep it disabled by default to avoid configure errors + # not everyone has py-numpy activated, keep it disabled by default to avoid + # configure errors variant('python', default=False, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') + variant('boost', default=True, description='Compile with Boost') # Everything should be compiled with -fpic depends_on('blas') depends_on('lapack') - depends_on('boost') + depends_on('boost', when='+boost') depends_on('matio') depends_on('glm') depends_on('swig') - depends_on('metis@5:',when='+metis') - depends_on('suite-sparse',when='+suite-sparse') + depends_on('metis@5:', when='+metis') + depends_on('suite-sparse', when='+suite-sparse') # MPI related dependencies depends_on('mpi') depends_on('netcdf+mpi') - depends_on('parmetis',when='+metis') - # Trilinos' Tribits config system is limited which makes it - # very tricky to link Amesos with static MUMPS, see + depends_on('parmetis', when='+metis') + # Trilinos' Tribits config system is limited which makes it very tricky to + # link Amesos with static MUMPS, see # https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html - # One could work it out by getting linking flags from mpif90 --showme:link (or alike) - # and adding results to -DTrilinos_EXTRA_LINK_FLAGS - # together with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and - # it may work at the end. But let's avoid all this by simply using shared libs - depends_on('mumps@5.0:+mpi+shared',when='+mumps') - depends_on('scalapack',when='+mumps') - depends_on('superlu-dist',when='+superlu-dist') - depends_on('hypre~internal-superlu',when='+hypre') - depends_on('hdf5+mpi',when='+hdf5') - - depends_on('python',when='+python') + # One could work it out by getting linking flags from mpif90 --showme:link + # (or alike) and adding results to -DTrilinos_EXTRA_LINK_FLAGS together + # with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and it may + # work at the end. But let's avoid all this by simply using shared libs + depends_on('mumps@5.0:+mpi+shared', when='+mumps') + depends_on('scalapack', when='+mumps') + depends_on('superlu-dist@:4.3', when='@:12.6.1+superlu-dist') + depends_on('superlu-dist', when='@12.6.2:+superlu-dist') + depends_on('hypre~internal-superlu', when='+hypre') + depends_on('hdf5+mpi', when='+hdf5') + depends_on('python', when='+python') patch('umfpack_from_suitesparse.patch') # check that the combination of variants makes sense def variants_check(self): if '+superlu-dist' in self.spec and self.spec.satisfies('@:11.4.3'): - # For Trilinos v11 we need to force SuperLUDist=OFF, - # since only the deprecated SuperLUDist v3.3 together with an Amesos patch - # is working. - raise RuntimeError('The superlu-dist variant can only be used with Trilinos @12.0.1:') + # For Trilinos v11 we need to force SuperLUDist=OFF, since only the + # deprecated SuperLUDist v3.3 together with an Amesos patch is + # working. + raise RuntimeError('The superlu-dist variant can only be used' + + ' with Trilinos @12.0.1:') def install(self, spec, prefix): self.variants_check() @@ -106,54 +116,75 @@ class Trilinos(Package): options.extend(std_cmake_args) mpi_bin = spec['mpi'].prefix.bin - options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', - '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DCMAKE_BUILD_TYPE:STRING=%s' % ('DEBUG' if '+debug' in spec else 'RELEASE'), - '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), - '-DTPL_ENABLE_MPI:BOOL=ON', - '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, - '-DTPL_ENABLE_BLAS=ON', - '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names - '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, - '-DTPL_ENABLE_LAPACK=ON', - '-DLAPACK_LIBRARY_NAMES=lapack', - '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, - '-DTPL_ENABLE_Boost:BOOL=ON', - '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, - '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', - '-DTrilinos_ENABLE_CXX11:BOOL=ON', - '-DTPL_ENABLE_Netcdf:BOOL=ON', - '-DTPL_ENABLE_HYPRE:BOOL=%s' % ('ON' if '+hypre' in spec else 'OFF'), - '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), - ]) + options.extend([ + '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', + '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DCMAKE_BUILD_TYPE:STRING=%s' % ( + 'DEBUG' if '+debug' in spec else 'RELEASE'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ( + 'ON' if '+shared' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:BOOL=ON', + '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, + '-DTPL_ENABLE_BLAS=ON', + '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names + '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, + '-DTPL_ENABLE_LAPACK=ON', + '-DLAPACK_LIBRARY_NAMES=lapack', + '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, + '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', + '-DTrilinos_ENABLE_CXX11:BOOL=ON', + '-DTPL_ENABLE_Netcdf:BOOL=ON', + '-DTPL_ENABLE_HYPRE:BOOL=%s' % ( + 'ON' if '+hypre' in spec else 'OFF'), + '-DTPL_ENABLE_HDF5:BOOL=%s' % ( + 'ON' if '+hdf5' in spec else 'OFF'), + ]) + + if '+boost' in spec: + options.extend([ + '-DTPL_ENABLE_Boost:BOOL=ON', + '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib + ]) + else: + options.extend(['-DTPL_ENABLE_Boost:BOOL=OFF']) # Fortran lib - libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) + libgfortran = os.path.dirname(os.popen( + '%s --print-file-name libgfortran.a' % + join_path(mpi_bin, 'mpif90')).read()) options.extend([ - '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, + '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % ( + libgfortran), '-DTrilinos_ENABLE_Fortran=ON' ]) # for build-debug only: - #options.extend([ - # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' - #]) + # options.extend([ + # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + # ]) # suite-sparse related if '+suite-sparse' in spec: options.extend([ - '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? - #'-DTPL_ENABLE_Cholmod:BOOL=ON', - #'-DCholmod_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - #'-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + # FIXME: Trilinos seems to be looking for static libs only, + # patch CMake TPL file? + '-DTPL_ENABLE_Cholmod:BOOL=OFF', + # '-DTPL_ENABLE_Cholmod:BOOL=ON', + # '-DCholmod_LIBRARY_DIRS:PATH=%s' % ( + # spec['suite-sparse'].prefix.lib, + # '-DCholmod_INCLUDE_DIRS:PATH=%s' % ( + # spec['suite-sparse'].prefix.include, '-DTPL_ENABLE_UMFPACK:BOOL=ON', - '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, - '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' + '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % ( + spec['suite-sparse'].prefix.lib), + '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % ( + spec['suite-sparse'].prefix.include), + '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;' + + 'suitesparseconfig' ]) else: options.extend([ @@ -169,9 +200,11 @@ class Trilinos(Package): '-DMETIS_LIBRARY_NAMES=metis', '-DTPL_METIS_INCLUDE_DIRS=%s' % spec['metis'].prefix.include, '-DTPL_ENABLE_ParMETIS:BOOL=ON', - '-DParMETIS_LIBRARY_DIRS=%s;%s' % (spec['parmetis'].prefix.lib,spec['metis'].prefix.lib), + '-DParMETIS_LIBRARY_DIRS=%s;%s' % ( + spec['parmetis'].prefix.lib, spec['metis'].prefix.lib), '-DParMETIS_LIBRARY_NAMES=parmetis;metis', - '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include + '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % ( + spec['parmetis'].prefix.include) ]) else: options.extend([ @@ -184,11 +217,14 @@ class Trilinos(Package): options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, - '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', # order is important! + # order is important! + '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', '-DTPL_ENABLE_SCALAPACK:BOOL=ON', - '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + '-DSCALAPACK_LIBRARY_NAMES=scalapack' ]) - # see https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS + # see + # https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS cxx_flags.extend([ '-DMUMPS_5_0' ]) @@ -201,16 +237,20 @@ class Trilinos(Package): # superlu-dist: if '+superlu-dist' in spec: # Amesos, conflicting types of double and complex SLU_D - # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html - # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html + # see + # https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html + # and + # https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html options.extend([ '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' ]) options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', - '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, - '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include + '-DSuperLUDist_LIBRARY_DIRS=%s' % + spec['superlu-dist'].prefix.lib, + '-DSuperLUDist_INCLUDE_DIRS=%s' % + spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): options.extend([ @@ -221,7 +261,6 @@ class Trilinos(Package): '-DTPL_ENABLE_SuperLUDist:BOOL=OFF', ]) - # python if '+python' in spec: options.extend([ @@ -248,23 +287,26 @@ class Trilinos(Package): '-DTrilinos_ENABLE_FEI=OFF' ]) - with working_dir('spack-build', create=True): cmake('..', *options) make() make('install') - # When trilinos is built with Python, libpytrilinos is included through - # cmake configure files. Namely, Trilinos_LIBRARIES in TrilinosConfig.cmake - # contains pytrilinos. This leads to a run-time error: - # Symbol not found: _PyBool_Type - # and prevents Trilinos to be used in any C++ code, which links executable - # against the libraries listed in Trilinos_LIBRARIES. - # See https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 + # When trilinos is built with Python, libpytrilinos is included + # through cmake configure files. Namely, Trilinos_LIBRARIES in + # TrilinosConfig.cmake contains pytrilinos. This leads to a + # run-time error: Symbol not found: _PyBool_Type and prevents + # Trilinos to be used in any C++ code, which links executable + # against the libraries listed in Trilinos_LIBRARIES. See + # https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 # A workaround it to remove PyTrilinos from the COMPONENTS_LIST : if '+python' in self.spec: - filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', (r'\1\3'), '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) + filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', + (r'\1\3'), + '%s/cmake/Trilinos/TrilinosConfig.cmake' % + prefix.lib) - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly on Darwin; + # correct this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/turbomole/package.py b/var/spack/repos/builtin/packages/turbomole/package.py index 3362113d13..6ccce23f97 100644 --- a/var/spack/repos/builtin/packages/turbomole/package.py +++ b/var/spack/repos/builtin/packages/turbomole/package.py @@ -26,21 +26,22 @@ from spack import * import os import subprocess + class Turbomole(Package): """TURBOMOLE: Program Package for ab initio Electronic Structure Calculations. NB: Requires a license to download.""" - - # NOTE: Turbomole requires purchase of a license to download. Go to the - # NOTE: Turbomole home page, http://www.turbomole-gmbh.com, for details. - # NOTE: Spack will search the current directory for this file. It is - # NOTE: probably best to add this file to a Spack mirror so that it can be - # NOTE: found from anywhere. For information on setting up a Spack mirror - # NOTE: see http://software.llnl.gov/spack/mirrors.html + + # NOTE: Turbomole requires purchase of a license to download. Go to the + # NOTE: Turbomole home page, http://www.turbomole-gmbh.com, for details. + # NOTE: Spack will search the current directory for this file. It is + # NOTE: probably best to add this file to a Spack mirror so that it can be + # NOTE: found from anywhere. For information on setting up a Spack mirror + # NOTE: see http://software.llnl.gov/spack/mirrors.html homepage = "http://www.turbomole-gmbh.com/" version('7.0.2', '92b97e1e52e8dcf02a4d9ac0147c09d6', - url="file://%s/turbolinux702.tar.gz" % os.getcwd()) + url="file://%s/turbolinux702.tar.gz" % os.getcwd()) variant('mpi', default=False, description='Set up MPI environment') variant('smp', default=False, description='Set up SMP environment') @@ -56,33 +57,31 @@ class Turbomole(Package): def do_fetch(self, mirror_only=True): if '+mpi' in self.spec and '+smp' in self.spec: - raise InstallError('Can not have both SMP and MPI enabled in the same build.') + raise InstallError('Can not have both SMP and MPI enabled in the ' + 'same build.') super(Turbomole, self).do_fetch(mirror_only) def get_tm_arch(self): - # For python-2.7 we could use `tm_arch = subprocess.check_output()` - # Use the following for compatibility with python 2.6 if 'TURBOMOLE' in os.getcwd(): - tm_arch = subprocess.Popen(['sh', 'scripts/sysname'], - stdout=subprocess.PIPE).communicate()[0] + tm_sysname = Executable('./scripts/sysname') + tm_arch = tm_sysname(output=str) return tm_arch.rstrip('\n') else: return - + def install(self, spec, prefix): if spec.satisfies('@:7.0.2'): calculate_version = 'calculate_2.4_linux64' molecontrol_version = 'MoleControl_2.5' - tm_arch=self.get_tm_arch() + tm_arch = self.get_tm_arch() tar = which('tar') dst = join_path(prefix, 'TURBOMOLE') tar('-x', '-z', '-f', 'thermocalc.tar.gz') with working_dir('thermocalc'): - cmd = 'sh install <<<y' - subprocess.call(cmd, shell=True) + subprocess.call('./install<<<y', shell=True) install_tree('basen', join_path(dst, 'basen')) install_tree('cabasen', join_path(dst, 'cabasen')) @@ -108,13 +107,19 @@ class Turbomole(Package): install('TURBOMOLE_702_LinuxPC', dst) if '+mpi' in spec: - install_tree('bin/%s_mpi' % tm_arch, join_path(dst, 'bin', '%s_mpi' % tm_arch)) - install_tree('libso/%s_mpi' % tm_arch, join_path(dst, 'libso', '%s_mpi' % tm_arch)) - install_tree('mpirun_scripts/%s_mpi' % tm_arch, join_path(dst, 'mpirun_scripts', '%s_mpi' % tm_arch)) + install_tree('bin/%s_mpi' % tm_arch, + join_path(dst, 'bin', '%s_mpi' % tm_arch)) + install_tree('libso/%s_mpi' % tm_arch, + join_path(dst, 'libso', '%s_mpi' % tm_arch)) + install_tree('mpirun_scripts/%s_mpi' % tm_arch, + join_path(dst, 'mpirun_scripts', '%s_mpi' % tm_arch)) elif '+smp' in spec: - install_tree('bin/%s_smp' % tm_arch, join_path(dst, 'bin', '%s_smp' % tm_arch)) - install_tree('libso/%s_smp' % tm_arch, join_path(dst, 'libso', '%s_smp' % tm_arch)) - install_tree('mpirun_scripts/%s_smp' % tm_arch, join_path(dst, 'mpirun_scripts', '%s_smp' % tm_arch)) + install_tree('bin/%s_smp' % tm_arch, + join_path(dst, 'bin', '%s_smp' % tm_arch)) + install_tree('libso/%s_smp' % tm_arch, + join_path(dst, 'libso', '%s_smp' % tm_arch)) + install_tree('mpirun_scripts/%s_smp' % tm_arch, + join_path(dst, 'mpirun_scripts', '%s_smp' % tm_arch)) else: install_tree('bin/%s' % tm_arch, join_path(dst, 'bin', tm_arch)) if '+mpi' in spec or '+smp' in spec: @@ -131,18 +136,29 @@ class Turbomole(Package): if self.spec.satisfies('@:7.0.2'): molecontrol_version = 'MoleControl_2.5' - tm_arch=self.get_tm_arch() + tm_arch = self.get_tm_arch() run_env.set('TURBODIR', join_path(self.prefix, 'TURBOMOLE')) - run_env.set('MOLE_CONTROL', join_path(self.prefix, 'TURBOMOLE', molecontrol_version)) + run_env.set('MOLE_CONTROL', + join_path(self.prefix, 'TURBOMOLE', molecontrol_version)) - run_env.prepend_path('PATH', join_path(self.prefix, 'TURBOMOLE', 'thermocalc')) - run_env.prepend_path('PATH', join_path(self.prefix, 'TURBOMOLE', 'scripts')) + run_env.prepend_path('PATH', + join_path(self.prefix, 'TURBOMOLE', 'thermocalc')) + run_env.prepend_path('PATH', + join_path(self.prefix, 'TURBOMOLE', 'scripts')) if '+mpi' in self.spec: run_env.set('PARA_ARCH', 'MPI') - run_env.prepend_path('PATH', join_path(self.prefix, 'TURBOMOLE', 'bin', '%s_mpi' % tm_arch)) + run_env.prepend_path('PATH', + join_path(self.prefix, + 'TURBOMOLE', 'bin', '%s_mpi' + % tm_arch)) elif '+smp' in self.spec: run_env.set('PARA_ARCH', 'SMP') - run_env.prepend_path('PATH', join_path(self.prefix, 'TURBOMOLE', 'bin', '%s_smp' % tm_arch)) + run_env.prepend_path('PATH', + join_path(self.prefix, + 'TURBOMOLE', 'bin', '%s_smp' + % tm_arch)) else: - run_env.prepend_path('PATH', join_path(self.prefix, 'TURBOMOLE', 'bin', tm_arch)) + run_env.prepend_path('PATH', + join_path(self.prefix, + 'TURBOMOLE', 'bin', tm_arch)) diff --git a/var/spack/repos/builtin/packages/xorg-util-macros/package.py b/var/spack/repos/builtin/packages/xorg-util-macros/package.py new file mode 100644 index 0000000000..963d93442f --- /dev/null +++ b/var/spack/repos/builtin/packages/xorg-util-macros/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class XorgUtilMacros(Package): + """The util-macros package contains the m4 macros used by all of the Xorg packages.""" + + homepage = "http://www.example.com" + url = "http://ftp.x.org/pub/individual/util/util-macros-1.19.0.tar.bz2" + + version('1.19.0', '1cf984125e75f8204938d998a8b6c1e1') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/xproto/package.py b/var/spack/repos/builtin/packages/xproto/package.py new file mode 100644 index 0000000000..7be6defb83 --- /dev/null +++ b/var/spack/repos/builtin/packages/xproto/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Xproto(Package): + """The Xorg protocol headers provide the header files required to + build the system, and to allow other applications to build against + the installed X Window system.""" + homepage = "http://www.x.org/" + url = "https://www.x.org/archive//individual/proto/xproto-7.0.29.tar.gz" + + version('7.0.29', '16a78dd2c5ad73011105c96235f6a0af') + + depends_on("xorg-util-macros") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") |