From efa506b235523c8a9a925c1ecf91690266da4fd4 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Sat, 14 May 2016 17:09:11 -0400 Subject: Preparing spack setup command for merge. Try this out for a while... --- lib/spack/docs/packaging_guide.rst | 123 +++++++++++++++++++++++++++++++------ lib/spack/spack/cmd/setup.py | 91 +++++++++++++++++++++++++++ lib/spack/spack/cmd/spconfig.py | 95 ---------------------------- lib/spack/spack/package.py | 30 ++++----- 4 files changed, 209 insertions(+), 130 deletions(-) create mode 100644 lib/spack/spack/cmd/setup.py delete mode 100644 lib/spack/spack/cmd/spconfig.py diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index d8e7cdfa80..438d787a12 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -377,6 +377,8 @@ add a line like this in the package class: version('8.2.1', '4136d7b4c04df68b686570afa26988ac') ... +Versions should be listed with the newest version first. + Version URLs ~~~~~~~~~~~~~~~~~ @@ -385,8 +387,21 @@ in the package. For example, Spack is smart enough to download version ``8.2.1.`` of the ``Foo`` package above from ``http://example.com/foo-8.2.1.tar.gz``. -If spack *cannot* extrapolate the URL from the ``url`` field, or if -the package doesn't have a ``url`` field, you can add a URL explicitly +If spack *cannot* extrapolate the URL from the ``url`` field by +default, you can write your own URL generation algorithm in place of +the ``url`` declaration. For example: + +.. code-block:: python + :linenos: + + class Foo(Package): + def url_for_version(self, version): + return 'http://example.com/version_%s/foo-%s.tar.gz' \ + % (version, version) + version('8.2.1', '4136d7b4c04df68b686570afa26988ac') + ... + +If a URL cannot be derived systematically, you can add an explicit URL for a particular version: .. code-block:: python @@ -1216,6 +1231,19 @@ Now, the ``py-numpy`` package can be used as an argument to ``spack activate``. When it is activated, all the files in its prefix will be symbolically linked into the prefix of the python package. +Many packages produce Python extensions for *some* variants, but not +others: they should extend ``python`` only if the apropriate +variant(s) are selected. This may be accomplished with conditional +``extends()`` declarations: + +.. code-block:: python + + class FooLib(Package): + variant('python', default=True, description= \ + 'Build the Python extension Module') + extends('python', when='+python') + ... + Sometimes, certain files in one package will conflict with those in another, which means they cannot both be activated (symlinked) at the same time. In this case, you can tell Spack to ignore those files @@ -2392,6 +2420,59 @@ File functions .. _package-lifecycle: +Coding Style Guidelines +--------------------------- + +The following guidelines are provided, in the interests of making +Spack packages work in a consistent manner: + + +Variant Names +~~~~~~~~~~~~~~ + +Spack packages with variants similar to already-existing Spack +packages should use the same name for their variants. Standard +variant names are: + +======= ======== ======================== +Name Default Description +------- -------- ------------------------ +shared True Build shared libraries +static Build static libraries +mpi Use MPI +python Build Python extension +------- -------- ------------------------ + +If specified in this table, the corresponding default should be used +when declaring a variant. + + +Version Lists +~~~~~~~~~~~~~~ + +Spack packges should list supported versions with the newest first. + +Special Versions +~~~~~~~~~~~~~~~~~ + +The following *special* version names may be used when building a package: + +* *@system*: Indicates a hook to the OS-installed version of the + package. This is useful, for example, to tell Spack to use the + OS-installed version in ``packages.yaml``:: + + openssl: + paths: + openssl@system: /usr + buildable: False + + Certain Spack internals look for the *@system* version and do + appropriate things in that case. + +* *@local*: Indicates the version was built manually from some source + tree of unknown provenance (see ``spack setup``). + + Packaging workflow commands --------------------------------- @@ -2715,7 +2796,7 @@ Imagine a developer creating a CMake-based (or Autotools) project in a local directory, which depends on libraries A-Z. Once Spack has installed those dependencies, one would like to run ``cmake`` with appropriate command line and environment so CMake can find them. The ``spack -spconfig`` command does this conveniently, producing a CMake +setup`` command does this conveniently, producing a CMake configuration that is essentially the same as how Spack *would have* configured the project. This can be demonstrated with a usage example: @@ -2723,7 +2804,7 @@ example: .. code-block:: bash cd myproject - spack spconfig myproject@local + spack setup myproject@local mkdir build; cd build ../spconfig.py .. make @@ -2732,29 +2813,31 @@ example: Notes: * Spack must have ``myproject/package.py`` in its repository for this to work. - * ``spack spconfig`` produces the executable script ``spconfig.py`` - in the local directory, and also creates the module file for the - package. ``spconfig.py`` is normally run from the top level of - the source tree. - - * The version number given to ``spack spconfig`` is arbitrary (just - like ``spack diy``). ``myproject/package.py`` does not need to + * ``spack setup`` produces the executable script ``spconfig.py`` in + the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the user's + out-of-source build directory. + * The version number given to ``spack setup`` is arbitrary, just + like ``spack diy``. ``myproject/package.py`` does not need to have any valid downloadable versions listed (typical when a project is new). * spconfig.py produces a CMake configuration that *does not* use the Spack wrappers. Any resulting binaries *will not* use RPATH, unless the user has enabled it. This is recommended for development purposes, not production. - * spconfig.py is easily legible, and can serve as a developer + * ``spconfig.py`` is human readable, and can serve as a developer reference of what dependencies are being used. * ``make install`` installs the package into the Spack repository, where it may be used by other Spack packages. - * CMake-generated makefiles re-run CMake in some circumstances. Use of ``spconfig.py`` breaks this behavior, requiring the developer to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file has changed. + * CMake-generated makefiles re-run CMake in some circumstances. Use + of ``spconfig.py`` breaks this behavior, requiring the developer + to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file + has changed. CMakePackage ~~~~~~~~~~~~ -In order ot enable ``spack spconfig`` functionality, the author of +In order ot enable ``spack setup`` functionality, the author of ``myproject/package.py`` must subclass from ``CMakePackage`` instead of the standard ``Package`` superclass. Because CMake is standardized, the packager does not need to tell Spack how to run @@ -2784,18 +2867,18 @@ StagedPackage ``CMakePackage`` is implemented by subclassing the ``StagedPackage`` superclass, which breaks down the standard ``Package.install()`` -method into several sub-stages: ``spconfig``, ``configure``, ``build`` +method into several sub-stages: ``setup``, ``configure``, ``build`` and ``install``. Details: * Instead of implementing the standard ``install()`` method, package authors implement the methods for the sub-stages - ``install_spconfig()``, ``install_configure()``, + ``install_setup()``, ``install_configure()``, ``install_build()``, and ``install_install()``. * The ``spack install`` command runs the sub-stages ``configure``, - ``build`` and ``install`` in order. (The ``spconfig`` stage is + ``build`` and ``install`` in order. (The ``setup`` stage is not run by default; see below). -* The ``spack spconfig`` command runs the sub-stages ``spconfig`` +* The ``spack setup`` command runs the sub-stages ``setup`` and a dummy install (to create the module file). * The sub-stage install methods take no arguments (other than ``self``). The arguments ``spec`` and ``prefix`` to the standard @@ -2805,9 +2888,9 @@ and ``install``. Details: GNU Autotools ~~~~~~~~~~~~~ -The ``spconfig`` functionality is currently only available for +The ``setup`` functionality is currently only available for CMake-based packages. Extending this functionality to GNU Autotools-based packages would be easy (and should be done by a developer who actively uses Autotools). Packages that use -non-standard build systems can gain ``spconfig`` functionality by +non-standard build systems can gain ``setup`` functionality by subclassing ``StagedPackage`` directly. diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py new file mode 100644 index 0000000000..02e9bfd281 --- /dev/null +++ b/lib/spack/spack/cmd/setup.py @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Elizabeth Fischer +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import os +import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +from spack.cmd.edit import edit_package +from spack.stage import DIYStage + +description = "Create a configuration script and module, but don't build." + +def setup_parser(subparser): + subparser.add_argument( + '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + help="Do not try to install dependencies of requested packages.") + subparser.add_argument( + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, + help="specs to use for install. Must contain package AND verison.") + + +def setup(self, args): + if not args.spec: + tty.die("spack setup requires a package spec argument.") + + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack setup only takes one spec.") + + # Take a write lock before checking for existence. + with spack.installed_db.write_transaction(): + spec = specs[0] + if not spack.repo.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, spack.repo.first_repo(), None, True) + return + + if not spec.versions.concrete: + tty.die("spack setup spec must have a single, concrete version. Did you forget a package version number?") + + spec.concretize() + package = spack.repo.get(spec) + + # It's OK if the package is already installed. + + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) + + # TODO: make this an argument, not a global. + spack.do_checksum = False + + package.do_install( + keep_prefix=True, # Don't remove install directory, even if you think you should + ignore_deps=args.ignore_deps, + verbose=args.verbose, + keep_stage=True, # don't remove source dir for SETUP. + install_phases = set(['setup', 'provenance'])) diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py deleted file mode 100644 index fcd9bdd63d..0000000000 --- a/lib/spack/spack/cmd/spconfig.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Elizabeth Fischer -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import sys -import os -import argparse - -import llnl.util.tty as tty - -import spack -import spack.cmd -from spack.cmd.edit import edit_package -from spack.stage import DIYStage - -description = "Create a configuration script and module, but don't build." - -def setup_parser(subparser): - subparser.add_argument( - '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - help="Do not try to install dependencies of requested packages.") - subparser.add_argument( - '-v', '--verbose', action='store_true', dest='verbose', - help="Display verbose build output while installing.") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="specs to use for install. Must contain package AND verison.") - - -def spconfig(self, args): - if not args.spec: - tty.die("spack spconfig requires a package spec argument.") - - specs = spack.cmd.parse_specs(args.spec) - if len(specs) > 1: - tty.die("spack spconfig only takes one spec.") - - # Take a write lock before checking for existence. - with spack.installed_db.write_transaction(): - spec = specs[0] - if not spack.repo.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, spack.repo.first_repo(), None, True) - return - - if not spec.versions.concrete: - tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") - - spec.concretize() - package = spack.repo.get(spec) - - # It's OK if the package is already installed. - #if package.installed: - # tty.error("Already installed in %s" % package.prefix) - # tty.msg("Uninstall or try adding a version suffix for this SPCONFIG build.") - # sys.exit(1) - - # Forces the build to run out of the current directory. - package.stage = DIYStage(os.getcwd()) - - # TODO: make this an argument, not a global. - spack.do_checksum = False - - package.do_install( - keep_prefix=True, # Don't remove install directory, even if you think you should - ignore_deps=args.ignore_deps, - verbose=args.verbose, - keep_stage=True, # don't remove source dir for SPCONFIG. - install_phases = set(['spconfig', 'provenance'])) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 959e618005..19198c3b28 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -935,7 +935,7 @@ class Package(object): packages_dir = spack.install_layout.build_packages_path(self.spec) # Remove first if we're overwriting another build - # (can happen with spack spconfig) + # (can happen with spack setup) try: shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this except: @@ -1456,9 +1456,9 @@ def _hms(seconds): class StagedPackage(Package): """A Package subclass where the install() is split up into stages.""" - def install_spconfig(self): - """Creates an spconfig.py script to configure the package later if we like.""" - raise InstallError("Package %s provides no install_spconfig() method!" % self.name) + def install_setup(self): + """Creates an spack_setup.py script to configure the package later if we like.""" + raise InstallError("Package %s provides no install_setup() method!" % self.name) def install_configure(self): """Runs the configure process.""" @@ -1473,8 +1473,8 @@ class StagedPackage(Package): raise InstallError("Package %s provides no install_install() method!" % self.name) def install(self, spec, prefix): - if 'spconfig' in self.install_phases: - self.install_spconfig() + if 'setup' in self.install_phases: + self.install_setup() if 'configure' in self.install_phases: self.install_configure() @@ -1520,13 +1520,13 @@ class CMakePackage(StagedPackage): """Returns package-specific environment under which the configure command should be run.""" return dict() - def cmake_transitive_include_path(self): + def spack_transitive_include_path(self): return ';'.join( os.path.join(dep, 'include') for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) ) - def install_spconfig(self): + def install_setup(self): cmd = [str(which('cmake'))] + \ spack.build_environment.get_std_cmake_args(self) + \ ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], @@ -1537,11 +1537,11 @@ class CMakePackage(StagedPackage): env = dict() env['PATH'] = os.environ['PATH'] - env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - spconfig_fname = 'spconfig.py' - with open(spconfig_fname, 'w') as fout: + setup_fname = 'spconfig.py' + with open(setup_fname, 'w') as fout: fout.write(\ r"""#!%s # @@ -1552,7 +1552,6 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) -#env = dict() env = dict(os.environ) """ % sys.executable) @@ -1562,7 +1561,7 @@ env = dict(os.environ) if string.find(name, 'PATH') < 0: fout.write('env[%s] = %s\n' % (repr(name),repr(val))) else: - if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': + if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': sep = ';' else: sep = ':' @@ -1572,20 +1571,21 @@ env = dict(os.environ) fout.write(' %s\n' % part) fout.write('"""))\n') + fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") fout.write('\ncmd = cmdlist("""\n') fout.write('%s\n' % cmd[0]) for arg in cmd[1:]: fout.write(' %s\n' % arg) fout.write('""") + sys.argv[1:]\n') fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') - make_executable(spconfig_fname) + make_executable(setup_fname) def install_configure(self): cmake = which('cmake') with working_dir(self.build_directory, create=True): os.environ.update(self.configure_env()) - os.environ['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + os.environ['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) cmake(self.source_directory, *options) -- cgit v1.2.3-70-g09d2