From 1f49493feed526f715aec0fef0ffe83c56aab117 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 16 Jan 2017 19:13:12 -0600 Subject: Major improvements to spack create (#2707) * Initial changes to spack create command * Get 'spack create ' working again * Simplify call to BuildSystemGuesser * More verbose output of spack create * Remove duplicated code from spack create and spack checksum * Add better documentation to spack create docstrings * Fix pluralization bug * Flake8 * Update documentation on spack create and deprecate spack edit --force * Make it more obvious when we are renaming a package * Further deprecate spack edit --force * Fix unit tests * Rename default template to generic template * Don't add automake/autoconf deps to Autotools packages * Remove changes to default $EDITOR * Completely remove all traces of spack edit --force * Remove grammar changes to make the PR easier to review --- lib/spack/docs/packaging_guide.rst | 297 +++++++++---------- lib/spack/spack/cmd/checksum.py | 138 +++++---- lib/spack/spack/cmd/create.py | 438 +++++++++++++++++------------ lib/spack/spack/cmd/diy.py | 12 +- lib/spack/spack/cmd/edit.py | 66 ++--- lib/spack/spack/cmd/setup.py | 13 +- lib/spack/spack/test/build_system_guess.py | 11 +- 7 files changed, 526 insertions(+), 449 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 708dd71c76..8a39ee28e2 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -17,7 +17,7 @@ There are two key parts of Spack: software according to a spec. Specs allow a user to describe a *particular* build in a way that a -package author can understand. Packages allow a the packager to +package author can understand. Packages allow the packager to encapsulate the build logic for different versions, compilers, options, platforms, and dependency combinations in one place. Essentially, a package translates a spec into build logic. @@ -40,87 +40,68 @@ Creating & editing packages ^^^^^^^^^^^^^^^^ The ``spack create`` command creates a directory with the package name and -generates a ``package.py`` file with a boilerplate package template from a URL. -The URL should point to a tarball or other software archive. In most cases, -``spack create`` plus a few modifications is all you need to get a package -working. +generates a ``package.py`` file with a boilerplate package template. If given +a URL pointing to a tarball or other software archive, ``spack create`` is +smart enough to determine basic information about the package, including its name +and build system. In most cases, ``spack create`` plus a few modifications is +all you need to get a package working. Here's an example: .. code-block:: console - $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz + $ spack create https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 Spack examines the tarball URL and tries to figure out the name of the package -to be created. Once the name is determined a directory in the appropriate -repository is created with that name. Spack prefers, but does not require, that -names be lower case so the directory name will be lower case when ``spack -create`` generates it. In cases where it is desired to have mixed case or upper -case simply rename the directory. Spack also tries to determine what version -strings look like for this package. Using this information, it will try to find -*additional* versions by spidering the package's webpage. If it finds multiple -versions, Spack prompts you to tell it how many versions you want to download -and checksum: +to be created. If the name contains uppercase letters, these are automatically +converted to lowercase. If the name contains underscores or periods, these are +automatically converted to dashes. + +Spack also searches for *additional* versions located in the same directory of +the website. Spack prompts you to tell you how many versions it found and asks +you how many you would like to download and checksum: .. code-block:: console - $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz - ==> This looks like a URL for cmake version 2.8.12.1. - ==> Creating template for package cmake - ==> Found 18 versions of cmake. - 2.8.12.1 http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz - 2.8.12 http://www.cmake.org/files/v2.8/cmake-2.8.12.tar.gz - 2.8.11.2 http://www.cmake.org/files/v2.8/cmake-2.8.11.2.tar.gz + $ spack create https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 + ==> This looks like a URL for gmp + ==> Found 16 versions of gmp: + + 6.1.2 https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 + 6.1.1 https://gmplib.org/download/gmp/gmp-6.1.1.tar.bz2 + 6.1.0 https://gmplib.org/download/gmp/gmp-6.1.0.tar.bz2 ... - 2.8.0 http://www.cmake.org/files/v2.8/cmake-2.8.0.tar.gz + 5.0.0 https://gmplib.org/download/gmp/gmp-5.0.0.tar.bz2 - Include how many checksums in the package file? (default is 5, q to abort) + How many would you like to checksum? (default is 1, q to abort) Spack will automatically download the number of tarballs you specify (starting with the most recent) and checksum each of them. You do not *have* to download all of the versions up front. You can always choose to download just one tarball initially, and run -:ref:`cmd-spack-checksum` later if you need more. - -.. note:: - - If ``spack create`` fails to detect the package name correctly, - you can try supplying it yourself, e.g.: - - .. code-block:: console - - $ spack create --name cmake http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz - - If it fails entirely, you can get minimal boilerplate by using - :ref:`spack edit --force `, or you can manually create a - directory and ``package.py`` file for the package in - ``var/spack/repos/builtin/packages``, or within your own :ref:`package - repository `. - -.. note:: - - Spack can fetch packages from source code repositories, but, - ``spack create`` will *not* currently create a boilerplate package - from a repository URL. You will need to use :ref:`spack edit --force ` - and manually edit the ``version()`` directives to fetch from a - repo. See :ref:`vcs-fetch` for details. +:ref:`cmd-spack-checksum` later if you need more versions. Let's say you download 3 tarballs: -.. code-block:: none - - Include how many checksums in the package file? (default is 5, q to abort) 3 - ==> Downloading... - ==> Fetching http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz - ###################################################################### 98.6% - ==> Fetching http://www.cmake.org/files/v2.8/cmake-2.8.12.tar.gz - ##################################################################### 96.7% - ==> Fetching http://www.cmake.org/files/v2.8/cmake-2.8.11.2.tar.gz - #################################################################### 95.2% +.. code-block:: console -Now Spack generates boilerplate code and opens a new ``package.py`` -file in your favorite ``$EDITOR``: + How many would you like to checksum? (default is 1, q to abort) 3 + ==> Downloading... + ==> Fetching https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 + ######################################################################## 100.0% + ==> Fetching https://gmplib.org/download/gmp/gmp-6.1.1.tar.bz2 + ######################################################################## 100.0% + ==> Fetching https://gmplib.org/download/gmp/gmp-6.1.0.tar.bz2 + ######################################################################## 100.0% + ==> Checksummed 3 versions of gmp: + ==> This package looks like it uses the autotools build system + ==> Created template for gmp package + ==> Created package file: /Users/Adam/spack/var/spack/repos/builtin/packages/gmp/package.py + +Spack automatically creates a directory in the appropriate repository, +generates a boilerplate template for your package, and opens up the new +``package.py`` in your favorite ``$EDITOR``: .. code-block:: python :linenos: @@ -130,11 +111,11 @@ file in your favorite ``$EDITOR``: # next to all the things you'll want to change. Once you've handled # them, you can save this file and test your package like this: # - # spack install cmake + # spack install gmp # # You can edit this file again by typing: # - # spack edit cmake + # spack edit gmp # # See the Spack documentation for more information on packaging. # If you submit this package back to Spack as a pull request, @@ -143,33 +124,46 @@ file in your favorite ``$EDITOR``: from spack import * - class Cmake(Package): + class Gmp(AutotoolsPackage): """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. homepage = "http://www.example.com" - url = "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz" - - version('2.8.12.1', '9d38cd4e2c94c3cea97d0e2924814acc') - version('2.8.12', '105bc6d21cc2e9b6aff901e43c53afea') - version('2.8.11.2', '6f5d7b8e7534a5d9e1a7664ba63cf882') + url = "https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2" - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + version('6.1.2', '8ddbb26dc3bd4e2302984debba1406a5') + version('6.1.1', '4c175f86e11eb32d8bf9872ca3a8e11d') + version('6.1.0', '86ee6e54ebfc4a90b643a65e402c4048') - def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. - configure("--prefix=" + prefix) + # FIXME: Add dependencies if required. + # depends_on('foo') - # FIXME: Add logic to build and install here - make() - make("install") + def configure_args(self): + # FIXME: Add arguments other than --prefix + # FIXME: If not needed delete the function + args = [] + return args The tedious stuff (creating the class, checksumming archives) has been -done for you. +done for you. You'll notice that ``spack create`` correctly detected that +``gmp`` uses the Autotools build system. It created a new ``Gmp`` package +that subclasses the ``AutotoolsPackage`` base class. This base class +provides basic installation methods common to all Autotools packages: + +.. code-block:: bash + + ./configure --prefix=/path/to/installation/directory + + make + make check + make install + +For most Autotools packages, this is sufficient. If you need to add +additional arguments to the ``./configure`` call, add them via the +``configure_args`` function. In the generated package, the download ``url`` attribute is already -set. All the things you still need to change are marked with +set. All the things you still need to change are marked with ``FIXME`` labels. You can delete the commented instructions between the license and the first import statement after reading them. The rest of the tasks you need to do are as follows: @@ -177,105 +171,96 @@ The rest of the tasks you need to do are as follows: #. Add a description. Immediately inside the package class is a *docstring* in - triple-quotes (``"""``). It's used to generate the description + triple-quotes (``"""``). It is used to generate the description shown when users run ``spack info``. #. Change the ``homepage`` to a useful URL. The ``homepage`` is displayed when users run ``spack info`` so - that they can learn about packages. + that they can learn more about your package. #. Add ``depends_on()`` calls for the package's dependencies. ``depends_on`` tells Spack that other packages need to be built - and installed before this one. See :ref:`dependencies`. + and installed before this one. See :ref:`dependencies`. -#. Get the ``install()`` method working. +#. Get the installation working. - The ``install()`` method implements the logic to build a - package. The code should look familiar; it is designed to look - like a shell script. Specifics will differ depending on the package, - and :ref:`implementing the install method ` is + Your new package may require specific flags during ``configure``. + These can be added via ``configure_args``. Specifics will differ + depending on the package and its build system. + :ref:`Implementing the install method ` is covered in detail later. -Before going into details, we'll cover a few more basics. - -.. _cmd-spack-edit: - -^^^^^^^^^^^^^^ -``spack edit`` -^^^^^^^^^^^^^^ - -One of the easiest ways to learn to write packages is to look at -existing ones. You can edit a package file by name with the ``spack -edit`` command: +Passing a URL to ``spack create`` is a convenient and easy way to get +a basic package template, but what if your software is licensed and +cannot be downloaded from a URL? You can still create a boilerplate +``package.py`` by telling ``spack create`` what name you want to use: .. code-block:: console - $ spack edit cmake + $ spack create --name intel -So, if you used ``spack create`` to create a package, then saved and -closed the resulting file, you can get back to it with ``spack edit``. -The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, -but this provides a much simpler shortcut and saves you the trouble -of typing the full path. +This will create a simple ``intel`` package with an ``install()`` +method that you can craft to install your package. -If you try to edit a package that doesn't exist, Spack will recommend -using ``spack create`` or ``spack edit --force``: +What if ``spack create `` guessed the wrong name or build system? +For example, if your package uses the Autotools build system but does +not come with a ``configure`` script, Spack won't realize it uses +Autotools. You can overwrite the old package with ``--force`` and specify +a name with ``--name`` or a build system template to use with ``--template``: .. code-block:: console - $ spack edit foo - ==> Error: No package 'foo'. Use spack create, or supply -f/--force to edit a new file. - -.. _spack-edit-f: - -^^^^^^^^^^^^^^^^^^^^^^ -``spack edit --force`` -^^^^^^^^^^^^^^^^^^^^^^ + $ spack create --name gmp https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 + $ spack create --force --template autotools https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 -``spack edit --force`` can be used to create a new, minimal boilerplate -package: +.. note:: -.. code-block:: console + If you are creating a package that uses the Autotools build system + but does not come with a ``configure`` script, you'll need to add an + ``autoreconf`` method to your package that explains how to generate + the ``configure`` script. You may also need the following dependencies: - $ spack edit --force foo + .. code-block:: python -Unlike ``spack create``, which infers names and versions, and which -actually downloads the tarball and checksums it for you, ``spack edit ---force`` has no such fanciness. It will substitute dummy values for you -to fill in yourself: + depends_on('autoconf', type='build') + depends_on('automake', type='build') + depends_on('libtool', type='build') + depends_on('m4', type='build') -.. code-block:: python - :linenos: +A complete list of available build system templates can be found by running +``spack create --help``. - from spack import * +.. _cmd-spack-edit: - class Foo(Package): - """Description""" +^^^^^^^^^^^^^^ +``spack edit`` +^^^^^^^^^^^^^^ - homepage = "http://www.example.com" - url = "http://www.example.com/foo-1.0.tar.gz" +One of the easiest ways to learn how to write packages is to look at +existing ones. You can edit a package file by name with the ``spack +edit`` command: - version('1.0', '0123456789abcdef0123456789abcdef') +.. code-block:: console - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") + $ spack edit gmp -This is useful when ``spack create`` cannot figure out the name and -version of your package from the archive URL. +So, if you used ``spack create`` to create a package, then saved and +closed the resulting file, you can get back to it with ``spack edit``. +The ``gmp`` package actually lives in +``$SPACK_ROOT/var/spack/repos/builtin/packages/gmp/package.py``, +but ``spack edit`` provides a much simpler shortcut and saves you the +trouble of typing the full path. ---------------------------- Naming & directory structure ---------------------------- This section describes how packages need to be named, and where they -live in Spack's directory structure. In general, :ref:`cmd-spack-create` and -:ref:`cmd-spack-edit` handle creating package files for you, so you can skip -most of the details here. +live in Spack's directory structure. In general, :ref:`cmd-spack-create` +handles creating package files for you, so you can skip most of the +details here. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``var/spack/repos/builtin/packages`` @@ -308,10 +293,9 @@ directories or files (like patches) that it needs to build. Package Names ^^^^^^^^^^^^^ -Packages are named after the directory containing ``package.py``. It is -preferred, but not required, that the directory, and thus the package name, are -lower case. So, ``libelf``'s ``package.py`` lives in a directory called -``libelf``. The ``package.py`` file defines a class called ``Libelf``, which +Packages are named after the directory containing ``package.py``. So, +``libelf``'s ``package.py`` lives in a directory called ``libelf``. +The ``package.py`` file defines a class called ``Libelf``, which extends Spack's ``Package`` class. For example, here is ``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: @@ -336,21 +320,22 @@ these: .. code-block:: console - $ spack install libelf + $ spack info libelf + $ spack versions libelf $ spack install libelf@0.8.13 Spack sees the package name in the spec and looks for -``libelf/package.py`` in ``var/spack/repos/builtin/packages``. Likewise, if you say -``spack install py-numpy``, then Spack looks for +``libelf/package.py`` in ``var/spack/repos/builtin/packages``. +Likewise, if you run ``spack install py-numpy``, Spack looks for ``py-numpy/package.py``. Spack uses the directory name as the package name in order to give -packagers more freedom in naming their packages. Package names can -contain letters, numbers, dashes, and underscores. Using a Python -identifier (e.g., a class name or a module name) would make it -difficult to support these options. So, you can name a package -``3proxy`` or ``_foo`` and Spack won't care. It just needs to see -that name in the package spec. +packagers more freedom in naming their packages. Package names can +contain letters, numbers, and dashes. Using a Python identifier +(e.g., a class name or a module name) would make it difficult to +support these options. So, you can name a package ``3proxy`` or +``foo-bar`` and Spack won't care. It just needs to see that name +in the packages directory. ^^^^^^^^^^^^^^^^^^^ Package class names @@ -359,16 +344,14 @@ Package class names Spack loads ``package.py`` files dynamically, and it needs to find a special class name in the file for the load to succeed. The **class name** (``Libelf`` in our example) is formed by converting words -separated by `-` or ``_`` in the file name to camel case. If the name +separated by ``-`` in the file name to CamelCase. If the name starts with a number, we prefix the class name with ``_``. Here are some examples: ================= ================= Module Name Class Name ================= ================= - ``foo_bar`` ``FooBar`` - ``docbook-xml`` ``DocbookXml`` - ``FooBar`` ``Foobar`` + ``foo-bar`` ``FooBar`` ``3proxy`` ``_3proxy`` ================= ================= @@ -2719,7 +2702,7 @@ running: from spack import * This is already part of the boilerplate for packages created with -``spack create`` or ``spack edit``. +``spack create``. ^^^^^^^^^^^^^^^^^^^ Filtering functions diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index b45cfcbd39..8e4de0efc3 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -22,6 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from __future__ import print_function + import argparse import hashlib @@ -30,6 +32,7 @@ import spack import spack.cmd import spack.util.crypto from spack.stage import Stage, FailedDownloadError +from spack.util.naming import * from spack.version import * description = "Checksum available versions of a package." @@ -37,86 +40,125 @@ description = "Checksum available versions of a package." def setup_parser(subparser): subparser.add_argument( - 'package', metavar='PACKAGE', help='Package to list versions for') + 'package', + help='Package to checksum versions for') subparser.add_argument( - '--keep-stage', action='store_true', dest='keep_stage', + '--keep-stage', action='store_true', help="Don't clean up staging area when command completes.") subparser.add_argument( 'versions', nargs=argparse.REMAINDER, help='Versions to generate checksums for') -def get_checksums(versions, urls, **kwargs): - # Allow commands like create() to do some analysis on the first - # archive after it is downloaded. +def get_checksums(url_dict, name, **kwargs): + """Fetches and checksums archives from URLs. + + This function is called by both ``spack checksum`` and ``spack create``. + The ``first_stage_function`` kwarg allows ``spack create`` to determine + things like the build system of the archive. + + :param dict url_dict: A dictionary of the form: version -> URL + :param str name: The name of the package + :param callable first_stage_function: Function to run on first staging area + :param bool keep_stage: Don't clean up staging area when command completes + + :returns: A multi-line string containing versions and corresponding hashes + :rtype: str + """ first_stage_function = kwargs.get('first_stage_function', None) keep_stage = kwargs.get('keep_stage', False) + sorted_versions = sorted(url_dict.keys(), reverse=True) + + # Find length of longest string in the list for padding + max_len = max(len(str(v)) for v in sorted_versions) + num_ver = len(sorted_versions) + + tty.msg("Found {0} version{1} of {2}:".format( + num_ver, '' if num_ver == 1 else 's', name), + "", + *spack.cmd.elide_list( + ["{0:{1}} {2}".format(v, max_len, url_dict[v]) + for v in sorted_versions])) + print() + + archives_to_fetch = tty.get_number( + "How many would you like to checksum?", default=1, abort='q') + + if not archives_to_fetch: + tty.die("Aborted.") + + versions = sorted_versions[:archives_to_fetch] + urls = [url_dict[v] for v in versions] + tty.msg("Downloading...") - hashes = [] + version_hashes = [] i = 0 for url, version in zip(urls, versions): try: with Stage(url, keep=keep_stage) as stage: + # Fetch the archive stage.fetch() if i == 0 and first_stage_function: + # Only run first_stage_function the first time, + # no need to run it every time first_stage_function(stage, url) - hashes.append((version, spack.util.crypto.checksum( + # Checksum the archive and add it to the list + version_hashes.append((version, spack.util.crypto.checksum( hashlib.md5, stage.archive_file))) i += 1 - except FailedDownloadError as e: - tty.msg("Failed to fetch %s" % url) + except FailedDownloadError: + tty.msg("Failed to fetch {0}".format(url)) except Exception as e: - tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) + tty.msg("Something failed on {0}, skipping.".format(url), + " ({0})".format(e)) - return hashes + if not version_hashes: + tty.die("Could not fetch any versions for {0}".format(name)) + + # Find length of longest string in the list for padding + max_len = max(len(str(v)) for v, h in version_hashes) + + # Generate the version directives to put in a package.py + version_lines = "\n".join([ + " version('{0}', {1}'{2}')".format( + v, ' ' * (max_len - len(str(v))), h) for v, h in version_hashes + ]) + + num_hash = len(version_hashes) + tty.msg("Checksummed {0} version{1} of {2}".format( + num_hash, '' if num_hash == 1 else 's', name)) + + return version_lines def checksum(parser, args): - # get the package we're going to generate checksums for + # Make sure the user provided a package and not a URL + if not valid_fully_qualified_module_name(args.package): + tty.die("`spack checksum` accepts package names, not URLs. " + "Use `spack md5 ` instead.") + + # Get the package we're going to generate checksums for pkg = spack.repo.get(args.package) - # If the user asked for specific versions, use those. if args.versions: - versions = {} + # If the user asked for specific versions, use those + url_dict = {} for version in args.versions: version = ver(version) if not isinstance(version, Version): - tty.die("Cannot generate checksums for version lists or " + - "version ranges. Use unambiguous versions.") - versions[version] = pkg.url_for_version(version) + tty.die("Cannot generate checksums for version lists or " + "version ranges. Use unambiguous versions.") + url_dict[version] = pkg.url_for_version(version) else: - versions = pkg.fetch_remote_versions() - if not versions: - tty.die("Could not fetch any versions for %s" % pkg.name) - - sorted_versions = sorted(versions, reverse=True) - - # Find length of longest string in the list for padding - maxlen = max(len(str(v)) for v in versions) + # Otherwise, see what versions we can find online + url_dict = pkg.fetch_remote_versions() + if not url_dict: + tty.die("Could not find any versions for {0}".format(pkg.name)) - tty.msg("Found %s versions of %s" % (len(versions), pkg.name), - *spack.cmd.elide_list( - ["{0:{1}} {2}".format(v, maxlen, versions[v]) - for v in sorted_versions])) - print - archives_to_fetch = tty.get_number( - "How many would you like to checksum?", default=5, abort='q') - - if not archives_to_fetch: - tty.msg("Aborted.") - return - - version_hashes = get_checksums( - sorted_versions[:archives_to_fetch], - [versions[v] for v in sorted_versions[:archives_to_fetch]], - keep_stage=args.keep_stage) - - if not version_hashes: - tty.die("Could not fetch any versions for %s" % pkg.name) + version_lines = get_checksums( + url_dict, pkg.name, keep_stage=args.keep_stage) - version_lines = [ - " version('%s', '%s')" % (v, h) for v, h in version_hashes - ] - tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) + print() + print(version_lines) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index cac67f22af..a488104282 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -26,7 +26,6 @@ from __future__ import print_function import os import re -import string import llnl.util.tty as tty import spack @@ -35,15 +34,14 @@ import spack.cmd.checksum import spack.url import spack.util.web from llnl.util.filesystem import mkdirp -from ordereddict_backport import OrderedDict -from spack.repository import Repo, RepoError +from spack.repository import Repo from spack.spec import Spec from spack.util.executable import which from spack.util.naming import * -description = "Create a new package file from an archive URL" +description = "Create a new package file" -package_template = string.Template("""\ +package_template = '''\ ############################################################################## # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -73,11 +71,11 @@ package_template = string.Template("""\ # next to all the things you'll want to change. Once you've handled # them, you can save this file and test your package like this: # -# spack install ${name} +# spack install {name} # # You can edit this file again by typing: # -# spack edit ${name} +# spack edit {name} # # See the Spack documentation for more information on packaging. # If you submit this package back to Spack as a pull request, @@ -86,23 +84,24 @@ package_template = string.Template("""\ from spack import * -class ${class_name}(${base_class_name}): - ""\"FIXME: Put a proper description of your package here.""\" +class {class_name}({base_class_name}): + """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. homepage = "http://www.example.com" - url = "${url}" + url = "{url}" -${versions} +{versions} -${dependencies} +{dependencies} -${body} -""") +{body} +''' -class DefaultGuess(object): +class PackageTemplate(object): """Provides the default values to be used for the package file template""" + base_class_name = 'Package' dependencies = """\ @@ -115,57 +114,61 @@ class DefaultGuess(object): make() make('install')""" - def __init__(self, name, url, version_hash_tuples): - self.name = name + def __init__(self, name, url, versions): + self.name = name self.class_name = mod_to_class(name) - self.url = url - self.version_hash_tuples = version_hash_tuples + self.url = url + self.versions = versions + + def write(self, pkg_path): + """Writes the new package file.""" + + # Write out a template for the file + with open(pkg_path, "w") as pkg_file: + pkg_file.write(package_template.format( + name=self.name, + class_name=self.class_name, + base_class_name=self.base_class_name, + url=self.url, + versions=self.versions, + dependencies=self.dependencies, + body=self.body)) - @property - def versions(self): - """Adds a version() call to the package for each version found.""" - max_len = max(len(str(v)) for v, h in self.version_hash_tuples) - format = " version(%%-%ds, '%%s')" % (max_len + 2) - return '\n'.join( - format % ("'%s'" % v, h) for v, h in self.version_hash_tuples - ) +class AutotoolsPackageTemplate(PackageTemplate): + """Provides appropriate overrides for Autotools-based packages""" -class AutotoolsGuess(DefaultGuess): - """Provides appropriate overrides for autotools-based packages""" base_class_name = 'AutotoolsPackage' dependencies = """\ # FIXME: Add dependencies if required. - # depends_on('m4', type='build') - # depends_on('autoconf', type='build') - # depends_on('automake', type='build') - # depends_on('libtool', type='build') # depends_on('foo')""" body = """\ def configure_args(self): # FIXME: Add arguments other than --prefix - # FIXME: If not needed delete the function + # FIXME: If not needed delete this function args = [] return args""" -class CMakeGuess(DefaultGuess): - """Provides appropriate overrides for cmake-based packages""" +class CMakePackageTemplate(PackageTemplate): + """Provides appropriate overrides for CMake-based packages""" + base_class_name = 'CMakePackage' body = """\ def cmake_args(self): # FIXME: Add arguments other than # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE - # FIXME: If not needed delete the function + # FIXME: If not needed delete this function args = [] return args""" -class SconsGuess(DefaultGuess): - """Provides appropriate overrides for scons-based packages""" +class SconsPackageTemplate(PackageTemplate): + """Provides appropriate overrides for SCons-based packages""" + dependencies = """\ # FIXME: Add additional dependencies if required. depends_on('scons', type='build')""" @@ -177,8 +180,9 @@ class SconsGuess(DefaultGuess): scons('install')""" -class BazelGuess(DefaultGuess): - """Provides appropriate overrides for bazel-based packages""" +class BazelPackageTemplate(PackageTemplate): + """Provides appropriate overrides for Bazel-based packages""" + dependencies = """\ # FIXME: Add additional dependencies if required. depends_on('bazel', type='build')""" @@ -189,8 +193,9 @@ class BazelGuess(DefaultGuess): bazel()""" -class PythonGuess(DefaultGuess): - """Provides appropriate overrides for python extensions""" +class PythonPackageTemplate(PackageTemplate): + """Provides appropriate overrides for Python extensions""" + dependencies = """\ extends('python') @@ -204,12 +209,18 @@ class PythonGuess(DefaultGuess): setup_py('install', '--prefix={0}'.format(prefix))""" def __init__(self, name, *args): - name = 'py-{0}'.format(name) - super(PythonGuess, self).__init__(name, *args) + # If the user provided `--name py-numpy`, don't rename it py-py-numpy + if not name.startswith('py-'): + # Make it more obvious that we are renaming the package + tty.msg("Changing package name from {0} to py-{0}".format(name)) + name = 'py-{0}'.format(name) + super(PythonPackageTemplate, self).__init__(name, *args) -class RGuess(DefaultGuess): + +class RPackageTemplate(PackageTemplate): """Provides appropriate overrides for R extensions""" + dependencies = """\ # FIXME: Add dependencies if required. # depends_on('r-foo', type=('build', 'run'))""" @@ -218,12 +229,18 @@ class RGuess(DefaultGuess): # FIXME: Override install() if necessary.""" def __init__(self, name, *args): - name = 'r-{0}'.format(name) - super(RGuess, self).__init__(name, *args) + # If the user provided `--name r-rcpp`, don't rename it r-r-rcpp + if not name.startswith('r-'): + # Make it more obvious that we are renaming the package + tty.msg("Changing package name from {0} to r-{0}".format(name)) + name = 'r-{0}'.format(name) + super(RPackageTemplate, self).__init__(name, *args) -class OctaveGuess(DefaultGuess): + +class OctavePackageTemplate(PackageTemplate): """Provides appropriate overrides for octave packages""" + dependencies = """\ extends('octave') @@ -240,43 +257,58 @@ class OctaveGuess(DefaultGuess): prefix, self.stage.archive_file))""" def __init__(self, name, *args): - name = 'octave-{0}'.format(name) - super(OctaveGuess, self).__init__(name, *args) + # If the user provided `--name octave-splines`, don't rename it + # octave-octave-splines + if not name.startswith('octave-'): + # Make it more obvious that we are renaming the package + tty.msg("Changing package name from {0} to octave-{0}".format(name)) # noqa + name = 'octave-{0}'.format(name) + + super(OctavePackageTemplate, self).__init__(name, *args) + + +templates = { + 'autotools': AutotoolsPackageTemplate, + 'cmake': CMakePackageTemplate, + 'scons': SconsPackageTemplate, + 'bazel': BazelPackageTemplate, + 'python': PythonPackageTemplate, + 'r': RPackageTemplate, + 'octave': OctavePackageTemplate, + 'generic': PackageTemplate +} def setup_parser(subparser): - subparser.add_argument('url', nargs='?', help="url of package archive") + subparser.add_argument( + 'url', nargs='?', + help="url of package archive") subparser.add_argument( '--keep-stage', action='store_true', help="Don't clean up staging area when command completes.") subparser.add_argument( - '-n', '--name', dest='alternate_name', default=None, metavar='NAME', - help="Override the autodetected name for the created package.") + '-n', '--name', + help="name of the package to create") + subparser.add_argument( + '-t', '--template', metavar='TEMPLATE', choices=templates.keys(), + help="build system template to use. options: %(choices)s") subparser.add_argument( - '-r', '--repo', default=None, + '-r', '--repo', help="Path to a repository where the package should be created.") subparser.add_argument( '-N', '--namespace', help="Specify a namespace for the package. Must be the namespace of " "a repository registered with Spack.") subparser.add_argument( - '-f', '--force', action='store_true', dest='force', + '-f', '--force', action='store_true', help="Overwrite any existing package file with the same name.") - setup_parser.subparser = subparser - - -class BuildSystemGuesser(object): - _choices = { - 'autotools': AutotoolsGuess, - 'cmake': CMakeGuess, - 'scons': SconsGuess, - 'bazel': BazelGuess, - 'python': PythonGuess, - 'r': RGuess, - 'octave': OctaveGuess - } +class BuildSystemGuesser: + """An instance of BuildSystemGuesser provides a callable object to be used + during ``spack create``. By passing this object to ``spack checksum``, we + can take a peek at the fetched tarball and discern the build system it uses + """ def __call__(self, stage, url): """Try to guess the type of build system used by a project based on @@ -319,65 +351,173 @@ class BuildSystemGuesser(object): # Determine the build system based on the files contained # in the archive. - build_system = 'unknown' + build_system = 'generic' for pattern, bs in clues: if any(re.search(pattern, l) for l in lines): build_system = bs self.build_system = build_system - def make_guess(self, name, url, ver_hash_tuples): - cls = self._choices.get(self.build_system, DefaultGuess) - return cls(name, url, ver_hash_tuples) +def get_name(args): + """Get the name of the package based on the supplied arguments. -def guess_name_and_version(url, args): - # Try to deduce name and version of the new package from the URL - version = spack.url.parse_version(url) - if not version: - tty.die("Couldn't guess a version string from %s" % url) + If a name was provided, always use that. Otherwise, if a URL was + provided, extract the name from that. Otherwise, use a default. - # Try to guess a name. If it doesn't work, allow the user to override. - if args.alternate_name: - name = args.alternate_name - else: + :param argparse.Namespace args: The arguments given to ``spack create`` + + :returns: The name of the package + :rtype: str + """ + + # Default package name + name = 'example' + + if args.name: + # Use a user-supplied name if one is present + name = args.name + tty.msg("Using specified package name: '{0}'".format(name)) + elif args.url: + # Try to guess the package name based on the URL try: - name = spack.url.parse_name(url, version) + name = spack.url.parse_name(args.url) + tty.msg("This looks like a URL for {0}".format(name)) except spack.url.UndetectableNameError: - # Use a user-supplied name if one is present - tty.die("Couldn't guess a name for this package. Try running:", "", - "spack create --name ") + tty.die("Couldn't guess a name for this package.", + " Please report this bug. In the meantime, try running:", + " `spack create --name `") if not valid_fully_qualified_module_name(name): - tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'") + tty.die("Package name can only contain a-z, 0-9, and '-'") + + return name + + +def get_url(args): + """Get the URL to use. + + Use a default URL if none is provided. + + :param argparse.Namespace args: The arguments given to ``spack create`` + + :returns: The URL of the package + :rtype: str + """ + + # Default URL + url = 'http://www.example.com/example-1.2.3.tar.gz' + + if args.url: + # Use a user-supplied URL if one is present + url = args.url + + return url + + +def get_versions(args, name): + """Returns a list of versions and hashes for a package. + + Also returns a BuildSystemGuesser object. + + Returns default values if no URL is provided. + + :param argparse.Namespace args: The arguments given to ``spack create`` + :param str name: The name of the package - return name, version + :returns: Versions and hashes, and a BuildSystemGuesser object + :rtype: str and BuildSystemGuesser + """ + # Default version, hash, and guesser + versions = """\ + # FIXME: Add proper versions and checksums here. + # version('1.2.3', '0123456789abcdef0123456789abcdef')""" -def find_repository(spec, args): - # figure out namespace for spec + guesser = BuildSystemGuesser() + + if args.url: + # Find available versions + url_dict = spack.util.web.find_versions_of_archive(args.url) + + if not url_dict: + # If no versions were found, revert to what the user provided + version = spack.url.parse_version(args.url) + url_dict = {version: args.url} + + versions = spack.cmd.checksum.get_checksums( + url_dict, name, first_stage_function=guesser, + keep_stage=args.keep_stage) + + return versions, guesser + + +def get_build_system(args, guesser): + """Determine the build system template. + + If a template is specified, always use that. Otherwise, if a URL + is provided, download the tarball and peek inside to guess what + build system it uses. Otherwise, use a generic template by default. + + :param argparse.Namespace args: The arguments given to ``spack create`` + :param BuildSystemGuesser guesser: The first_stage_function given to \ + ``spack checksum`` which records the build system it detects + + :returns: The name of the build system template to use + :rtype: str + """ + + # Default template + template = 'generic' + + if args.template: + # Use a user-supplied template if one is present + template = args.template + tty.msg("Using specified package template: '{0}'".format(template)) + elif args.url: + # Use whatever build system the guesser detected + template = guesser.build_system + if template == 'generic': + tty.warn("Unable to detect a build system. " + "Using a generic package template.") + else: + msg = "This package looks like it uses the {0} build system" + tty.msg(msg.format(template)) + + return template + + +def get_repository(args, name): + """Returns a Repo object that will allow us to determine the path where + the new package file should be created. + + :param argparse.Namespace args: The arguments given to ``spack create`` + :param str name: The name of the package to create + + :returns: A Repo object capable of determining the path to the package file + :rtype: Repo + """ + spec = Spec(name) + # Figure out namespace for spec if spec.namespace and args.namespace and spec.namespace != args.namespace: - tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, - args.namespace)) + tty.die("Namespaces '{0}' and '{1}' do not match.".format( + spec.namespace, args.namespace)) if not spec.namespace and args.namespace: spec.namespace = args.namespace - # Figure out where the new package should live. + # Figure out where the new package should live repo_path = args.repo if repo_path is not None: - try: - repo = Repo(repo_path) - if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with " - "namespace %s" % (spec.namespace, repo.namespace)) - except RepoError as e: - tty.die(str(e)) + repo = Repo(repo_path) + if spec.namespace and spec.namespace != repo.namespace: + tty.die("Can't create package with namespace {0} in repo with " + "namespace {0}".format(spec.namespace, repo.namespace)) else: if spec.namespace: repo = spack.repo.get_repo(spec.namespace, None) if not repo: - tty.die("Unknown namespace: %s" % spec.namespace) + tty.die("Unknown namespace: '{0}'".format(spec.namespace)) else: repo = spack.repo.first_repo() @@ -388,84 +528,30 @@ def find_repository(spec, args): return repo -def fetch_tarballs(url, name, version): - """Try to find versions of the supplied archive by scraping the web. - Prompts the user to select how many to download if many are found.""" - versions = spack.util.web.find_versions_of_archive(url) - rkeys = sorted(versions.keys(), reverse=True) - versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) - - archives_to_fetch = 1 - if not versions: - # If the fetch failed for some reason, revert to what the user provided - versions = {version: url} - elif len(versions) > 1: - tty.msg("Found %s versions of %s:" % (len(versions), name), - *spack.cmd.elide_list( - ["%-10s%s" % (v, u) for v, u in versions.iteritems()])) - print('') - archives_to_fetch = tty.get_number( - "Include how many checksums in the package file?", - default=5, abort='q') - - if not archives_to_fetch: - tty.die("Aborted.") - - sorted_versions = sorted(versions.keys(), reverse=True) - sorted_urls = [versions[v] for v in sorted_versions] - return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch] - - def create(parser, args): - url = args.url - if not url: - setup_parser.subparser.print_help() - return - - # Figure out a name and repo for the package. - name, version = guess_name_and_version(url, args) - spec = Spec(name) - repo = find_repository(spec, args) - - tty.msg("This looks like a URL for %s version %s" % (name, version)) - tty.msg("Creating template for package %s" % name) - - # Fetch tarballs (prompting user if necessary) - versions, urls = fetch_tarballs(url, name, version) - - # Try to guess what build system is used. - guesser = BuildSystemGuesser() - ver_hash_tuples = spack.cmd.checksum.get_checksums( - versions, urls, - first_stage_function=guesser, - keep_stage=args.keep_stage) - - if not ver_hash_tuples: - tty.die("Could not fetch any tarballs for %s" % name) - - guess = guesser.make_guess(name, url, ver_hash_tuples) - - # Create a directory for the new package. - pkg_path = repo.filename_for_package_name(guess.name) + # Gather information about the package to be created + name = get_name(args) + url = get_url(args) + versions, guesser = get_versions(args, name) + build_system = get_build_system(args, guesser) + + # Create the package template object + PackageClass = templates[build_system] + package = PackageClass(name, url, versions) + tty.msg("Created template for {0} package".format(package.name)) + + # Create a directory for the new package + repo = get_repository(args, name) + pkg_path = repo.filename_for_package_name(package.name) if os.path.exists(pkg_path) and not args.force: - tty.die("%s already exists." % pkg_path) + tty.die('{0} already exists.'.format(pkg_path), + ' Try running `spack create --force` to overwrite it.') else: mkdirp(os.path.dirname(pkg_path)) - # Write out a template for the file - with open(pkg_path, "w") as pkg_file: - pkg_file.write( - package_template.substitute( - name=guess.name, - class_name=guess.class_name, - base_class_name=guess.base_class_name, - url=guess.url, - versions=guess.versions, - dependencies=guess.dependencies, - body=guess.body - ) - ) - - # If everything checks out, go ahead and edit. + # Write the new package file + package.write(pkg_path) + tty.msg("Created package file: {0}".format(pkg_path)) + + # Open up the new package file in your $EDITOR spack.editor(pkg_path) - tty.msg("Created package %s" % pkg_path) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 22966a26eb..dbb5a253ec 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -31,7 +31,6 @@ import llnl.util.tty as tty import spack import spack.cmd import spack.cmd.common.arguments as arguments -from spack.cmd.edit import edit_package from spack.stage import DIYStage description = "Do-It-Yourself: build from an existing source directory." @@ -68,15 +67,8 @@ def diy(self, args): spec = specs[0] if not spack.repo.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, spack.repo.first_repo(), None, True) - return + tty.die("No package for '{0}' was found.".format(spec.name), + " Use `spack create` to create a new package") if not spec.versions.concrete: tty.die( diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index 286136dd67..77f23333b6 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -23,39 +23,26 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import string import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp, join_path +from llnl.util.filesystem import join_path import spack import spack.cmd from spack.spec import Spec from spack.repository import Repo -from spack.util.naming import mod_to_class description = "Open package files in $EDITOR" -# When -f is supplied, we'll create a very minimal skeleton. -package_template = string.Template("""\ -from spack import * -class ${class_name}(Package): - ""\"Description""\" +def edit_package(name, repo_path, namespace): + """Opens the requested package file in your favorite $EDITOR. - homepage = "http://www.example.com" - url = "http://www.example.com/${name}-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") -""") - - -def edit_package(name, repo_path, namespace, force=False): + :param str name: The name of the package + :param str repo_path: The path to the repository containing this package + :param str namespace: A valid namespace registered with Spack + """ + # Find the location of the package if repo_path: repo = Repo(repo_path) elif namespace: @@ -67,37 +54,29 @@ def edit_package(name, repo_path, namespace, force=False): spec = Spec(name) if os.path.exists(path): if not os.path.isfile(path): - tty.die("Something's wrong. '%s' is not a file!" % path) + tty.die("Something is wrong. '{0}' is not a file!".format(path)) if not os.access(path, os.R_OK | os.W_OK): tty.die("Insufficient permissions on '%s'!" % path) - elif not force: - tty.die("No package '%s'. Use spack create, or supply -f/--force " - "to edit a new file." % spec.name) else: - mkdirp(os.path.dirname(path)) - with open(path, "w") as pkg_file: - pkg_file.write( - package_template.substitute( - name=spec.name, class_name=mod_to_class(spec.name))) + tty.die("No package for '{0}' was found.".format(spec.name), + " Use `spack create` to create a new package") spack.editor(path) def setup_parser(subparser): - subparser.add_argument( - '-f', '--force', dest='force', action='store_true', - help="Open a new file in $EDITOR even if package doesn't exist.") - excl_args = subparser.add_mutually_exclusive_group() - # Various filetypes you can edit directly from the cmd line. + # Various types of Spack files that can be edited + # Edits package files by default excl_args.add_argument( '-c', '--command', dest='path', action='store_const', const=spack.cmd.command_path, help="Edit the command with the supplied name.") excl_args.add_argument( '-t', '--test', dest='path', action='store_const', - const=spack.test_path, help="Edit the test with the supplied name.") + const=spack.test_path, + help="Edit the test with the supplied name.") excl_args.add_argument( '-m', '--module', dest='path', action='store_const', const=spack.module_path, @@ -112,23 +91,26 @@ def setup_parser(subparser): help="Namespace of package to edit.") subparser.add_argument( - 'name', nargs='?', default=None, help="name of package to edit") + 'name', nargs='?', default=None, + help="name of package to edit") def edit(parser, args): name = args.name + # By default, edit package files path = spack.packages_path + + # If `--command`, `--test`, or `--module` is chosen, edit those instead if args.path: path = args.path if name: path = join_path(path, name + ".py") - if not args.force and not os.path.exists(path): - tty.die("No command named '%s'." % name) + if not os.path.exists(path): + tty.die("No command for '{0}' was found.".format(name)) spack.editor(path) - elif name: - edit_package(name, args.repo, args.namespace, args.force) + edit_package(name, args.repo, args.namespace) else: - # By default open the directory where packages or commands live. + # By default open the directory where packages live spack.editor(path) diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index 746674ad32..5d8aaefa72 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -36,7 +36,6 @@ import spack.cmd.install as install import spack.cmd.common.arguments as arguments from llnl.util.filesystem import set_executable from spack import which -from spack.cmd.edit import edit_package from spack.stage import DIYStage description = "Create a configuration script and module, but don't build." @@ -134,16 +133,8 @@ def setup(self, args): with spack.store.db.write_transaction(): spec = specs[0] if not spack.repo.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, spack.repo.first_repo(), None, True) - return - + tty.die("No package for '{0}' was found.".format(spec.name), + " Use `spack create` to create a new package") if not spec.versions.concrete: tty.die( "spack setup spec must have a single, concrete version. " diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py index 86c1c9da13..82bf1964b2 100644 --- a/lib/spack/spack/test/build_system_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -32,12 +32,13 @@ import spack.stage @pytest.fixture( scope='function', params=[ - ('configure', 'autotools'), + ('configure', 'autotools'), ('CMakeLists.txt', 'cmake'), - ('SConstruct', 'scons'), - ('setup.py', 'python'), - ('NAMESPACE', 'r'), - ('foobar', 'unknown') + ('SConstruct', 'scons'), + ('setup.py', 'python'), + ('NAMESPACE', 'r'), + ('WORKSPACE', 'bazel'), + ('foobar', 'generic') ] ) def url_and_build_system(request, tmpdir): -- cgit v1.2.3-60-g2f50