From e9f42c1a94f58e5fa916705f67716c26818a7eae Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 10:21:15 -0500 Subject: Allow spack create to automatically detect octave build system --- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/create.py | 145 +++++++++++++++++++++------------------- 2 files changed, 78 insertions(+), 69 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 95bd4771ed..9cdb0bd012 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -59,7 +59,7 @@ def get_checksums(versions, urls, **kwargs): with Stage(url, keep=keep_stage) as stage: stage.fetch() if i == 0 and first_stage_function: - first_stage_function(stage) + first_stage_function(stage, url) hashes.append((version, spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index c9fa687b74..009b2f308b 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -103,6 +103,64 @@ ${versions} ${install} """) +# Build dependencies and extensions +dependencies_dict = { + 'autotools': "# depends_on('foo')", + 'cmake': "depends_on('cmake')", + 'scons': "depends_on('scons')", + 'python': "extends('python')", + 'R': "extends('R')", + 'octave': "extends('octave')", + 'unknown': "# depends_on('foo')" +} + +# Default installation instructions +install_dict = { + 'autotools': """\ + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix={0}'.format(prefix)) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'cmake': """\ + with working_dir('spack-build', create=True): + # FIXME: Modify the cmake line to suit your build system here. + cmake('..', *std_cmake_args) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'scons': """\ + # FIXME: Add logic to build and install here. + scons('prefix={0}'.format(prefix)) + scons('install')""", + + 'python': """\ + # FIXME: Add logic to build and install here. + python('setup.py', 'install', '--prefix={0}'.format(prefix))""", + + 'R': """\ + # FIXME: Add logic to build and install here. + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path)""", + + 'octave': """\ + # FIXME: Add logic to build and install here. + octave('--quiet', '--norc', + '--built-in-docstrings-file=/dev/null', + '--texi-macros-file=/dev/null', + '--eval', 'pkg prefix {0}; pkg install {1}'.format( + prefix, self.stage.archive_file))""", + + 'unknown': """\ + # FIXME: Unknown build system + make() + make('install')""" +} + def make_version_calls(ver_hash_tuples): """Adds a version() call to the package for each version found.""" @@ -133,60 +191,17 @@ def setup_parser(subparser): setup_parser.subparser = subparser -class ConfigureGuesser(object): - def __call__(self, stage): - """Try to guess the type of build system used by the project. - Set any necessary build dependencies or extensions. - Set the appropriate default installation instructions.""" - - # Build dependencies and extensions - dependenciesDict = { - 'autotools': "# depends_on('foo')", - 'cmake': "depends_on('cmake', type='build')", - 'scons': "depends_on('scons', type='build')", - 'python': "extends('python', type=nolink)", - 'R': "extends('R')", - 'unknown': "# depends_on('foo')" - } - - # Default installation instructions - installDict = { - 'autotools': """\ - # FIXME: Modify the configure line to suit your build system here. - configure('--prefix={0}'.format(prefix)) - - # FIXME: Add logic to build and install here. - make() - make('install')""", - - 'cmake': """\ - with working_dir('spack-build', create=True): - # FIXME: Modify the cmake line to suit your build system here. - cmake('..', *std_cmake_args) +class BuildSystemGuesser(object): + def __call__(self, stage, url): + """Try to guess the type of build system used by a project based on + the contents of its archive or the URL it was downloaded from.""" - # FIXME: Add logic to build and install here. - make() - make('install')""", - - 'scons': """\ - # FIXME: Add logic to build and install here. - scons('prefix={0}'.format(prefix)) - scons('install')""", - - 'python': """\ - # FIXME: Add logic to build and install here. - python('setup.py', 'install', '--prefix={0}'.format(prefix))""", - - 'R': """\ - # FIXME: Add logic to build and install here. - R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), - self.stage.source_path)""", - - 'unknown': """\ - # FIXME: Unknown build system - make() - make('install')""" - } + # Most octave extensions are hosted on Octave-Forge: + # http://octave.sourceforge.net/index.html + # They all have the same base URL. + if 'downloads.sourceforge.net/octave/' in url: + self.build_system = 'octave' + return # A list of clues that give us an idea of the build system a package # uses. If the regular expression matches a file contained in the @@ -224,12 +239,6 @@ class ConfigureGuesser(object): self.build_system = build_system - # Set any necessary build dependencies or extensions. - self.dependencies = dependenciesDict[build_system] - - # Set the appropriate default installation instructions - self.install = installDict[build_system] - def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -334,8 +343,8 @@ def create(parser, args): # Fetch tarballs (prompting user if necessary) versions, urls = fetch_tarballs(url, name, version) - # Try to guess what configure system is used. - guesser = ConfigureGuesser() + # Try to guess what build system is used. + guesser = BuildSystemGuesser() ver_hash_tuples = spack.cmd.checksum.get_checksums( versions, urls, first_stage_function=guesser, @@ -344,13 +353,13 @@ def create(parser, args): if not ver_hash_tuples: tty.die("Could not fetch any tarballs for %s" % name) - # Prepend 'py-' to python package names, by convention. + # Add prefix to package name if it is an extension. if guesser.build_system == 'python': - name = 'py-%s' % name - - # Prepend 'r-' to R package names, by convention. + name = 'py-{0}'.format(name) if guesser.build_system == 'R': - name = 'r-%s' % name + name = 'r-{0}'.format(name) + if guesser.build_system == 'octave': + name = 'octave-{0}'.format(name) # Create a directory for the new package. pkg_path = repo.filename_for_package_name(name) @@ -367,8 +376,8 @@ def create(parser, args): class_name=mod_to_class(name), url=url, versions=make_version_calls(ver_hash_tuples), - dependencies=guesser.dependencies, - install=guesser.install)) + dependencies=dependencies_dict[guesser.build_system], + install=install_dict[guesser.build_system])) # If everything checks out, go ahead and edit. spack.editor(pkg_path) -- cgit v1.2.3-60-g2f50 From 199a8af7cc4e75e66cf53c3a2bb4c9b28aec64f1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 10:39:40 -0500 Subject: Flake8 --- lib/spack/spack/cmd/checksum.py | 19 +++++++++++-------- lib/spack/spack/cmd/create.py | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 9cdb0bd012..aedb0fd99c 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -42,7 +42,8 @@ def setup_parser(subparser): '--keep-stage', action='store_true', dest='keep_stage', help="Don't clean up staging area when command completes.") subparser.add_argument( - 'versions', nargs=argparse.REMAINDER, help='Versions to generate checksums for') + 'versions', nargs=argparse.REMAINDER, + help='Versions to generate checksums for') def get_checksums(versions, urls, **kwargs): @@ -61,8 +62,8 @@ def get_checksums(versions, urls, **kwargs): if i == 0 and first_stage_function: first_stage_function(stage, url) - hashes.append((version, - spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) + hashes.append((version, spack.util.crypto.checksum( + hashlib.md5, stage.archive_file))) i += 1 except FailedDownloadError as e: tty.msg("Failed to fetch %s" % url) @@ -79,12 +80,12 @@ def checksum(parser, args): # If the user asked for specific versions, use those. if args.versions: versions = {} - for v in args.versions: - v = ver(v) - if not isinstance(v, Version): + for version in args.versions: + version = ver(version) + if not isinstance(version, Version): tty.die("Cannot generate checksums for version lists or " + "version ranges. Use unambiguous versions.") - versions[v] = pkg.url_for_version(v) + versions[version] = pkg.url_for_version(version) else: versions = pkg.fetch_remote_versions() if not versions: @@ -111,5 +112,7 @@ def checksum(parser, args): if not version_hashes: tty.die("Could not fetch any versions for %s" % pkg.name) - version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes] + version_lines = [ + " version('%s', '%s')" % (v, h) for v, h in version_hashes + ] tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 009b2f308b..2c440096d1 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -124,7 +124,7 @@ install_dict = { make() make('install')""", - 'cmake': """\ + 'cmake': """\ with working_dir('spack-build', create=True): # FIXME: Modify the cmake line to suit your build system here. cmake('..', *std_cmake_args) -- cgit v1.2.3-60-g2f50 From 262ab401884a8b46992872b8c010fa844cb14271 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 11:33:29 -0500 Subject: Update build_system_guess test with new class name --- lib/spack/spack/test/__init__.py | 2 +- lib/spack/spack/test/build_system_guess.py | 89 ++++++++++++++++++++++++++++++ lib/spack/spack/test/configure_guess.py | 83 ---------------------------- 3 files changed, 90 insertions(+), 84 deletions(-) create mode 100644 lib/spack/spack/test/build_system_guess.py delete mode 100644 lib/spack/spack/test/configure_guess.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index fb91f24721..19bc1d89d3 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -38,7 +38,7 @@ test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'pack 'directory_layout', 'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', 'optional_deps', - 'make_executable', 'configure_guess', 'lock', 'database', + 'make_executable', 'build_system_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py new file mode 100644 index 0000000000..80b867a528 --- /dev/null +++ b/lib/spack/spack/test/build_system_guess.py @@ -0,0 +1,89 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import shutil +import tempfile +import unittest + +from llnl.util.filesystem import * +from spack.cmd.create import BuildSystemGuesser +from spack.stage import Stage +from spack.test.mock_packages_test import * +from spack.util.executable import which + + +class InstallTest(unittest.TestCase): + """Tests the build system guesser in spack create""" + + def setUp(self): + self.tar = which('tar') + self.tmpdir = tempfile.mkdtemp() + self.orig_dir = os.getcwd() + os.chdir(self.tmpdir) + self.stage = None + + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + os.chdir(self.orig_dir) + + + def check_archive(self, filename, system): + mkdirp('archive') + touch(join_path('archive', filename)) + self.tar('czf', 'archive.tar.gz', 'archive') + + url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') + print url + with Stage(url) as stage: + stage.fetch() + + guesser = BuildSystemGuesser() + guesser(stage, url) + self.assertEqual(system, guesser.build_system) + + + def test_autotools(self): + self.check_archive('configure', 'autotools') + + + def test_cmake(self): + self.check_archive('CMakeLists.txt', 'cmake') + + + def test_scons(self): + self.check_archive('SConstruct', 'scons') + + + def test_python(self): + self.check_archive('setup.py', 'python') + + + def test_R(self): + self.check_archive('NAMESPACE', 'R') + + + def test_unknown(self): + self.check_archive('foobar', 'unknown') diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py deleted file mode 100644 index bad3673e7a..0000000000 --- a/lib/spack/spack/test/configure_guess.py +++ /dev/null @@ -1,83 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import shutil -import tempfile -import unittest - -from llnl.util.filesystem import * -from spack.cmd.create import ConfigureGuesser -from spack.stage import Stage -from spack.test.mock_packages_test import * -from spack.util.executable import which - - -class InstallTest(unittest.TestCase): - """Tests the configure guesser in spack create""" - - def setUp(self): - self.tar = which('tar') - self.tmpdir = tempfile.mkdtemp() - self.orig_dir = os.getcwd() - os.chdir(self.tmpdir) - self.stage = None - - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - os.chdir(self.orig_dir) - - - def check_archive(self, filename, system): - mkdirp('archive') - touch(join_path('archive', filename)) - self.tar('czf', 'archive.tar.gz', 'archive') - - url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') - print url - with Stage(url) as stage: - stage.fetch() - - guesser = ConfigureGuesser() - guesser(stage) - self.assertEqual(system, guesser.build_system) - - - def test_python(self): - self.check_archive('setup.py', 'python') - - - def test_autotools(self): - self.check_archive('configure', 'autotools') - - - def test_cmake(self): - self.check_archive('CMakeLists.txt', 'cmake') - - - def test_unknown(self): - self.check_archive('foobar', 'unknown') - - -- cgit v1.2.3-60-g2f50 From 89c9bec81ec3afa96ba25dd6aea9727abc8a8f0f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 11:36:07 -0500 Subject: Flake8 --- lib/spack/spack/test/__init__.py | 19 ++++++++++--------- lib/spack/spack/test/build_system_guess.py | 8 -------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 19bc1d89d3..9f7da46a1a 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -32,15 +32,16 @@ from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage', - 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', - 'multimethod', 'install', 'package_sanity', 'config', - 'directory_layout', 'pattern', 'python_version', 'git_fetch', - 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', - 'cc', 'link_tree', 'spec_yaml', 'optional_deps', - 'make_executable', 'build_system_guess', 'lock', 'database', - 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', - 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] +test_names = [ + 'architecture', 'versions', 'url_parse', 'url_substitution', 'packages', + 'stage', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', + 'multimethod', 'install', 'package_sanity', 'config', 'directory_layout', + 'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch', + 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', + 'optional_deps', 'make_executable', 'build_system_guess', 'lock', + 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', + 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd' +] def list_tests(): diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py index 80b867a528..e728a47cf4 100644 --- a/lib/spack/spack/test/build_system_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -44,12 +44,10 @@ class InstallTest(unittest.TestCase): os.chdir(self.tmpdir) self.stage = None - def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) os.chdir(self.orig_dir) - def check_archive(self, filename, system): mkdirp('archive') touch(join_path('archive', filename)) @@ -64,26 +62,20 @@ class InstallTest(unittest.TestCase): guesser(stage, url) self.assertEqual(system, guesser.build_system) - def test_autotools(self): self.check_archive('configure', 'autotools') - def test_cmake(self): self.check_archive('CMakeLists.txt', 'cmake') - def test_scons(self): self.check_archive('SConstruct', 'scons') - def test_python(self): self.check_archive('setup.py', 'python') - def test_R(self): self.check_archive('NAMESPACE', 'R') - def test_unknown(self): self.check_archive('foobar', 'unknown') -- cgit v1.2.3-60-g2f50