diff options
author | Adam J. Stewart <ajstewart426@gmail.com> | 2017-04-03 17:34:16 -0500 |
---|---|---|
committer | Todd Gamblin <tgamblin@llnl.gov> | 2017-04-03 15:34:16 -0700 |
commit | 50df071ad9a1b936ffbb4121036a36ed5ab38ffa (patch) | |
tree | a35d73aec9ef2ec615e0d1fc93a6fcac4066818c /lib/spack | |
parent | 7e9777f294a0a672f550e2dede033e4dc3e5dfda (diff) | |
download | spack-50df071ad9a1b936ffbb4121036a36ed5ab38ffa.tar.gz spack-50df071ad9a1b936ffbb4121036a36ed5ab38ffa.tar.bz2 spack-50df071ad9a1b936ffbb4121036a36ed5ab38ffa.tar.xz spack-50df071ad9a1b936ffbb4121036a36ed5ab38ffa.zip |
Overhaul Spack's URL parsing (#2972)
* Remove fake URLs from Spack
* Ignore long lines for URLs that start with ftp:
* Preliminary changes to version regexes
* New redesign of version regexes
* Allow letters in version-only
* Fix detection of versions that end in Final
* Rearrange a few regexes and add examples
* Add tests for common download repositories
* Add test cases for common tarball naming schemes
* Finalize version regexes
* spack url test -> spack url summary
* Clean up comments
* Rearrange suffix checks
* Use query strings for name detection
* Remove no longer necessary url_for_version functions
* Strip off extraneous information after package name
* Add one more test
* Dot in square brackets does not need to be escaped
* Move renaming outside of parse_name_offset
* Fix versions for a couple more packages
* Fix flake8 and doc tests
* Correctly parse Python, Lua, and Bio++ package names
* Use effective URLs for mfem
* Add checksummed version to mitos
* Remove url_for_version from STAR-CCM+ package
* Revert changes to version numbers with underscores and dashes
* Fix name detection for tbb
* Correctly parse Ruby gems
* Reverted mfem back to shortened URLs.
* Updated instructions for better security
* Remove preferred=True from newest version
* Add tests for new `spack url list` flags
* Add tests for strip_name_suffixes
* Add unit tests for version separators
* Fix bugs related to parseable name but in parseable version
* Remove dead code, update docstring
* Ignore 'binary' at end of version string
* Remove platform from version
* Flip libedit version numbers
* Re-support weird NCO alpha/beta versions
* Rebase and remove one new fake URL
* Add / to beginning of regex to avoid picking up similarly named packages
* Ignore weird tar versions
* Fix bug in url parse --spider when no versions found
* Less strict version matching for spack versions
* Don't rename Python packages
* Be a little more selective, version must begin with a digit
* Re-add fake URLs
* Fix up several other packages
* Ignore more file endings
* Add parsing support for Miniconda
* Update tab completion
* XFAILS are now PASSES for 2 web tests
Diffstat (limited to 'lib/spack')
-rw-r--r-- | lib/spack/docs/developer_guide.rst | 8 | ||||
-rw-r--r-- | lib/spack/spack/cmd/create.py | 25 | ||||
-rw-r--r-- | lib/spack/spack/cmd/flake8.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/cmd/url.py | 92 | ||||
-rw-r--r-- | lib/spack/spack/cmd/versions.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/cmd/url.py | 21 | ||||
-rw-r--r-- | lib/spack/spack/test/url_extrapolate.py | 101 | ||||
-rw-r--r-- | lib/spack/spack/test/url_parse.py | 794 | ||||
-rw-r--r-- | lib/spack/spack/test/url_substitution.py | 84 | ||||
-rw-r--r-- | lib/spack/spack/test/web.py | 3 | ||||
-rw-r--r-- | lib/spack/spack/url.py | 513 | ||||
-rw-r--r-- | lib/spack/spack/util/naming.py | 45 | ||||
-rw-r--r-- | lib/spack/spack/util/web.py | 12 | ||||
-rw-r--r-- | lib/spack/spack/version.py | 29 |
14 files changed, 1166 insertions, 565 deletions
diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 0ce4029950..ea8d50c6ca 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -447,16 +447,16 @@ the string that it detected to be the name and version. The ``--incorrect-name`` and ``--incorrect-version`` flags can be used to print URLs that were not being parsed correctly. -"""""""""""""""""" -``spack url test`` -"""""""""""""""""" +""""""""""""""""""""" +``spack url summary`` +""""""""""""""""""""" This command attempts to parse every URL for every package in Spack and prints a summary of how many of them are being correctly parsed. It also prints a histogram showing which regular expressions are being matched and how frequently: -.. command-output:: spack url test +.. command-output:: spack url summary This command is essential for anyone adding or changing the regular expressions that parse names and versions. By running this command diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index cc90669158..906c7e1aec 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -31,13 +31,13 @@ import llnl.util.tty as tty import spack import spack.cmd import spack.cmd.checksum -import spack.url import spack.util.web from llnl.util.filesystem import mkdirp from spack.repository import Repo from spack.spec import Spec from spack.util.executable import which from spack.util.naming import * +from spack.url import * description = "create a new package file" @@ -382,6 +382,10 @@ class BuildSystemGuesser: can take a peek at the fetched tarball and discern the build system it uses """ + def __init__(self): + """Sets the default build system.""" + self.build_system = 'generic' + def __call__(self, stage, url): """Try to guess the type of build system used by a project based on the contents of its archive or the URL it was downloaded from.""" @@ -427,14 +431,11 @@ class BuildSystemGuesser: # Determine the build system based on the files contained # in the archive. - build_system = 'generic' for pattern, bs in clues: if any(re.search(pattern, l) for l in lines): - build_system = bs + self.build_system = bs break - self.build_system = build_system - def get_name(args): """Get the name of the package based on the supplied arguments. @@ -458,9 +459,9 @@ def get_name(args): elif args.url: # Try to guess the package name based on the URL try: - name = spack.url.parse_name(args.url) + name = parse_name(args.url) tty.msg("This looks like a URL for {0}".format(name)) - except spack.url.UndetectableNameError: + except UndetectableNameError: tty.die("Couldn't guess a name for this package.", " Please report this bug. In the meantime, try running:", " `spack create --name <name> <url>`") @@ -515,11 +516,16 @@ def get_versions(args, name): if args.url: # Find available versions - url_dict = spack.util.web.find_versions_of_archive(args.url) + try: + url_dict = spack.util.web.find_versions_of_archive(args.url) + except UndetectableVersionError: + # Use fake versions + tty.warn("Couldn't detect version in: {0}".format(args.url)) + return versions, guesser if not url_dict: # If no versions were found, revert to what the user provided - version = spack.url.parse_version(args.url) + version = parse_version(args.url) url_dict = {version: args.url} versions = spack.cmd.checksum.get_checksums( @@ -611,6 +617,7 @@ def create(parser, args): url = get_url(args) versions, guesser = get_versions(args, name) build_system = get_build_system(args, guesser) + name = simplify_name(name) # Create the package template object PackageClass = templates[build_system] diff --git a/lib/spack/spack/cmd/flake8.py b/lib/spack/spack/cmd/flake8.py index 19724142bb..a6dc941190 100644 --- a/lib/spack/spack/cmd/flake8.py +++ b/lib/spack/spack/cmd/flake8.py @@ -70,7 +70,7 @@ exemptions = { # exemptions applied to all files. r'.py$': { # Exempt lines with URLs from overlong line errors. - 501: [r'(https?|file)\:'] + 501: [r'(https?|ftp|file)\:'] }, } diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py index 6823f0febd..1128e08a43 100644 --- a/lib/spack/spack/cmd/url.py +++ b/lib/spack/spack/cmd/url.py @@ -31,6 +31,7 @@ import spack from llnl.util import tty from spack.url import * from spack.util.web import find_versions_of_archive +from spack.util.naming import simplify_name description = "debugging tool for url parsing" @@ -66,19 +67,26 @@ def setup_parser(subparser): '-n', '--incorrect-name', action='store_true', help='only list urls for which the name was incorrectly parsed') excl_args.add_argument( + '-N', '--correct-name', action='store_true', + help='only list urls for which the name was correctly parsed') + excl_args.add_argument( '-v', '--incorrect-version', action='store_true', help='only list urls for which the version was incorrectly parsed') + excl_args.add_argument( + '-V', '--correct-version', action='store_true', + help='only list urls for which the version was correctly parsed') - # Test + # Summary sp.add_parser( - 'test', help='print a summary of how well we are parsing package urls') + 'summary', + help='print a summary of how well we are parsing package urls') def url(parser, args): action = { - 'parse': url_parse, - 'list': url_list, - 'test': url_test + 'parse': url_parse, + 'list': url_list, + 'summary': url_summary } action[args.subcommand](args) @@ -116,6 +124,10 @@ def url_parse(args): tty.msg('Spidering for versions:') versions = find_versions_of_archive(url) + if not versions: + print(' Found no versions for {0}'.format(name)) + return + max_len = max(len(str(v)) for v in versions) for v in sorted(versions): @@ -145,7 +157,7 @@ def url_list(args): return len(urls) -def url_test(args): +def url_summary(args): # Collect statistics on how many URLs were correctly parsed total_urls = 0 correct_names = 0 @@ -205,19 +217,19 @@ def url_test(args): correct_versions, total_urls, correct_versions / total_urls)) print() - tty.msg('Statistics on name regular expresions:') + tty.msg('Statistics on name regular expressions:') print() - print(' Index Count Regular Expresion') + print(' Index Count Regular Expression') for ni in name_regex_dict: print(' {0:>3}: {1:>6} r{2!r}'.format( ni, name_count_dict[ni], name_regex_dict[ni])) print() - tty.msg('Statistics on version regular expresions:') + tty.msg('Statistics on version regular expressions:') print() - print(' Index Count Regular Expresion') + print(' Index Count Regular Expression') for vi in version_regex_dict: print(' {0:>3}: {1:>6} r{2!r}'.format( vi, version_count_dict[vi], version_regex_dict[vi])) @@ -257,22 +269,38 @@ def url_list_parsing(args, urls, url, pkg): :rtype: set """ if url: - if args.incorrect_name: - # Only add URLs whose name was incorrectly parsed + if args.correct_name or args.incorrect_name: + # Attempt to parse the name try: name = parse_name(url) - if not name_parsed_correctly(pkg, name): + if (args.correct_name and + name_parsed_correctly(pkg, name)): + # Add correctly parsed URLs + urls.add(url) + elif (args.incorrect_name and + not name_parsed_correctly(pkg, name)): + # Add incorrectly parsed URLs urls.add(url) except UndetectableNameError: - urls.add(url) - elif args.incorrect_version: - # Only add URLs whose version was incorrectly parsed + if args.incorrect_name: + # Add incorrectly parsed URLs + urls.add(url) + elif args.correct_version or args.incorrect_version: + # Attempt to parse the version try: version = parse_version(url) - if not version_parsed_correctly(pkg, version): + if (args.correct_version and + version_parsed_correctly(pkg, version)): + # Add correctly parsed URLs + urls.add(url) + elif (args.incorrect_version and + not version_parsed_correctly(pkg, version)): + # Add incorrectly parsed URLs urls.add(url) except UndetectableVersionError: - urls.add(url) + if args.incorrect_version: + # Add incorrectly parsed URLs + urls.add(url) else: urls.add(url) @@ -289,6 +317,8 @@ def name_parsed_correctly(pkg, name): """ pkg_name = pkg.name + name = simplify_name(name) + # After determining a name, `spack create` determines a build system. # Some build systems prepend a special string to the front of the name. # Since this can't be guessed from the URL, it would be unfair to say @@ -311,9 +341,33 @@ def version_parsed_correctly(pkg, version): :returns: True if the name was correctly parsed, else False :rtype: bool """ + version = remove_separators(version) + # If the version parsed from the URL is listed in a version() # directive, we assume it was correctly parsed for pkg_version in pkg.versions: - if str(pkg_version) == str(version): + pkg_version = remove_separators(pkg_version) + if pkg_version == version: return True return False + + +def remove_separators(version): + """Removes separator characters ('.', '_', and '-') from a version. + + A version like 1.2.3 may be displayed as 1_2_3 in the URL. + Make sure 1.2.3, 1-2-3, 1_2_3, and 123 are considered equal. + Unfortunately, this also means that 1.23 and 12.3 are equal. + + :param version: A version + :type version: str or Version + :returns: The version with all separator characters removed + :rtype: str + """ + version = str(version) + + version = version.replace('.', '') + version = version.replace('_', '') + version = version.replace('-', '') + + return version diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index f65ef14406..a6f6805fb0 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -53,6 +53,6 @@ def versions(parser, args): tty.debug("Check the list_url and list_depth attribute on the " "package to help Spack find versions.") else: - print(" Found no unckecksummed versions for %s" % pkg.name) + print(" Found no unchecksummed versions for %s" % pkg.name) else: colify(sorted(remote_versions, reverse=True), indent=2) diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py index 4c60d814ce..3bc0bc7820 100644 --- a/lib/spack/spack/test/cmd/url.py +++ b/lib/spack/spack/test/cmd/url.py @@ -48,11 +48,12 @@ def test_name_parsed_correctly(): assert name_parsed_correctly(MyPackage('r-devtools', []), 'devtools') assert name_parsed_correctly(MyPackage('py-numpy', []), 'numpy') assert name_parsed_correctly(MyPackage('octave-splines', []), 'splines') + assert name_parsed_correctly(MyPackage('imagemagick', []), 'ImageMagick') # noqa + assert name_parsed_correctly(MyPackage('th-data', []), 'TH.data') # Expected False assert not name_parsed_correctly(MyPackage('', []), 'hdf5') assert not name_parsed_correctly(MyPackage('hdf5', []), '') - assert not name_parsed_correctly(MyPackage('imagemagick', []), 'ImageMagick') # noqa assert not name_parsed_correctly(MyPackage('yaml-cpp', []), 'yamlcpp') assert not name_parsed_correctly(MyPackage('yamlcpp', []), 'yaml-cpp') assert not name_parsed_correctly(MyPackage('r-py-parser', []), 'parser') @@ -64,6 +65,8 @@ def test_version_parsed_correctly(): assert version_parsed_correctly(MyPackage('', ['1.2.3']), '1.2.3') assert version_parsed_correctly(MyPackage('', ['5.4a', '5.4b']), '5.4a') assert version_parsed_correctly(MyPackage('', ['5.4a', '5.4b']), '5.4b') + assert version_parsed_correctly(MyPackage('', ['1.63.0']), '1_63_0') + assert version_parsed_correctly(MyPackage('', ['0.94h']), '094h') # Expected False assert not version_parsed_correctly(MyPackage('', []), '1.2.3') @@ -95,7 +98,7 @@ def test_url_list(parser): colored_urls = url_list(args) assert colored_urls == total_urls - # The following two options should print fewer URLs than the default. + # The following options should print fewer URLs than the default. # If they print the same number of URLs, something is horribly broken. # If they say we missed 0 URLs, something is probably broken too. args = parser.parse_args(['list', '--incorrect-name']) @@ -106,11 +109,19 @@ def test_url_list(parser): incorrect_version_urls = url_list(args) assert 0 < incorrect_version_urls < total_urls + args = parser.parse_args(['list', '--correct-name']) + correct_name_urls = url_list(args) + assert 0 < correct_name_urls < total_urls -def test_url_test(parser): - args = parser.parse_args(['test']) + args = parser.parse_args(['list', '--correct-version']) + correct_version_urls = url_list(args) + assert 0 < correct_version_urls < total_urls + + +def test_url_summary(parser): + args = parser.parse_args(['summary']) (total_urls, correct_names, correct_versions, - name_count_dict, version_count_dict) = url_test(args) + name_count_dict, version_count_dict) = url_summary(args) assert 0 < correct_names <= sum(name_count_dict.values()) <= total_urls # noqa assert 0 < correct_versions <= sum(version_count_dict.values()) <= total_urls # noqa diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py deleted file mode 100644 index 5f5cf555ae..0000000000 --- a/lib/spack/spack/test/url_extrapolate.py +++ /dev/null @@ -1,101 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -"""Tests ability of spack to extrapolate URL versions from -existing versions. -""" -import unittest - -import spack.url as url - - -class UrlExtrapolateTest(unittest.TestCase): - - def check_url(self, base, version, new_url): - self.assertEqual(url.substitute_version(base, version), new_url) - - def test_libelf_version(self): - base = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" - self.check_url(base, '0.8.13', base) - self.check_url( - base, '0.8.12', "http://www.mr511.de/software/libelf-0.8.12.tar.gz") - self.check_url( - base, '0.3.1', "http://www.mr511.de/software/libelf-0.3.1.tar.gz") - self.check_url( - base, '1.3.1b', "http://www.mr511.de/software/libelf-1.3.1b.tar.gz") - - def test_libdwarf_version(self): - base = "http://www.prevanders.net/libdwarf-20130729.tar.gz" - self.check_url(base, '20130729', base) - self.check_url( - base, '8.12', "http://www.prevanders.net/libdwarf-8.12.tar.gz") - - def test_dyninst_version(self): - # Dyninst has a version twice in the URL. - base = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz" - self.check_url(base, '8.1.2', base) - self.check_url(base, '8.2', - "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.2/DyninstAPI-8.2.tgz") - self.check_url(base, '8.3.1', - "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.3.1/DyninstAPI-8.3.1.tgz") - - def test_partial_version_prefix(self): - # Test now with a partial prefix earlier in the URL -- this is - # hard to figure out so Spack only substitutes the last - # instance of the version. - base = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.1.2.tgz" - self.check_url(base, '8.1.2', base) - self.check_url(base, '8.1.4', - "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.1.4.tgz") - self.check_url(base, '8.2', - "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.2.tgz") - self.check_url(base, '8.3.1', - "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.3.1.tgz") - - def test_scalasca_partial_version(self): - # Note that this probably doesn't actually work, but sites are - # inconsistent about their directory structure, so it's not - # clear what is right. This test is for consistency and to - # document behavior. If you figure out a good way to handle - # this case, fix the tests too. - self.check_url('http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz', '8.3.1', - 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-8.3.1.tar.gz') - self.check_url('http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz', '8.3.1', - 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-8.3.1.tar.gz') - - def test_mpileaks_version(self): - self.check_url('https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz', '2.1.3', - 'https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz') - - def test_gcc(self): - self.check_url('http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2', '4.7', - 'http://open-source-box.org/gcc/gcc-4.7/gcc-4.7.tar.bz2') - self.check_url('http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2', '4.4.7', - 'http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2') - - def test_github_raw(self): - self.check_url('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '2.0.7', - 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true') - self.check_url('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '4.7', - 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true') diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index 8913de94d0..2af7c6ae0b 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -22,246 +22,667 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -"""\ -This file has a bunch of versions tests taken from the excellent version -detection in Homebrew. -""" +"""Tests Spack's ability to parse the name and version of a package +based on its URL.""" + +import os import unittest -import spack.url as url +from spack.url import * -class UrlParseTest(unittest.TestCase): +class UrlStripVersionSuffixesTest(unittest.TestCase): + """Tests for spack.url.strip_version_suffixes""" - def assert_not_detected(self, string): - self.assertRaises( - url.UndetectableVersionError, url.parse_name_and_version, string) + def check(self, before, after): + stripped = strip_version_suffixes(before) + self.assertEqual(stripped, after) - def check(self, name, v, string, **kwargs): - # Make sure correct name and version are extracted. - parsed_name, parsed_v = url.parse_name_and_version(string) - self.assertEqual(parsed_name, name) - self.assertEqual(parsed_v, url.Version(v)) + def test_no_suffix(self): + self.check('rgb-1.0.6', + 'rgb-1.0.6') - # Some URLs (like boost) are special and need to override the - # built-in functionality. - if kwargs.get('no_check_url', False): - return + def test_misleading_prefix(self): + self.check('jpegsrc.v9b', + 'jpegsrc.v9b') + self.check('turbolinux702', + 'turbolinux702') + self.check('converge_install_2.3.16', + 'converge_install_2.3.16') - # Make sure Spack formulates the right URL when we try to - # build one with a specific version. - self.assertEqual(string, url.substitute_version(string, v)) + # Download type - def test_wwwoffle_version(self): - self.check( - 'wwwoffle', '2.9h', - 'http://www.gedanken.demon.co.uk/download-wwwoffle/wwwoffle-2.9h.tgz') + def test_src(self): + self.check('apache-ant-1.9.7-src', + 'apache-ant-1.9.7') + self.check('go1.7.4.src', + 'go1.7.4') + + def test_source(self): + self.check('bowtie2-2.2.5-source', + 'bowtie2-2.2.5') + self.check('grib_api-1.17.0-Source', + 'grib_api-1.17.0') + + def test_full(self): + self.check('julia-0.4.3-full', + 'julia-0.4.3') + + def test_bin(self): + self.check('apache-maven-3.3.9-bin', + 'apache-maven-3.3.9') + + def test_binary(self): + self.check('Jmol-14.8.0-binary', + 'Jmol-14.8.0') + + def test_gem(self): + self.check('rubysl-date-2.0.9.gem', + 'rubysl-date-2.0.9') + + def test_tar(self): + self.check('gromacs-4.6.1-tar', + 'gromacs-4.6.1') + + def test_sh(self): + self.check('Miniconda2-4.3.11-Linux-x86_64.sh', + 'Miniconda2-4.3.11') + + # Download version + + def test_stable(self): + self.check('libevent-2.0.21-stable', + 'libevent-2.0.21') + + def test_final(self): + self.check('2.6.7-final', + '2.6.7') + + def test_rel(self): + self.check('v1.9.5.1rel', + 'v1.9.5.1') + + def test_orig(self): + self.check('dash_0.5.5.1.orig', + 'dash_0.5.5.1') + + def test_plus(self): + self.check('ncbi-blast-2.6.0+-src', + 'ncbi-blast-2.6.0') + + # License + + def test_gpl(self): + self.check('cppad-20170114.gpl', + 'cppad-20170114') + + # OS + + def test_linux(self): + self.check('astyle_2.04_linux', + 'astyle_2.04') + + def test_unix(self): + self.check('install-tl-unx', + 'install-tl') + + def test_macos(self): + self.check('astyle_1.23_macosx', + 'astyle_1.23') + self.check('haxe-2.08-osx', + 'haxe-2.08') + + # PyPI + + def test_wheel(self): + self.check('entrypoints-0.2.2-py2.py3-none-any.whl', + 'entrypoints-0.2.2') + self.check('numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl', # noqa + 'numpy-1.12.0') + + def test_exe(self): + self.check('PyYAML-3.12.win-amd64-py3.5.exe', + 'PyYAML-3.12') + + # Combinations of multiple patterns + + def test_complex_all(self): + self.check('p7zip_9.04_src_all', + 'p7zip_9.04') + + def test_complex_run(self): + self.check('cuda_8.0.44_linux.run', + 'cuda_8.0.44') + + def test_complex_file(self): + self.check('ack-2.14-single-file', + 'ack-2.14') + + def test_complex_jar(self): + self.check('antlr-3.4-complete.jar', + 'antlr-3.4') + + def test_complex_oss(self): + self.check('tbb44_20160128oss_src_0', + 'tbb44_20160128') + + def test_complex_darwin(self): + self.check('ghc-7.0.4-x86_64-apple-darwin', + 'ghc-7.0.4') + self.check('ghc-7.0.4-i386-apple-darwin', + 'ghc-7.0.4') + + def test_complex_arch(self): + self.check('VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install', + 'VizGlow_v2.2alpha17-R21November2016') + self.check('jdk-8u92-linux-x64', + 'jdk-8u92') + self.check('cuda_6.5.14_linux_64.run', + 'cuda_6.5.14') + + def test_complex_with(self): + self.check('mafft-7.221-with-extensions-src', + 'mafft-7.221') + self.check('spark-2.0.0-bin-without-hadoop', + 'spark-2.0.0') + + def test_complex_public(self): + self.check('dakota-6.3-public.src', + 'dakota-6.3') + + def test_complex_universal(self): + self.check('synergy-1.3.6p2-MacOSX-Universal', + 'synergy-1.3.6p2') + + +class UrlStripNameSuffixesTest(unittest.TestCase): + """Tests for spack.url.strip_name_suffixes""" + + def check(self, before, version, after): + stripped = strip_name_suffixes(before, version) + self.assertEqual(stripped, after) + + def test_no_suffix(self): + self.check('rgb-1.0.6', '1.0.6', + 'rgb') + self.check('nauty26r7', '26r7', + 'nauty') + + # Download type + + def test_install(self): + self.check('converge_install_2.3.16', '2.3.16', + 'converge') + + def test_src(self): + self.check('jpegsrc.v9b', '9b', + 'jpeg') + + def test_std(self): + self.check('ghostscript-fonts-std-8.11', '8.11', + 'ghostscript-fonts') + + # Download version + + def test_snapshot(self): + self.check('gts-snapshot-121130', '121130', + 'gts') + + def test_distrib(self): + self.check('zoltan_distrib_v3.83', '3.83', + 'zoltan') + + # VCS - def test_version_sourceforge_download(self): + def test_bazaar(self): + self.check('libvterm-0+bzr681', '681', + 'libvterm') + + # License + + def test_gpl(self): + self.check('PyQt-x11-gpl-4.11.3', '4.11.3', + 'PyQt-x11') + + +class UrlParseOffsetTest(unittest.TestCase): + + def check(self, name, noffset, ver, voffset, path): + # Make sure parse_name_offset and parse_name_version are working + v, vstart, vlen, vi, vre = parse_version_offset(path) + n, nstart, nlen, ni, nre = parse_name_offset(path, v) + + self.assertEqual(n, name) + self.assertEqual(v, ver) + self.assertEqual(nstart, noffset) + self.assertEqual(vstart, voffset) + + def test_name_in_path(self): self.check( - 'foo-bar', '1.21', - 'http://sourceforge.net/foo_bar-1.21.tar.gz/download') + 'antlr', 25, '2.7.7', 40, + 'https://github.com/antlr/antlr/tarball/v2.7.7') + + def test_name_in_stem(self): self.check( - 'foo-bar', '1.21', - 'http://sf.net/foo_bar-1.21.tar.gz/download') + 'gmp', 32, '6.0.0a', 36, + 'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2') - def test_no_version(self): - self.assert_not_detected('http://example.com/blah.tar') - self.assert_not_detected('foo') + def test_name_in_suffix(self): + # Don't think I've ever seen one of these before + # We don't look for it, so it would probably fail anyway + pass - def test_version_all_dots(self): + def test_version_in_path(self): self.check( - 'foo-bar-la', '1.14', 'http://example.com/foo.bar.la.1.14.zip') + 'nextflow', 31, '0.20.1', 59, + 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow') - def test_version_underscore_separator(self): + def test_version_in_stem(self): self.check( - 'grc', '1.1', - 'http://example.com/grc_1.1.tar.gz') - - def test_boost_version_style(self): + 'zlib', 24, '1.2.10', 29, + 'http://zlib.net/fossils/zlib-1.2.10.tar.gz') + self.check( + 'slepc', 51, '3.6.2', 57, + 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz') self.check( - 'boost', '1.39.0', - 'http://example.com/boost_1_39_0.tar.bz2', - no_check_url=True) + 'cloog', 61, '0.18.1', 67, + 'http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz') + self.check( + 'libxc', 58, '2.2.2', 64, + 'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz') - def test_erlang_version_style(self): + def test_version_in_suffix(self): + self.check( + 'swiftsim', 36, '0.3.0', 76, + 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0') self.check( - 'otp', 'R13B', - 'http://erlang.org/download/otp_src_R13B.tar.gz') + 'sionlib', 30, '1.7.1', 59, + 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1') - def test_another_erlang_version_style(self): + def test_regex_in_name(self): self.check( - 'otp', 'R15B01', - 'https://github.com/erlang/otp/tarball/OTP_R15B01') + 'voro++', 40, '0.4.6', 47, + 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz') + + +class UrlParseNameAndVersionTest(unittest.TestCase): + + def assert_not_detected(self, string): + self.assertRaises( + UndetectableVersionError, parse_name_and_version, string) + + def check(self, name, v, string, **kwargs): + # Make sure correct name and version are extracted. + parsed_name, parsed_v = parse_name_and_version(string) + self.assertEqual(parsed_name, name) + self.assertEqual(parsed_v, Version(v)) + + # Make sure Spack formulates the right URL when we try to + # build one with a specific version. + self.assertEqual(string, substitute_version(string, v)) + + # Common Repositories - def test_yet_another_erlang_version_style(self): + def test_github_downloads(self): + # name/archive/ver.ver self.check( - 'otp', 'R15B03-1', - 'https://github.com/erlang/otp/tarball/OTP_R15B03-1') + 'nco', '4.6.2', + 'https://github.com/nco/nco/archive/4.6.2.tar.gz') + # name/archive/vver.ver + self.check( + 'vim', '8.0.0134', + 'https://github.com/vim/vim/archive/v8.0.0134.tar.gz') + # name/archive/name-ver.ver + self.check( + 'oce', '0.18', + 'https://github.com/tpaviot/oce/archive/OCE-0.18.tar.gz') + # name/releases/download/vver/name-ver.ver + self.check( + 'libmesh', '1.0.0', + 'https://github.com/libMesh/libmesh/releases/download/v1.0.0/libmesh-1.0.0.tar.bz2') + # name/tarball/vver.ver + self.check( + 'git', '2.7.1', + 'https://github.com/git/git/tarball/v2.7.1') + # name/zipball/vver.ver + self.check( + 'git', '2.7.1', + 'https://github.com/git/git/zipball/v2.7.1') - def test_p7zip_version_style(self): + def test_gitlab_downloads(self): + # name/repository/archive.ext?ref=vver.ver + self.check( + 'swiftsim', '0.3.0', + 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0') + # name/repository/archive.ext?ref=name-ver.ver self.check( - 'p7zip', '9.04', - 'http://kent.dl.sourceforge.net/sourceforge/p7zip/p7zip_9.04_src_all.tar.bz2') + 'icet', '1.2.3', + 'https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3') - def test_new_github_style(self): + def test_bitbucket_downloads(self): + # name/get/ver.ver self.check( - 'libnet', '1.1.4', - 'https://github.com/sam-github/libnet/tarball/libnet-1.1.4') + 'eigen', '3.2.7', + 'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2') + # name/get/vver.ver + self.check( + 'hoomd-blue', '1.3.3', + 'https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2') + # name/downloads/name-ver.ver + self.check( + 'dolfin', '2016.1.0', + 'https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-2016.1.0.tar.gz') - def test_gloox_beta_style(self): + def test_sourceforge_downloads(self): + # name-ver.ver self.check( - 'gloox', '1.0-beta7', - 'http://camaya.net/download/gloox-1.0-beta7.tar.bz2') + 'libpng', '1.6.27', + 'http://download.sourceforge.net/libpng/libpng-1.6.27.tar.gz') + self.check( + 'lcms2', '2.6', + 'http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz') + self.check( + 'modules', '3.2.10', + 'http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz') + # name-ver.ver.ext/download + self.check( + 'glew', '2.0.0', + 'https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download') - def test_sphinx_beta_style(self): + def test_cran_downloads(self): + # name.name_ver.ver-ver.ver self.check( - 'sphinx', '1.10-beta', - 'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz') + 'TH.data', '1.0-8', + 'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz') + self.check( + 'knitr', '1.14', + 'https://cran.rstudio.com/src/contrib/knitr_1.14.tar.gz') + self.check( + 'devtools', '1.12.0', + 'https://cloud.r-project.org/src/contrib/devtools_1.12.0.tar.gz') - def test_astyle_verson_style(self): + def test_pypi_downloads(self): + # name.name_name-ver.ver + self.check( + '3to2', '1.1.1', + 'https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip') self.check( - 'astyle', '1.23', - 'http://kent.dl.sourceforge.net/sourceforge/astyle/astyle_1.23_macosx.tar.gz') + 'mpmath', '0.19', + 'https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz') + self.check( + 'pandas', '0.16.0', + 'https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73') + self.check( + 'sphinx_rtd_theme', '0.1.10a0', + 'https://pypi.python.org/packages/da/6b/1b75f13d8aa3333f19c6cdf1f0bc9f52ea739cae464fbee050307c121857/sphinx_rtd_theme-0.1.10a0.tar.gz') + self.check( + 'backports.ssl_match_hostname', '3.5.0.1', + 'https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz') - def test_version_dos2unix(self): + def test_bazaar_downloads(self): self.check( - 'dos2unix', '3.1', - 'http://www.sfr-fresh.com/linux/misc/dos2unix-3.1.tar.gz') + 'libvterm', '681', + 'http://www.leonerd.org.uk/code/libvterm/libvterm-0+bzr681.tar.gz') - def test_version_internal_dash(self): + # Common Tarball Formats + + def test_version_only(self): + # ver.ver + self.check( + 'eigen', '3.2.7', + 'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2') + # ver.ver-ver + self.check( + 'ImageMagick', '7.0.2-7', + 'https://github.com/ImageMagick/ImageMagick/archive/7.0.2-7.tar.gz') + # vver.ver self.check( - 'foo-arse', '1.1-2', - 'http://example.com/foo-arse-1.1-2.tar.gz') + 'CGNS', '3.3.0', + 'https://github.com/CGNS/CGNS/archive/v3.3.0.tar.gz') + # vver_ver + self.check( + 'luafilesystem', '1_6_3', + 'https://github.com/keplerproject/luafilesystem/archive/v1_6_3.tar.gz') - def test_version_single_digit(self): + def test_no_separators(self): + # namever + self.check( + 'turbolinux', '702', + 'file://{0}/turbolinux702.tar.gz'.format(os.getcwd())) self.check( - 'foo-bar', '45', - 'http://example.com/foo_bar.45.tar.gz') + 'nauty', '26r7', + 'http://pallini.di.uniroma1.it/nauty26r7.tar.gz') - def test_noseparator_single_digit(self): + def test_dashes_only(self): + # name-name-ver-ver + self.check( + 'Trilinos', '12-10-1', + 'https://github.com/trilinos/Trilinos/archive/trilinos-release-12-10-1.tar.gz') + self.check( + 'panda', '2016-03-07', + 'http://comopt.ifi.uni-heidelberg.de/software/PANDA/downloads/panda-2016-03-07.tar') self.check( - 'foo-bar', '45', - 'http://example.com/foo_bar45.tar.gz') + 'gts', '121130', + 'http://gts.sourceforge.net/tarballs/gts-snapshot-121130.tar.gz') + self.check( + 'cdd', '061a', + 'http://www.cs.mcgill.ca/~fukuda/download/cdd/cdd-061a.tar.gz') - def test_version_developer_that_hates_us_format(self): + def test_underscores_only(self): + # name_name_ver_ver + self.check( + 'tinyxml', '2_6_2', + 'https://sourceforge.net/projects/tinyxml/files/tinyxml/2.6.2/tinyxml_2_6_2.tar.gz') + self.check( + 'boost', '1_55_0', + 'http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2') self.check( - 'foo-bar-la', '1.2.3', - 'http://example.com/foo-bar-la.1.2.3.tar.gz') + 'yorick', '2_2_04', + 'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz') + # name_namever_ver + self.check( + 'tbb', '44_20160413', + 'https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz') - def test_version_regular(self): + def test_dots_only(self): + # name.name.ver.ver + self.check( + 'prank', '150803', + 'http://wasabiapp.org/download/prank/prank.source.150803.tgz') + self.check( + 'jpeg', '9b', + 'http://www.ijg.org/files/jpegsrc.v9b.tar.gz') + self.check( + 'openjpeg', '2.1', + 'https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz') + # name.namever.ver + self.check( + 'atlas', '3.11.34', + 'http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2') self.check( - 'foo-bar', '1.21', - 'http://example.com/foo_bar-1.21.tar.gz') + 'visit', '2.10.1', + 'http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz') + self.check( + 'geant', '4.10.01.p03', + 'http://geant4.cern.ch/support/source/geant4.10.01.p03.tar.gz') + self.check( + 'tcl', '8.6.5', + 'http://prdownloads.sourceforge.net/tcl/tcl8.6.5-src.tar.gz') - def test_version_gitlab(self): + def test_dash_dot(self): + # name-name-ver.ver + # digit in name self.check( - 'vtk', '7.0.0', - 'https://gitlab.kitware.com/vtk/vtk/repository/' - 'archive.tar.bz2?ref=v7.0.0') + 'm4', '1.4.17', + 'https://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz') + # letter in version self.check( - 'icet', '1.2.3', - 'https://gitlab.kitware.com/icet/icet/repository/' - 'archive.tar.gz?ref=IceT-1.2.3') + 'gmp', '6.0.0a', + 'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2') + # version starts with 'v' + self.check( + 'LaunchMON', '1.0.2', + 'https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz') + # name-ver-ver.ver self.check( - 'foo', '42.1337', - 'http://example.com/org/foo/repository/' - 'archive.zip?ref=42.1337bar') + 'libedit', '20150325-3.1', + 'http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz') - def test_version_github(self): + def test_dash_underscore(self): + # name-name-ver_ver self.check( - 'yajl', '1.0.5', - 'http://github.com/lloyd/yajl/tarball/1.0.5') + 'icu4c', '57_1', + 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz') - def test_version_github_with_high_patch_number(self): + def test_underscore_dot(self): + # name_name_ver.ver self.check( - 'yajl', '1.2.34', - 'http://github.com/lloyd/yajl/tarball/v1.2.34') + 'superlu_dist', '4.1', + 'http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz') + self.check( + 'pexsi', '0.9.0', + 'https://math.berkeley.edu/~linlin/pexsi/download/pexsi_v0.9.0.tar.gz') + # name_name.ver.ver + self.check( + 'fer', '696', + 'ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.v696.tar.gz') - def test_yet_another_version(self): + def test_dash_dot_dash_dot(self): + # name-name-ver.ver-ver.ver + self.check( + 'sowing', '1.1.23-p1', + 'http://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz') self.check( - 'mad', '0.15.1b', - 'http://example.com/mad-0.15.1b.tar.gz') + 'bib2xhtml', '3.0-15-gf506', + 'http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz') + # namever.ver-ver.ver + self.check( + 'go', '1.4-bootstrap-20161024', + 'https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz') - def test_lame_version_style(self): + def test_underscore_dash_dot(self): + # name_name-ver.ver + self.check( + 'the_silver_searcher', '0.32.0', + 'http://geoff.greer.fm/ag/releases/the_silver_searcher-0.32.0.tar.gz') self.check( - 'lame', '398-2', - 'http://kent.dl.sourceforge.net/sourceforge/lame/lame-398-2.tar.gz') + 'sphinx_rtd_theme', '0.1.10a0', + 'https://pypi.python.org/packages/source/s/sphinx_rtd_theme/sphinx_rtd_theme-0.1.10a0.tar.gz') - def test_ruby_version_style(self): + def test_dot_underscore_dot_dash_dot(self): + # name.name_ver.ver-ver.ver self.check( - 'ruby', '1.9.1-p243', - 'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz') + 'TH.data', '1.0-8', + 'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz') + self.check( + 'XML', '3.98-1.4', + 'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz') - def test_omega_version_style(self): + def test_dash_dot_underscore_dot(self): + # name-name-ver.ver_ver.ver + self.check( + 'pypar', '2.1.5_108', + 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pypar/pypar-2.1.5_108.tgz') + # name-namever.ver_ver.ver self.check( - 'omega', '0.80.2', - 'http://www.alcyone.com/binaries/omega/omega-0.80.2-src.tar.gz') + 'STAR-CCM+', '11.06.010_02', + 'file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz'.format(os.getcwd())) - def test_rc_style(self): + # Weird URLS + + def test_version_in_path(self): + # github.com/repo/name/releases/download/name-vver/name self.check( - 'libvorbis', '1.2.2rc1', - 'http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2') + 'nextflow', '0.20.1', + 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow') - def test_dash_rc_style(self): + def test_suffix_queries(self): self.check( - 'js', '1.8.0-rc1', - 'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz') + 'swiftsim', '0.3.0', + 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0') + self.check( + 'sionlib', '1.7.1', + 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1') - def test_angband_version_style(self): + def test_stem_queries(self): + self.check( + 'slepc', '3.6.2', + 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz') self.check( - 'angband', '3.0.9b', - 'http://rephial.org/downloads/3.0/angband-3.0.9b-src.tar.gz') + 'otf', '1.12.5salmon', + 'http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz') - def test_stable_suffix(self): + def test_single_character_name(self): self.check( - 'libevent', '1.4.14b', - 'http://www.monkey.org/~provos/libevent-1.4.14b-stable.tar.gz') + 'R', '3.3.2', + 'https://cloud.r-project.org/src/base/R-3/R-3.3.2.tar.gz') + + def test_single_digit_version(self): + pass - def test_debian_style_1(self): + def test_name_starts_with_digit(self): self.check( - 'sl', '3.03', - 'http://ftp.de.debian.org/debian/pool/main/s/sl/sl_3.03.orig.tar.gz') + '3to2', '1.1.1', + 'https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip') - def test_debian_style_2(self): + def plus_in_name(self): self.check( - 'mmv', '1.01b', - 'http://ftp.de.debian.org/debian/pool/main/m/mmv/mmv_1.01b.orig.tar.gz') + 'gtk+', '2.24.31', + 'http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.31.tar.xz') + self.check( + 'voro++', '0.4.6', + 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz') + + def test_no_version(self): + self.assert_not_detected('http://www.netlib.org/blas/blast-forum/cblas.tgz') + self.assert_not_detected('http://www.netlib.org/voronoi/triangle.zip') - def test_imagemagick_style(self): + def test_download_php(self): + # Name comes before download.php + self.check( + 'sionlib', '1.7.1', + 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1') + # Ignore download.php + self.check( + 'slepc', '3.6.2', + 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz') self.check( - 'imagemagick', '6.7.5-7', + 'ScientificPython', '2.8.1', + 'https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz') - 'http://downloads.sf.net/project/machomebrew/mirror/ImageMagick-6.7.5-7.tar.bz2') + def test_gloox_beta_style(self): + self.check( + 'gloox', '1.0-beta7', + 'http://camaya.net/download/gloox-1.0-beta7.tar.bz2') - def test_dash_version_dash_style(self): + def test_sphinx_beta_style(self): self.check( - 'antlr', '3.4', - 'http://www.antlr.org/download/antlr-3.4-complete.jar') + 'sphinx', '1.10-beta', + 'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz') - def test_apache_version_style(self): + def test_ruby_version_style(self): self.check( - 'apache-cassandra', '1.2.0-rc2', - 'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz') + 'ruby', '1.9.1-p243', + 'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz') - def test_jpeg_style(self): + def test_rc_style(self): self.check( - 'jpegsrc', '8d', - 'http://www.ijg.org/files/jpegsrc.v8d.tar.gz') + 'libvorbis', '1.2.2rc1', + 'http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2') - def test_pypy_version(self): + def test_dash_rc_style(self): self.check( - 'pypy', '1.4.1', - 'http://pypy.org/download/pypy-1.4.1-osx.tar.bz2') + 'js', '1.8.0-rc1', + 'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz') - def test_openssl_version(self): + def test_apache_version_style(self): self.check( - 'openssl', '0.9.8s', - 'http://www.openssl.org/source/openssl-0.9.8s.tar.gz') + 'apache-cassandra', '1.2.0-rc2', + 'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz') def test_xaw3d_version(self): self.check( - 'xaw3d', '1.5E', + 'Xaw3d', '1.5E', 'ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz') def test_fann_version(self): @@ -269,16 +690,6 @@ class UrlParseTest(unittest.TestCase): 'fann', '2.1.0beta', 'http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip') - def test_iges_version(self): - self.check( - 'grads', '2.0.1', - 'ftp://iges.org/grads/2.0/grads-2.0.1-bin-darwin9.8-intel.tar.gz') - - def test_haxe_version(self): - self.check( - 'haxe', '2.08', - 'http://haxe.org/file/haxe-2.08-osx.tar.gz') - def test_imap_version(self): self.check( 'imap', '2007f', @@ -289,26 +700,6 @@ class UrlParseTest(unittest.TestCase): 'suite3270', '3.3.12ga7', 'http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz') - def test_synergy_version(self): - self.check( - 'synergy', '1.3.6p2', - 'http://synergy.googlecode.com/files/synergy-1.3.6p2-MacOSX-Universal.zip') - - def test_mvapich2_19_version(self): - self.check( - 'mvapich2', '1.9', - 'http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz') - - def test_mvapich2_20_version(self): - self.check( - 'mvapich2', '2.0', - 'http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - - def test_hdf5_version(self): - self.check( - 'hdf5', '1.8.13', - 'http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2') - def test_scalasca_version(self): self.check( 'cube', '4.2.3', @@ -317,55 +708,20 @@ class UrlParseTest(unittest.TestCase): 'cube', '4.3-TP1', 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz') - def test_mpileaks_version(self): - self.check( - 'mpileaks', '1.0', - 'https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz') - self.check( - 'mpileaks', '1.0', - 'https://github.com/hpc/mpileaks/releases/download/1.0/mpileaks-1.0.tar.gz') - - def test_gcc_version(self): - self.check( - 'gcc', '4.4.7', - 'http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2') - - def test_gcc_version_precedence(self): - # prefer the version in the tarball, not in the url prefix. - self.check( - 'gcc', '4.4.7', - 'http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.4.7.tar.bz2') - def test_github_raw_url(self): self.check( - 'powerparser', '2.0.7', + 'CLAMR', '2.0.7', 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true') - def test_r_xml_version(self): + def test_luaposix_version(self): self.check( - 'xml', '3.98-1.4', - 'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz') + 'luaposix', '33.4.0', + 'https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz') def test_nco_version(self): self.check( 'nco', '4.6.2-beta03', 'https://github.com/nco/nco/archive/4.6.2-beta03.tar.gz') - self.check( 'nco', '4.6.3-alpha04', 'https://github.com/nco/nco/archive/4.6.3-alpha04.tar.gz') - - def test_yorick_version(self): - self.check( - 'yorick', '2_2_04', - 'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz') - - def test_luaposix_version(self): - self.check( - 'luaposix', '33.4.0', - 'https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz') - - def test_sionlib_version(self): - self.check( - 'sionlib', '1.7.1', - 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1') diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index ea6374e3d2..449a3b29bf 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -22,44 +22,64 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -"""\ -This test does sanity checks on substituting new versions into URLs -""" +"""Tests Spack's ability to substitute a different version into a URL.""" + +import os import unittest -import spack.url as url +from spack.url import substitute_version + + +class UrlSubstitutionTest(unittest.TestCase): + def check(self, base, version, new_url): + self.assertEqual(substitute_version(base, version), new_url) -base = "https://comp.llnl.gov/linear_solvers/download/hypre-2.9.0b.tar.gz" -stem = "https://comp.llnl.gov/linear_solvers/download/hypre-" + def test_same_version(self): + # Ensures that substituting the same version results in the same URL + self.check( + 'http://www.mr511.de/software/libelf-0.8.13.tar.gz', '0.8.13', + 'http://www.mr511.de/software/libelf-0.8.13.tar.gz') + def test_different_version(self): + # Test a completely different version syntax + self.check( + 'http://www.prevanders.net/libdwarf-20130729.tar.gz', '8.12', + 'http://www.prevanders.net/libdwarf-8.12.tar.gz') -class PackageSanityTest(unittest.TestCase): + def test_double_version(self): + # Test a URL where the version appears twice + # It should get substituted both times + self.check( + 'https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz', '2.1.3', + 'https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz') - def test_hypre_url_substitution(self): - self.assertEqual(url.substitute_version(base, '2.9.0b'), base) - self.assertEqual( - url.substitute_version(base, '2.8.0b'), stem + "2.8.0b.tar.gz") - self.assertEqual( - url.substitute_version(base, '2.7.0b'), stem + "2.7.0b.tar.gz") - self.assertEqual( - url.substitute_version(base, '2.6.0b'), stem + "2.6.0b.tar.gz") - self.assertEqual( - url.substitute_version(base, '1.14.0b'), stem + "1.14.0b.tar.gz") - self.assertEqual( - url.substitute_version(base, '1.13.0b'), stem + "1.13.0b.tar.gz") - self.assertEqual( - url.substitute_version(base, '2.0.0'), stem + "2.0.0.tar.gz") - self.assertEqual( - url.substitute_version(base, '1.6.0'), stem + "1.6.0.tar.gz") + def test_partial_version_prefix(self): + # Test now with a partial prefix earlier in the URL + # This is hard to figure out so Spack only substitutes + # the last instance of the version + self.check( + 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2', '2.2.0', + 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.0.tar.bz2') + self.check( + 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2', '2.2', + 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.tar.bz2') - def test_otf2_url_substitution(self): - base = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz" + def test_no_separator(self): + # No separator between the name and version of the package + self.check( + 'file://{0}/turbolinux702.tar.gz'.format(os.getcwd()), '703', + 'file://{0}/turbolinux703.tar.gz'.format(os.getcwd())) - self.assertEqual(url.substitute_version(base, '1.4'), base) + def test_github_raw(self): + self.check( + 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '2.0.7', + 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true') + self.check( + 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '4.7', + 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true') - self.assertEqual( - url.substitute_version(base, '1.3.1'), - "http://www.vi-hps.org/upload/packages/otf2/otf2-1.3.1.tar.gz") - self.assertEqual( - url.substitute_version(base, '1.2.1'), - "http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz") + def test_regex(self): + # Package name contains regex characters + self.check( + 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz', '1.2.3', + 'http://math.lbl.gov/voro++/download/dir/voro++-1.2.3.tar.gz') diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index 9a7f4d9f8b..9fa95a8d18 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """Tests for web.py.""" -import pytest import os import spack @@ -141,7 +140,6 @@ def test_find_versions_of_archive_2(): assert ver('2.0.0') in versions -@pytest.mark.xfail def test_find_exotic_versions_of_archive_2(): versions = find_versions_of_archive(root_tarball, root, list_depth=2) # up for grabs to make this better. @@ -157,7 +155,6 @@ def test_find_versions_of_archive_3(): assert ver('4.5') in versions -@pytest.mark.xfail def test_find_exotic_versions_of_archive_3(): versions = find_versions_of_archive(root_tarball, root, list_depth=3) assert ver('2.0.0b2') in versions diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 38ff74f7bc..174f7d0b3c 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -71,7 +71,7 @@ def find_list_url(url): url_types = [ # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz - (r'^(https://github.com/[^/]+/[^/]+)/archive/', + (r'(.*github\.com/[^/]+/[^/]+)/archive/', lambda m: m.group(1) + '/releases')] for pattern, fun in url_types: @@ -101,6 +101,177 @@ def strip_query_and_fragment(path): return (path, '') # Ignore URL parse errors here +def strip_version_suffixes(path): + """Some tarballs contain extraneous information after the version: + + * ``bowtie2-2.2.5-source`` + * ``libevent-2.0.21-stable`` + * ``cuda_8.0.44_linux.run`` + + These strings are not part of the version number and should be ignored. + This function strips those suffixes off and returns the remaining string. + The goal is that the version is always the last thing in ``path``: + + * ``bowtie2-2.2.5`` + * ``libevent-2.0.21`` + * ``cuda_8.0.44`` + + :param str path: The filename or URL for the package + :return: The ``path`` with any extraneous suffixes removed + :rtype: str + """ + # NOTE: This could be done with complicated regexes in parse_version_offset + # NOTE: The problem is that we would have to add these regexes to the end + # NOTE: of every single version regex. Easier to just strip them off + # NOTE: permanently + + suffix_regexes = [ + # Download type + '[Ii]nstall', + 'all', + 'src(_0)?', + '[Ss]ources?', + 'file', + 'full', + 'single', + 'public', + 'with[a-zA-Z_-]+', + 'bin', + 'binary', + 'run', + '[Uu]niversal', + 'jar', + 'complete', + 'oss', + 'gem', + 'tar', + 'sh', + + # Download version + 'stable', + '[Ff]inal', + 'rel', + 'orig', + 'dist', + '\+', + + # License + 'gpl', + + # Arch + # Needs to come before and after OS, appears in both orders + 'ia32', + 'intel', + 'amd64', + 'x64', + 'x86_64', + 'x86', + 'i[36]86', + 'ppc64(le)?', + 'armv?(7l|6l|64)', + + # OS + '[Ll]inux(_64)?', + '[Uu]ni?x', + '[Ss]un[Oo][Ss]', + '[Mm]ac[Oo][Ss][Xx]?', + '[Oo][Ss][Xx]', + '[Dd]arwin(64)?', + '[Aa]pple', + '[Ww]indows', + '[Ww]in(64|32)?', + '[Cc]ygwin(64|32)?', + '[Mm]ingw', + + # Arch + # Needs to come before and after OS, appears in both orders + 'ia32', + 'intel', + 'amd64', + 'x64', + 'x86_64', + 'x86', + 'i[36]86', + 'ppc64(le)?', + 'armv?(7l|6l|64)?', + + # PyPI + '[._-]py[23].*\.whl', + '[._-]cp[23].*\.whl', + '[._-]win.*\.exe', + ] + + for regex in suffix_regexes: + # Remove the suffix from the end of the path + # This may be done multiple times + path = re.sub(r'[._-]?' + regex + '$', '', path) + + return path + + +def strip_name_suffixes(path, version): + """Most tarballs contain a package name followed by a version number. + However, some also contain extraneous information in-between the name + and version: + + * ``rgb-1.0.6`` + * ``converge_install_2.3.16`` + * ``jpegsrc.v9b`` + + These strings are not part of the package name and should be ignored. + This function strips the version number and any extraneous suffixes + off and returns the remaining string. The goal is that the name is + always the last thing in ``path``: + + * ``rgb`` + * ``converge`` + * ``jpeg`` + + :param str path: The filename or URL for the package + :param str version: The version detected for this URL + :return: The ``path`` with any extraneous suffixes removed + :rtype: str + """ + # NOTE: This could be done with complicated regexes in parse_name_offset + # NOTE: The problem is that we would have to add these regexes to every + # NOTE: single name regex. Easier to just strip them off permanently + + suffix_regexes = [ + # Strip off the version and anything after it + + # name-ver + # name_ver + # name.ver + r'[._-]v?' + str(version) + '.*', + + # namever + str(version) + '.*', + + # Download type + 'install', + 'src', + '(open)?[Ss]ources?', + '[._-]std', + + # Download version + 'snapshot', + 'distrib', + + # VCS + '0\+bzr', + + # License + 'gpl', + ] + + for regex in suffix_regexes: + # Remove the suffix from the end of the path + # This may be done multiple times + path = re.sub('[._-]?' + regex + '$', '', path) + + return path + + def split_url_extension(path): """Some URLs have a query string, e.g.: @@ -125,7 +296,7 @@ def split_url_extension(path): prefix, ext, suffix = path, '', '' # Strip off sourceforge download suffix. - match = re.search(r'((?:sourceforge.net|sf.net)/.*)(/download)$', path) + match = re.search(r'((?:sourceforge\.net|sf\.net)/.*)(/download)$', path) if match: prefix, suffix = match.groups() @@ -189,8 +360,20 @@ def parse_version_offset(path): path, ext, suffix = split_url_extension(path) # stem: Everything from path after the final '/' - stem = os.path.basename(path) - offset = len(path) - len(stem) + original_stem = os.path.basename(path) + + # Try to strip off anything after the version number + stem = strip_version_suffixes(original_stem) + + # Assumptions: + # + # 1. version always comes after the name + # 2. separators include '-', '_', and '.' + # 3. names can contain A-Z, a-z, 0-9, '+', separators + # 4. versions can contain A-Z, a-z, 0-9, separators + # 5. versions always start with a digit + # 6. versions are often prefixed by a 'v' character + # 7. separators are most reliable to determine name/version boundaries # List of the following format: # @@ -202,87 +385,118 @@ def parse_version_offset(path): # The first regex that matches string will be used to determine # the version of the package. Thefore, hyperspecific regexes should # come first while generic, catch-all regexes should come last. + # With that said, regular expressions are slow, so if possible, put + # ones that only catch one or two URLs at the bottom. version_regexes = [ - # GitHub tarballs, e.g. v1.2.3 - (r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+)$', path), + # 1st Pass: Simplest case + # Assume name contains no digits and version contains no letters + # e.g. libpng-1.6.27 + (r'^[a-zA-Z+._-]+[._-]v?(\d[\d._-]*)$', stem), - # e.g. https://github.com/sam-github/libnet/tarball/libnet-1.1.4 - (r'github.com/.+/(?:zip|tar)ball/.*-((\d+\.)+\d+)$', path), + # 2nd Pass: Version only + # Assume version contains no letters - # e.g. https://github.com/isaacs/npm/tarball/v0.2.5-1 - (r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+-(\d+))$', path), + # ver + # e.g. 3.2.7, 7.0.2-7, v3.3.0, v1_6_3 + (r'^v?(\d[\d._-]*)$', stem), - # e.g. https://github.com/petdance/ack/tarball/1.93_02 - (r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+_(\d+))$', path), + # 3rd Pass: No separator characters are used + # Assume name contains no digits - # Yorick is very special. - # e.g. https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz - (r'github.com/[^/]+/yorick/archive/y_(\d+(?:_\d+)*)$', path), + # namever + # e.g. turbolinux702, nauty26r7 + (r'^[a-zA-Z+]*(\d[\da-zA-Z]*)$', stem), - # e.g. https://github.com/hpc/lwgrp/archive/v1.0.1.tar.gz - (r'github.com/[^/]+/[^/]+/archive/(?:release-)?v?(\w+(?:[.-]\w+)*)$', path), # noqa + # 4th Pass: A single separator character is used + # Assume name contains no digits - # e.g. https://github.com/erlang/otp/tarball/OTP_R15B01 (erlang style) - (r'[-_](R\d+[AB]\d*(-\d+)?)', path), + # name-name-ver-ver + # e.g. panda-2016-03-07, gts-snapshot-121130, cdd-061a + (r'^[a-zA-Z+-]*(\d[\da-zA-Z-]*)$', stem), - # e.g., https://github.com/hpc/libcircle/releases/download/0.2.1-rc.1/libcircle-0.2.1-rc.1.tar.gz - # e.g., - # https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz - (r'github.com/[^/]+/[^/]+/releases/download/v?([^/]+)/.*$', path), + # name_name_ver_ver + # e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128, v1_6_3 + (r'^[a-zA-Z+_]*(\d[\da-zA-Z_]*)$', stem), - # GitLab syntax: - # {baseUrl}{/organization}{/projectName}/repository/archive.{fileEnding}?ref={gitTag} - # as with github releases, we hope a version can be found in the - # git tag - # Search dotted versions: - # e.g., https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0 - # e.g., https://example.com/org/repo/repository/archive.tar.bz2?ref=SomePrefix-2.1.1 - # e.g., http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 - (r'\?ref=(?:.*-|v)*((\d+\.)+\d+).*$', suffix), - (r'\?version=((\d+\.)+\d+)', suffix), + # name.name.ver.ver + # e.g. prank.source.150803, jpegsrc.v9b, atlas3.11.34, geant4.10.01.p03 + (r'^[a-zA-Z+.]*(\d[\da-zA-Z.]*)$', stem), - # e.g. boost_1_39_0 - (r'((\d+_)+\d+)$', stem), + # 5th Pass: Two separator characters are used + # Name may contain digits, version may contain letters - # e.g. foobar-4.5.1-1 - # e.g. ruby-1.9.1-p243 - (r'-((\d+\.)*\d\.\d+-(p|rc|RC)?\d+)(?:[-._](?:bin|dist|stable|src|sources))?$', stem), # noqa + # name-name-ver.ver + # e.g. m4-1.4.17, gmp-6.0.0a, launchmon-v1.0.2 + (r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), - # e.g. lame-398-1 - (r'-((\d)+-\d)', stem), + # name-name-ver_ver + # e.g. icu4c-57_1 + (r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z_]*)$', stem), - # e.g. foobar_1.2-3 or 3.98-1.4 - (r'_((\d+\.)+\d+(-(\d+(\.\d+)?))?[a-z]?)', stem), + # name_name_ver.ver + # e.g. superlu_dist_4.1, pexsi_v0.9.0 + (r'^[a-zA-Z\d+_]+_v?(\d[\da-zA-Z.]*)$', stem), - # e.g. foobar-4.5.1 - (r'-((\d+\.)*\d+)$', stem), + # name_name.ver.ver + # e.g. fer_source.v696 + (r'^[a-zA-Z\d+_]+\.v?(\d[\da-zA-Z.]*)$', stem), - # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), + # name-name-ver.ver-ver.ver + # e.g. sowing-1.1.23-p1, bib2xhtml-v3.0-15-gf506, 4.6.3-alpha04 + (r'^(?:[a-zA-Z\d+-]+-)?v?(\d[\da-zA-Z.-]*)$', stem), - # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta - (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), + # namever.ver-ver.ver + # e.g. go1.4-bootstrap-20161024 + (r'^[a-zA-Z+]+v?(\d[\da-zA-Z.-]*)$', stem), - # e.g. foobar4.5.1 - (r'((\d+\.)*\d+)$', stem), + # 6th Pass: All three separator characters are used + # Name may contain digits, version may contain letters - # e.g. foobar-4.5.0-bin - (r'-((\d+\.)+\d+[a-z]?)[-._](bin|dist|stable|src|sources?)$', stem), + # name_name-ver.ver + # e.g. the_silver_searcher-0.32.0, sphinx_rtd_theme-0.1.10a0 + (r'^[a-zA-Z\d+_]+-v?(\d[\da-zA-Z.]*)$', stem), - # e.g. dash_0.5.5.1.orig.tar.gz (Debian style) - (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem), + # name.name_ver.ver-ver.ver + # e.g. TH.data_1.0-8, XML_3.98-1.4 + (r'^[a-zA-Z\d+.]+_v?(\d[\da-zA-Z.-]*)$', stem), - # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz - (r'-v?([^-]+(-alpha|-beta)?)', stem), + # name-name-ver.ver_ver.ver + # e.g. pypar-2.1.5_108 + (r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z._]*)$', stem), - # e.g. astyle_1.23_macosx.tar.gz - (r'_([^_]+(_alpha|_beta)?)', stem), + # name.name_name-ver.ver + # e.g. tap.py-1.6, backports.ssl_match_hostname-3.5.0.1 + (r'^[a-zA-Z\d+._]+-v?(\d[\da-zA-Z.]*)$', stem), - # e.g. http://mirrors.jenkins-ci.org/war/1.486/jenkins.war - (r'\/(\d\.\d+)\/', path), + # name-namever.ver_ver.ver + # e.g. STAR-CCM+11.06.010_02 + (r'^[a-zA-Z+-]+(\d[\da-zA-Z._]*)$', stem), - # e.g. http://www.ijg.org/files/jpegsrc.v8d.tar.gz - (r'\.v(\d+[a-z]?)', stem) + # 7th Pass: Specific VCS + + # bazaar + # e.g. libvterm-0+bzr681 + (r'bzr(\d[\da-zA-Z._-]*)$', stem), + + # 8th Pass: Version in path + + # github.com/repo/name/releases/download/vver/name + # e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow + (r'github\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/', path), # noqa + + # 9th Pass: Query strings + + # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 + (r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), + + # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 + (r'\?version=v?(\d[\da-zA-Z._-]*)$', suffix), + + # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz + (r'\?filename=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), + + # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz + (r'\?package=[a-zA-Z\d+-]+&get=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), # noqa ] for i, version_regex in enumerate(version_regexes): @@ -292,9 +506,15 @@ def parse_version_offset(path): version = match.group(1) start = match.start(1) - # if we matched from the basename, then add offset in. + # If we matched from the stem or suffix, we need to add offset + offset = 0 if match_string is stem: - start += offset + offset = len(path) - len(original_stem) + elif match_string is suffix: + offset = len(path) + if ext: + offset += len(ext) + 1 # .tar.gz is converted to tar.gz + start += offset return version, start, len(version), i, regex @@ -342,7 +562,7 @@ def parse_name_offset(path, v=None): except UndetectableVersionError: # Not all URLs contain a version. We still want to be able # to determine a name if possible. - v = '' + v = 'unknown' # path: The prefix of the URL, everything before the ext and suffix # ext: The file extension @@ -350,8 +570,10 @@ def parse_name_offset(path, v=None): path, ext, suffix = split_url_extension(path) # stem: Everything from path after the final '/' - stem = os.path.basename(path) - offset = len(path) - len(stem) + original_stem = os.path.basename(path) + + # Try to strip off anything after the package name + stem = strip_name_suffixes(original_stem, v) # List of the following format: # @@ -363,26 +585,45 @@ def parse_name_offset(path, v=None): # The first regex that matches string will be used to determine # the name of the package. Thefore, hyperspecific regexes should # come first while generic, catch-all regexes should come last. + # With that said, regular expressions are slow, so if possible, put + # ones that only catch one or two URLs at the bottom. name_regexes = [ - (r'/sourceforge/([^/]+)/', path), - (r'github.com/[^/]+/[^/]+/releases/download/%s/(.*)-%s$' % - (v, v), path), - (r'/([^/]+)/(tarball|zipball)/', path), - (r'/([^/]+)[_.-](bin|dist|stable|src|sources)[_.-]%s' % v, path), - (r'github.com/[^/]+/([^/]+)/archive', path), - (r'github.com/[^/]+/([^/]+)/releases', path), - (r'[^/]+/([^/]+)/repository/archive', path), # gitlab - (r'([^/]+)/download.php', path), - - (r'([^/]+)[_.-]v?%s' % v, stem), # prefer the stem - (r'([^/]+)%s' % v, stem), - - # accept the path if name is not in stem. - (r'/([^/]+)[_.-]v?%s' % v, path), - (r'/([^/]+)%s' % v, path), - - (r'^([^/]+)[_.-]v?%s' % v, path), - (r'^([^/]+)%s' % v, path) + # 1st Pass: Common repositories + + # GitHub: github.com/repo/name/ + # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz + (r'github\.com/[^/]+/([^/]+)', path), + + # GitLab: gitlab.*/repo/name/ + # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 + (r'gitlab[^/]+/[^/]+/([^/]+)', path), + + # Bitbucket: bitbucket.org/repo/name/ + # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2 + (r'bitbucket\.org/[^/]+/([^/]+)', path), + + # PyPI: pypi.(python.org|io)/packages/source/first-letter/name/ + # e.g. https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz + # e.g. https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz + (r'pypi\.(?:python\.org|io)/packages/source/[A-Za-z\d]/([^/]+)', path), + + # 2nd Pass: Query strings + + # ?filename=name-ver.ver + # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz + (r'\?filename=([A-Za-z\d+-]+)$', stem), + + # ?package=name + # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz + (r'\?package=([A-Za-z\d+-]+)', stem), + + # download.php + # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 + (r'([^/]+)/download.php$', path), + + # 3rd Pass: Name followed by version in archive + + (r'^([A-Za-z\d+\._-]+)$', stem), ] for i, name_regex in enumerate(name_regexes): @@ -392,13 +633,15 @@ def parse_name_offset(path, v=None): name = match.group(1) start = match.start(1) - # if we matched from the basename, then add offset in. + # If we matched from the stem or suffix, we need to add offset + offset = 0 if match_string is stem: - start += offset - - # package names should be lowercase and separated by dashes. - name = name.lower() - name = re.sub('[_.]', '-', name) + offset = len(path) - len(original_stem) + elif match_string is suffix: + offset = len(path) + if ext: + offset += len(ext) + 1 # .tar.gz is converted to tar.gz + start += offset return name, start, len(name), i, regex @@ -431,6 +674,9 @@ def parse_name_and_version(path): The version of the package :rtype: tuple + + :raises UndetectableVersionError: If the URL does not match any regexes + :raises UndetectableNameError: If the URL does not match any regexes """ ver = parse_version(path) name = parse_name(path, ver) @@ -457,6 +703,22 @@ def cumsum(elts, init=0, fn=lambda x: x): return sums +def find_all(substring, string): + """Returns a list containing the indices of + every occurrence of substring in string.""" + + occurrences = [] + index = 0 + while index < len(string): + index = string.find(substring, index) + if index == -1: + break + occurrences.append(index) + index += len(substring) + + return occurrences + + def substitution_offsets(path): """This returns offsets for substituting versions and names in the provided path. It is a helper for :func:`substitute_version`. @@ -468,65 +730,34 @@ def substitution_offsets(path): except UndetectableNameError: return (None, -1, -1, (), ver, vs, vl, (vs,)) except UndetectableVersionError: - return (None, -1, -1, (), None, -1, -1, ()) - - # protect extensions like bz2 from getting inadvertently - # considered versions. - path = comp.strip_extension(path) - - # Construct a case-insensitive regular expression for the package name. - name_re = '(%s)' % insensitize(name) - - # Split the string apart by things that match the name so that if the - # name contains numbers or things that look like versions, we don't - # accidentally substitute them with a version. - name_parts = re.split(name_re, path) - - offsets = cumsum(name_parts, 0, len) - name_offsets = offsets[1::2] + try: + name, ns, nl, ni, nregex = parse_name_offset(path) + return (name, ns, nl, (ns,), None, -1, -1, ()) + except UndetectableNameError: + return (None, -1, -1, (), None, -1, -1, ()) - ver_offsets = [] - for i in range(0, len(name_parts), 2): - vparts = re.split(ver, name_parts[i]) - voffsets = cumsum(vparts, offsets[i], len) - ver_offsets.extend(voffsets[1::2]) + # Find the index of every occurrence of name and ver in path + name_offsets = find_all(name, path) + ver_offsets = find_all(ver, path) - return (name, ns, nl, tuple(name_offsets), - ver, vs, vl, tuple(ver_offsets)) + return (name, ns, nl, name_offsets, + ver, vs, vl, ver_offsets) def wildcard_version(path): """Find the version in the supplied path, and return a regular expression that will match this path with any version in its place. """ - # Get name and version, so we can treat them specially - name, v = parse_name_and_version(path) + # Get version so we can replace it with a wildcard + version = parse_version(path) - path, ext, suffix = split_url_extension(path) + # Split path by versions + vparts = path.split(str(version)) + + # Replace each version with a generic capture group to find versions + # and escape everything else so it's not interpreted as a regex + result = '(\d.*)'.join(re.escape(vp) for vp in vparts) - # Construct a case-insensitive regular expression for the package name. - name_re = '(%s)' % insensitize(name) - - # Split the string apart by things that match the name so that if the - # name contains numbers or things that look like versions, we don't - # catch them with the version wildcard. - name_parts = re.split(name_re, path) - - # Even elements in the array did *not* match the name - for i in range(0, len(name_parts), 2): - # Split each part by things that look like versions. - vparts = re.split(v.wildcard(), name_parts[i]) - - # Replace each version with a generic capture group to find versions. - # And escape everything else so it's not interpreted as a regex - vgroup = '(%s)' % v.wildcard() - name_parts[i] = vgroup.join(re.escape(vp) for vp in vparts) - - # Put it all back together with original name matches intact. - result = ''.join(name_parts) - if ext: - result += '.' + ext - result += suffix return result diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 1f2bfa88cf..cd35008aed 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -39,6 +39,7 @@ __all__ = [ 'validate_fully_qualified_module_name', 'validate_module_name', 'possible_spack_module_names', + 'simplify_name', 'NamespaceTrie'] # Valid module names can contain '-' but can't start with it. @@ -108,6 +109,50 @@ def possible_spack_module_names(python_mod_name): return results +def simplify_name(name): + """Simplifies a name which may include uppercase letters, periods, + underscores, and pluses. In general, we want our package names to + only contain lowercase letters, digits, and dashes. + + :param str name: The original name of the package + :return: The new name of the package + :rtype: str + """ + # Convert CamelCase to Dashed-Names + # e.g. ImageMagick -> Image-Magick + # e.g. SuiteSparse -> Suite-Sparse + # name = re.sub('([a-z])([A-Z])', r'\1-\2', name) + + # Rename Intel downloads + # e.g. l_daal, l_ipp, l_mkl -> daal, ipp, mkl + if name.startswith('l_'): + name = name[2:] + + # Convert UPPERCASE to lowercase + # e.g. SAMRAI -> samrai + name = name.lower() + + # Replace '_' and '.' with '-' + # e.g. backports.ssl_match_hostname -> backports-ssl-match-hostname + name = name.replace('_', '-') + name = name.replace('.', '-') + + # Replace "++" with "pp" and "+" with "-plus" + # e.g. gtk+ -> gtk-plus + # e.g. voro++ -> voropp + name = name.replace('++', 'pp') + name = name.replace('+', '-plus') + + # Simplify Lua package names + # We don't want "lua" to occur multiple times in the name + name = re.sub('^(lua)([^-])', r'\1-\2', name) + + # Simplify Bio++ package names + name = re.sub('^(bpp)([^-])', r'\1-\2', name) + + return name + + def valid_module_name(mod_name): """Return whether mod_name is valid for use in Spack.""" return bool(re.match(_valid_module_re, mod_name)) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 8e2dd34635..f803c6cea3 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -268,6 +268,14 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0): # part, not the full path. url_regex = os.path.basename(url_regex) + # We need to add a / to the beginning of the regex to prevent + # Spack from picking up similarly named packages like: + # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz + # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz + # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz + # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz + url_regex = '/' + url_regex + # We need to add a $ anchor to the end of the regex to prevent # Spack from picking up signature files like: # .asc @@ -275,7 +283,9 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0): # .sha256 # .sig # However, SourceForge downloads still need to end in '/download'. - regexes.append(url_regex + '(\/download)?$') + url_regex += '(\/download)?$' + + regexes.append(url_regex) # Build a dict version -> URL from any links that match the wildcards. versions = {} diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index c8395aeb29..89fcc9aaa7 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -194,35 +194,6 @@ class Version(object): nother = len(other.version) return nother <= nself and self.version[:nother] == other.version - def wildcard(self): - """Create a regex that will match variants of this version string.""" - def a_or_n(seg): - if type(seg) == int: - return r'[0-9]+' - else: - return r'[a-zA-Z]+' - - version = self.version - - # Use a wildcard for separators, in case a version is written - # two different ways (e.g., boost writes 1_55_0 and 1.55.0) - sep_re = '[_.-]' - separators = ('',) + (sep_re,) * len(self.separators) - - version += (version[-1],) * 2 - separators += (sep_re,) * 2 - - segments = [a_or_n(seg) for seg in version] - - wc = segments[0] - for i in range(1, len(separators)): - wc += '(?:' + separators[i] + segments[i] - - # Add possible alpha or beta indicator at the end of each segemnt - # We treat these specially b/c they're so common. - wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1) - return wc - def __iter__(self): return iter(self.version) |