summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/developer_guide.rst8
-rw-r--r--lib/spack/spack/cmd/create.py25
-rw-r--r--lib/spack/spack/cmd/flake8.py2
-rw-r--r--lib/spack/spack/cmd/url.py92
-rw-r--r--lib/spack/spack/cmd/versions.py2
-rw-r--r--lib/spack/spack/test/cmd/url.py21
-rw-r--r--lib/spack/spack/test/url_extrapolate.py101
-rw-r--r--lib/spack/spack/test/url_parse.py794
-rw-r--r--lib/spack/spack/test/url_substitution.py84
-rw-r--r--lib/spack/spack/test/web.py3
-rw-r--r--lib/spack/spack/url.py513
-rw-r--r--lib/spack/spack/util/naming.py45
-rw-r--r--lib/spack/spack/util/web.py12
-rw-r--r--lib/spack/spack/version.py29
14 files changed, 1166 insertions, 565 deletions
diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst
index 0ce4029950..ea8d50c6ca 100644
--- a/lib/spack/docs/developer_guide.rst
+++ b/lib/spack/docs/developer_guide.rst
@@ -447,16 +447,16 @@ the string that it detected to be the name and version. The
``--incorrect-name`` and ``--incorrect-version`` flags can be used to
print URLs that were not being parsed correctly.
-""""""""""""""""""
-``spack url test``
-""""""""""""""""""
+"""""""""""""""""""""
+``spack url summary``
+"""""""""""""""""""""
This command attempts to parse every URL for every package in Spack
and prints a summary of how many of them are being correctly parsed.
It also prints a histogram showing which regular expressions are being
matched and how frequently:
-.. command-output:: spack url test
+.. command-output:: spack url summary
This command is essential for anyone adding or changing the regular
expressions that parse names and versions. By running this command
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index cc90669158..906c7e1aec 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -31,13 +31,13 @@ import llnl.util.tty as tty
import spack
import spack.cmd
import spack.cmd.checksum
-import spack.url
import spack.util.web
from llnl.util.filesystem import mkdirp
from spack.repository import Repo
from spack.spec import Spec
from spack.util.executable import which
from spack.util.naming import *
+from spack.url import *
description = "create a new package file"
@@ -382,6 +382,10 @@ class BuildSystemGuesser:
can take a peek at the fetched tarball and discern the build system it uses
"""
+ def __init__(self):
+ """Sets the default build system."""
+ self.build_system = 'generic'
+
def __call__(self, stage, url):
"""Try to guess the type of build system used by a project based on
the contents of its archive or the URL it was downloaded from."""
@@ -427,14 +431,11 @@ class BuildSystemGuesser:
# Determine the build system based on the files contained
# in the archive.
- build_system = 'generic'
for pattern, bs in clues:
if any(re.search(pattern, l) for l in lines):
- build_system = bs
+ self.build_system = bs
break
- self.build_system = build_system
-
def get_name(args):
"""Get the name of the package based on the supplied arguments.
@@ -458,9 +459,9 @@ def get_name(args):
elif args.url:
# Try to guess the package name based on the URL
try:
- name = spack.url.parse_name(args.url)
+ name = parse_name(args.url)
tty.msg("This looks like a URL for {0}".format(name))
- except spack.url.UndetectableNameError:
+ except UndetectableNameError:
tty.die("Couldn't guess a name for this package.",
" Please report this bug. In the meantime, try running:",
" `spack create --name <name> <url>`")
@@ -515,11 +516,16 @@ def get_versions(args, name):
if args.url:
# Find available versions
- url_dict = spack.util.web.find_versions_of_archive(args.url)
+ try:
+ url_dict = spack.util.web.find_versions_of_archive(args.url)
+ except UndetectableVersionError:
+ # Use fake versions
+ tty.warn("Couldn't detect version in: {0}".format(args.url))
+ return versions, guesser
if not url_dict:
# If no versions were found, revert to what the user provided
- version = spack.url.parse_version(args.url)
+ version = parse_version(args.url)
url_dict = {version: args.url}
versions = spack.cmd.checksum.get_checksums(
@@ -611,6 +617,7 @@ def create(parser, args):
url = get_url(args)
versions, guesser = get_versions(args, name)
build_system = get_build_system(args, guesser)
+ name = simplify_name(name)
# Create the package template object
PackageClass = templates[build_system]
diff --git a/lib/spack/spack/cmd/flake8.py b/lib/spack/spack/cmd/flake8.py
index 19724142bb..a6dc941190 100644
--- a/lib/spack/spack/cmd/flake8.py
+++ b/lib/spack/spack/cmd/flake8.py
@@ -70,7 +70,7 @@ exemptions = {
# exemptions applied to all files.
r'.py$': {
# Exempt lines with URLs from overlong line errors.
- 501: [r'(https?|file)\:']
+ 501: [r'(https?|ftp|file)\:']
},
}
diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py
index 6823f0febd..1128e08a43 100644
--- a/lib/spack/spack/cmd/url.py
+++ b/lib/spack/spack/cmd/url.py
@@ -31,6 +31,7 @@ import spack
from llnl.util import tty
from spack.url import *
from spack.util.web import find_versions_of_archive
+from spack.util.naming import simplify_name
description = "debugging tool for url parsing"
@@ -66,19 +67,26 @@ def setup_parser(subparser):
'-n', '--incorrect-name', action='store_true',
help='only list urls for which the name was incorrectly parsed')
excl_args.add_argument(
+ '-N', '--correct-name', action='store_true',
+ help='only list urls for which the name was correctly parsed')
+ excl_args.add_argument(
'-v', '--incorrect-version', action='store_true',
help='only list urls for which the version was incorrectly parsed')
+ excl_args.add_argument(
+ '-V', '--correct-version', action='store_true',
+ help='only list urls for which the version was correctly parsed')
- # Test
+ # Summary
sp.add_parser(
- 'test', help='print a summary of how well we are parsing package urls')
+ 'summary',
+ help='print a summary of how well we are parsing package urls')
def url(parser, args):
action = {
- 'parse': url_parse,
- 'list': url_list,
- 'test': url_test
+ 'parse': url_parse,
+ 'list': url_list,
+ 'summary': url_summary
}
action[args.subcommand](args)
@@ -116,6 +124,10 @@ def url_parse(args):
tty.msg('Spidering for versions:')
versions = find_versions_of_archive(url)
+ if not versions:
+ print(' Found no versions for {0}'.format(name))
+ return
+
max_len = max(len(str(v)) for v in versions)
for v in sorted(versions):
@@ -145,7 +157,7 @@ def url_list(args):
return len(urls)
-def url_test(args):
+def url_summary(args):
# Collect statistics on how many URLs were correctly parsed
total_urls = 0
correct_names = 0
@@ -205,19 +217,19 @@ def url_test(args):
correct_versions, total_urls, correct_versions / total_urls))
print()
- tty.msg('Statistics on name regular expresions:')
+ tty.msg('Statistics on name regular expressions:')
print()
- print(' Index Count Regular Expresion')
+ print(' Index Count Regular Expression')
for ni in name_regex_dict:
print(' {0:>3}: {1:>6} r{2!r}'.format(
ni, name_count_dict[ni], name_regex_dict[ni]))
print()
- tty.msg('Statistics on version regular expresions:')
+ tty.msg('Statistics on version regular expressions:')
print()
- print(' Index Count Regular Expresion')
+ print(' Index Count Regular Expression')
for vi in version_regex_dict:
print(' {0:>3}: {1:>6} r{2!r}'.format(
vi, version_count_dict[vi], version_regex_dict[vi]))
@@ -257,22 +269,38 @@ def url_list_parsing(args, urls, url, pkg):
:rtype: set
"""
if url:
- if args.incorrect_name:
- # Only add URLs whose name was incorrectly parsed
+ if args.correct_name or args.incorrect_name:
+ # Attempt to parse the name
try:
name = parse_name(url)
- if not name_parsed_correctly(pkg, name):
+ if (args.correct_name and
+ name_parsed_correctly(pkg, name)):
+ # Add correctly parsed URLs
+ urls.add(url)
+ elif (args.incorrect_name and
+ not name_parsed_correctly(pkg, name)):
+ # Add incorrectly parsed URLs
urls.add(url)
except UndetectableNameError:
- urls.add(url)
- elif args.incorrect_version:
- # Only add URLs whose version was incorrectly parsed
+ if args.incorrect_name:
+ # Add incorrectly parsed URLs
+ urls.add(url)
+ elif args.correct_version or args.incorrect_version:
+ # Attempt to parse the version
try:
version = parse_version(url)
- if not version_parsed_correctly(pkg, version):
+ if (args.correct_version and
+ version_parsed_correctly(pkg, version)):
+ # Add correctly parsed URLs
+ urls.add(url)
+ elif (args.incorrect_version and
+ not version_parsed_correctly(pkg, version)):
+ # Add incorrectly parsed URLs
urls.add(url)
except UndetectableVersionError:
- urls.add(url)
+ if args.incorrect_version:
+ # Add incorrectly parsed URLs
+ urls.add(url)
else:
urls.add(url)
@@ -289,6 +317,8 @@ def name_parsed_correctly(pkg, name):
"""
pkg_name = pkg.name
+ name = simplify_name(name)
+
# After determining a name, `spack create` determines a build system.
# Some build systems prepend a special string to the front of the name.
# Since this can't be guessed from the URL, it would be unfair to say
@@ -311,9 +341,33 @@ def version_parsed_correctly(pkg, version):
:returns: True if the name was correctly parsed, else False
:rtype: bool
"""
+ version = remove_separators(version)
+
# If the version parsed from the URL is listed in a version()
# directive, we assume it was correctly parsed
for pkg_version in pkg.versions:
- if str(pkg_version) == str(version):
+ pkg_version = remove_separators(pkg_version)
+ if pkg_version == version:
return True
return False
+
+
+def remove_separators(version):
+ """Removes separator characters ('.', '_', and '-') from a version.
+
+ A version like 1.2.3 may be displayed as 1_2_3 in the URL.
+ Make sure 1.2.3, 1-2-3, 1_2_3, and 123 are considered equal.
+ Unfortunately, this also means that 1.23 and 12.3 are equal.
+
+ :param version: A version
+ :type version: str or Version
+ :returns: The version with all separator characters removed
+ :rtype: str
+ """
+ version = str(version)
+
+ version = version.replace('.', '')
+ version = version.replace('_', '')
+ version = version.replace('-', '')
+
+ return version
diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py
index f65ef14406..a6f6805fb0 100644
--- a/lib/spack/spack/cmd/versions.py
+++ b/lib/spack/spack/cmd/versions.py
@@ -53,6 +53,6 @@ def versions(parser, args):
tty.debug("Check the list_url and list_depth attribute on the "
"package to help Spack find versions.")
else:
- print(" Found no unckecksummed versions for %s" % pkg.name)
+ print(" Found no unchecksummed versions for %s" % pkg.name)
else:
colify(sorted(remote_versions, reverse=True), indent=2)
diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py
index 4c60d814ce..3bc0bc7820 100644
--- a/lib/spack/spack/test/cmd/url.py
+++ b/lib/spack/spack/test/cmd/url.py
@@ -48,11 +48,12 @@ def test_name_parsed_correctly():
assert name_parsed_correctly(MyPackage('r-devtools', []), 'devtools')
assert name_parsed_correctly(MyPackage('py-numpy', []), 'numpy')
assert name_parsed_correctly(MyPackage('octave-splines', []), 'splines')
+ assert name_parsed_correctly(MyPackage('imagemagick', []), 'ImageMagick') # noqa
+ assert name_parsed_correctly(MyPackage('th-data', []), 'TH.data')
# Expected False
assert not name_parsed_correctly(MyPackage('', []), 'hdf5')
assert not name_parsed_correctly(MyPackage('hdf5', []), '')
- assert not name_parsed_correctly(MyPackage('imagemagick', []), 'ImageMagick') # noqa
assert not name_parsed_correctly(MyPackage('yaml-cpp', []), 'yamlcpp')
assert not name_parsed_correctly(MyPackage('yamlcpp', []), 'yaml-cpp')
assert not name_parsed_correctly(MyPackage('r-py-parser', []), 'parser')
@@ -64,6 +65,8 @@ def test_version_parsed_correctly():
assert version_parsed_correctly(MyPackage('', ['1.2.3']), '1.2.3')
assert version_parsed_correctly(MyPackage('', ['5.4a', '5.4b']), '5.4a')
assert version_parsed_correctly(MyPackage('', ['5.4a', '5.4b']), '5.4b')
+ assert version_parsed_correctly(MyPackage('', ['1.63.0']), '1_63_0')
+ assert version_parsed_correctly(MyPackage('', ['0.94h']), '094h')
# Expected False
assert not version_parsed_correctly(MyPackage('', []), '1.2.3')
@@ -95,7 +98,7 @@ def test_url_list(parser):
colored_urls = url_list(args)
assert colored_urls == total_urls
- # The following two options should print fewer URLs than the default.
+ # The following options should print fewer URLs than the default.
# If they print the same number of URLs, something is horribly broken.
# If they say we missed 0 URLs, something is probably broken too.
args = parser.parse_args(['list', '--incorrect-name'])
@@ -106,11 +109,19 @@ def test_url_list(parser):
incorrect_version_urls = url_list(args)
assert 0 < incorrect_version_urls < total_urls
+ args = parser.parse_args(['list', '--correct-name'])
+ correct_name_urls = url_list(args)
+ assert 0 < correct_name_urls < total_urls
-def test_url_test(parser):
- args = parser.parse_args(['test'])
+ args = parser.parse_args(['list', '--correct-version'])
+ correct_version_urls = url_list(args)
+ assert 0 < correct_version_urls < total_urls
+
+
+def test_url_summary(parser):
+ args = parser.parse_args(['summary'])
(total_urls, correct_names, correct_versions,
- name_count_dict, version_count_dict) = url_test(args)
+ name_count_dict, version_count_dict) = url_summary(args)
assert 0 < correct_names <= sum(name_count_dict.values()) <= total_urls # noqa
assert 0 < correct_versions <= sum(version_count_dict.values()) <= total_urls # noqa
diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py
deleted file mode 100644
index 5f5cf555ae..0000000000
--- a/lib/spack/spack/test/url_extrapolate.py
+++ /dev/null
@@ -1,101 +0,0 @@
-##############################################################################
-# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/llnl/spack
-# Please also see the LICENSE file for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License (as
-# published by the Free Software Foundation) version 2.1, February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-"""Tests ability of spack to extrapolate URL versions from
-existing versions.
-"""
-import unittest
-
-import spack.url as url
-
-
-class UrlExtrapolateTest(unittest.TestCase):
-
- def check_url(self, base, version, new_url):
- self.assertEqual(url.substitute_version(base, version), new_url)
-
- def test_libelf_version(self):
- base = "http://www.mr511.de/software/libelf-0.8.13.tar.gz"
- self.check_url(base, '0.8.13', base)
- self.check_url(
- base, '0.8.12', "http://www.mr511.de/software/libelf-0.8.12.tar.gz")
- self.check_url(
- base, '0.3.1', "http://www.mr511.de/software/libelf-0.3.1.tar.gz")
- self.check_url(
- base, '1.3.1b', "http://www.mr511.de/software/libelf-1.3.1b.tar.gz")
-
- def test_libdwarf_version(self):
- base = "http://www.prevanders.net/libdwarf-20130729.tar.gz"
- self.check_url(base, '20130729', base)
- self.check_url(
- base, '8.12', "http://www.prevanders.net/libdwarf-8.12.tar.gz")
-
- def test_dyninst_version(self):
- # Dyninst has a version twice in the URL.
- base = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz"
- self.check_url(base, '8.1.2', base)
- self.check_url(base, '8.2',
- "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.2/DyninstAPI-8.2.tgz")
- self.check_url(base, '8.3.1',
- "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.3.1/DyninstAPI-8.3.1.tgz")
-
- def test_partial_version_prefix(self):
- # Test now with a partial prefix earlier in the URL -- this is
- # hard to figure out so Spack only substitutes the last
- # instance of the version.
- base = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.1.2.tgz"
- self.check_url(base, '8.1.2', base)
- self.check_url(base, '8.1.4',
- "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.1.4.tgz")
- self.check_url(base, '8.2',
- "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.2.tgz")
- self.check_url(base, '8.3.1',
- "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.3.1.tgz")
-
- def test_scalasca_partial_version(self):
- # Note that this probably doesn't actually work, but sites are
- # inconsistent about their directory structure, so it's not
- # clear what is right. This test is for consistency and to
- # document behavior. If you figure out a good way to handle
- # this case, fix the tests too.
- self.check_url('http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz', '8.3.1',
- 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-8.3.1.tar.gz')
- self.check_url('http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz', '8.3.1',
- 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-8.3.1.tar.gz')
-
- def test_mpileaks_version(self):
- self.check_url('https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz', '2.1.3',
- 'https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz')
-
- def test_gcc(self):
- self.check_url('http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2', '4.7',
- 'http://open-source-box.org/gcc/gcc-4.7/gcc-4.7.tar.bz2')
- self.check_url('http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2', '4.4.7',
- 'http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2')
-
- def test_github_raw(self):
- self.check_url('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '2.0.7',
- 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true')
- self.check_url('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '4.7',
- 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true')
diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py
index 8913de94d0..2af7c6ae0b 100644
--- a/lib/spack/spack/test/url_parse.py
+++ b/lib/spack/spack/test/url_parse.py
@@ -22,246 +22,667 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-"""\
-This file has a bunch of versions tests taken from the excellent version
-detection in Homebrew.
-"""
+"""Tests Spack's ability to parse the name and version of a package
+based on its URL."""
+
+import os
import unittest
-import spack.url as url
+from spack.url import *
-class UrlParseTest(unittest.TestCase):
+class UrlStripVersionSuffixesTest(unittest.TestCase):
+ """Tests for spack.url.strip_version_suffixes"""
- def assert_not_detected(self, string):
- self.assertRaises(
- url.UndetectableVersionError, url.parse_name_and_version, string)
+ def check(self, before, after):
+ stripped = strip_version_suffixes(before)
+ self.assertEqual(stripped, after)
- def check(self, name, v, string, **kwargs):
- # Make sure correct name and version are extracted.
- parsed_name, parsed_v = url.parse_name_and_version(string)
- self.assertEqual(parsed_name, name)
- self.assertEqual(parsed_v, url.Version(v))
+ def test_no_suffix(self):
+ self.check('rgb-1.0.6',
+ 'rgb-1.0.6')
- # Some URLs (like boost) are special and need to override the
- # built-in functionality.
- if kwargs.get('no_check_url', False):
- return
+ def test_misleading_prefix(self):
+ self.check('jpegsrc.v9b',
+ 'jpegsrc.v9b')
+ self.check('turbolinux702',
+ 'turbolinux702')
+ self.check('converge_install_2.3.16',
+ 'converge_install_2.3.16')
- # Make sure Spack formulates the right URL when we try to
- # build one with a specific version.
- self.assertEqual(string, url.substitute_version(string, v))
+ # Download type
- def test_wwwoffle_version(self):
- self.check(
- 'wwwoffle', '2.9h',
- 'http://www.gedanken.demon.co.uk/download-wwwoffle/wwwoffle-2.9h.tgz')
+ def test_src(self):
+ self.check('apache-ant-1.9.7-src',
+ 'apache-ant-1.9.7')
+ self.check('go1.7.4.src',
+ 'go1.7.4')
+
+ def test_source(self):
+ self.check('bowtie2-2.2.5-source',
+ 'bowtie2-2.2.5')
+ self.check('grib_api-1.17.0-Source',
+ 'grib_api-1.17.0')
+
+ def test_full(self):
+ self.check('julia-0.4.3-full',
+ 'julia-0.4.3')
+
+ def test_bin(self):
+ self.check('apache-maven-3.3.9-bin',
+ 'apache-maven-3.3.9')
+
+ def test_binary(self):
+ self.check('Jmol-14.8.0-binary',
+ 'Jmol-14.8.0')
+
+ def test_gem(self):
+ self.check('rubysl-date-2.0.9.gem',
+ 'rubysl-date-2.0.9')
+
+ def test_tar(self):
+ self.check('gromacs-4.6.1-tar',
+ 'gromacs-4.6.1')
+
+ def test_sh(self):
+ self.check('Miniconda2-4.3.11-Linux-x86_64.sh',
+ 'Miniconda2-4.3.11')
+
+ # Download version
+
+ def test_stable(self):
+ self.check('libevent-2.0.21-stable',
+ 'libevent-2.0.21')
+
+ def test_final(self):
+ self.check('2.6.7-final',
+ '2.6.7')
+
+ def test_rel(self):
+ self.check('v1.9.5.1rel',
+ 'v1.9.5.1')
+
+ def test_orig(self):
+ self.check('dash_0.5.5.1.orig',
+ 'dash_0.5.5.1')
+
+ def test_plus(self):
+ self.check('ncbi-blast-2.6.0+-src',
+ 'ncbi-blast-2.6.0')
+
+ # License
+
+ def test_gpl(self):
+ self.check('cppad-20170114.gpl',
+ 'cppad-20170114')
+
+ # OS
+
+ def test_linux(self):
+ self.check('astyle_2.04_linux',
+ 'astyle_2.04')
+
+ def test_unix(self):
+ self.check('install-tl-unx',
+ 'install-tl')
+
+ def test_macos(self):
+ self.check('astyle_1.23_macosx',
+ 'astyle_1.23')
+ self.check('haxe-2.08-osx',
+ 'haxe-2.08')
+
+ # PyPI
+
+ def test_wheel(self):
+ self.check('entrypoints-0.2.2-py2.py3-none-any.whl',
+ 'entrypoints-0.2.2')
+ self.check('numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl', # noqa
+ 'numpy-1.12.0')
+
+ def test_exe(self):
+ self.check('PyYAML-3.12.win-amd64-py3.5.exe',
+ 'PyYAML-3.12')
+
+ # Combinations of multiple patterns
+
+ def test_complex_all(self):
+ self.check('p7zip_9.04_src_all',
+ 'p7zip_9.04')
+
+ def test_complex_run(self):
+ self.check('cuda_8.0.44_linux.run',
+ 'cuda_8.0.44')
+
+ def test_complex_file(self):
+ self.check('ack-2.14-single-file',
+ 'ack-2.14')
+
+ def test_complex_jar(self):
+ self.check('antlr-3.4-complete.jar',
+ 'antlr-3.4')
+
+ def test_complex_oss(self):
+ self.check('tbb44_20160128oss_src_0',
+ 'tbb44_20160128')
+
+ def test_complex_darwin(self):
+ self.check('ghc-7.0.4-x86_64-apple-darwin',
+ 'ghc-7.0.4')
+ self.check('ghc-7.0.4-i386-apple-darwin',
+ 'ghc-7.0.4')
+
+ def test_complex_arch(self):
+ self.check('VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install',
+ 'VizGlow_v2.2alpha17-R21November2016')
+ self.check('jdk-8u92-linux-x64',
+ 'jdk-8u92')
+ self.check('cuda_6.5.14_linux_64.run',
+ 'cuda_6.5.14')
+
+ def test_complex_with(self):
+ self.check('mafft-7.221-with-extensions-src',
+ 'mafft-7.221')
+ self.check('spark-2.0.0-bin-without-hadoop',
+ 'spark-2.0.0')
+
+ def test_complex_public(self):
+ self.check('dakota-6.3-public.src',
+ 'dakota-6.3')
+
+ def test_complex_universal(self):
+ self.check('synergy-1.3.6p2-MacOSX-Universal',
+ 'synergy-1.3.6p2')
+
+
+class UrlStripNameSuffixesTest(unittest.TestCase):
+ """Tests for spack.url.strip_name_suffixes"""
+
+ def check(self, before, version, after):
+ stripped = strip_name_suffixes(before, version)
+ self.assertEqual(stripped, after)
+
+ def test_no_suffix(self):
+ self.check('rgb-1.0.6', '1.0.6',
+ 'rgb')
+ self.check('nauty26r7', '26r7',
+ 'nauty')
+
+ # Download type
+
+ def test_install(self):
+ self.check('converge_install_2.3.16', '2.3.16',
+ 'converge')
+
+ def test_src(self):
+ self.check('jpegsrc.v9b', '9b',
+ 'jpeg')
+
+ def test_std(self):
+ self.check('ghostscript-fonts-std-8.11', '8.11',
+ 'ghostscript-fonts')
+
+ # Download version
+
+ def test_snapshot(self):
+ self.check('gts-snapshot-121130', '121130',
+ 'gts')
+
+ def test_distrib(self):
+ self.check('zoltan_distrib_v3.83', '3.83',
+ 'zoltan')
+
+ # VCS
- def test_version_sourceforge_download(self):
+ def test_bazaar(self):
+ self.check('libvterm-0+bzr681', '681',
+ 'libvterm')
+
+ # License
+
+ def test_gpl(self):
+ self.check('PyQt-x11-gpl-4.11.3', '4.11.3',
+ 'PyQt-x11')
+
+
+class UrlParseOffsetTest(unittest.TestCase):
+
+ def check(self, name, noffset, ver, voffset, path):
+ # Make sure parse_name_offset and parse_name_version are working
+ v, vstart, vlen, vi, vre = parse_version_offset(path)
+ n, nstart, nlen, ni, nre = parse_name_offset(path, v)
+
+ self.assertEqual(n, name)
+ self.assertEqual(v, ver)
+ self.assertEqual(nstart, noffset)
+ self.assertEqual(vstart, voffset)
+
+ def test_name_in_path(self):
self.check(
- 'foo-bar', '1.21',
- 'http://sourceforge.net/foo_bar-1.21.tar.gz/download')
+ 'antlr', 25, '2.7.7', 40,
+ 'https://github.com/antlr/antlr/tarball/v2.7.7')
+
+ def test_name_in_stem(self):
self.check(
- 'foo-bar', '1.21',
- 'http://sf.net/foo_bar-1.21.tar.gz/download')
+ 'gmp', 32, '6.0.0a', 36,
+ 'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2')
- def test_no_version(self):
- self.assert_not_detected('http://example.com/blah.tar')
- self.assert_not_detected('foo')
+ def test_name_in_suffix(self):
+ # Don't think I've ever seen one of these before
+ # We don't look for it, so it would probably fail anyway
+ pass
- def test_version_all_dots(self):
+ def test_version_in_path(self):
self.check(
- 'foo-bar-la', '1.14', 'http://example.com/foo.bar.la.1.14.zip')
+ 'nextflow', 31, '0.20.1', 59,
+ 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow')
- def test_version_underscore_separator(self):
+ def test_version_in_stem(self):
self.check(
- 'grc', '1.1',
- 'http://example.com/grc_1.1.tar.gz')
-
- def test_boost_version_style(self):
+ 'zlib', 24, '1.2.10', 29,
+ 'http://zlib.net/fossils/zlib-1.2.10.tar.gz')
+ self.check(
+ 'slepc', 51, '3.6.2', 57,
+ 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz')
self.check(
- 'boost', '1.39.0',
- 'http://example.com/boost_1_39_0.tar.bz2',
- no_check_url=True)
+ 'cloog', 61, '0.18.1', 67,
+ 'http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz')
+ self.check(
+ 'libxc', 58, '2.2.2', 64,
+ 'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz')
- def test_erlang_version_style(self):
+ def test_version_in_suffix(self):
+ self.check(
+ 'swiftsim', 36, '0.3.0', 76,
+ 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0')
self.check(
- 'otp', 'R13B',
- 'http://erlang.org/download/otp_src_R13B.tar.gz')
+ 'sionlib', 30, '1.7.1', 59,
+ 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
- def test_another_erlang_version_style(self):
+ def test_regex_in_name(self):
self.check(
- 'otp', 'R15B01',
- 'https://github.com/erlang/otp/tarball/OTP_R15B01')
+ 'voro++', 40, '0.4.6', 47,
+ 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz')
+
+
+class UrlParseNameAndVersionTest(unittest.TestCase):
+
+ def assert_not_detected(self, string):
+ self.assertRaises(
+ UndetectableVersionError, parse_name_and_version, string)
+
+ def check(self, name, v, string, **kwargs):
+ # Make sure correct name and version are extracted.
+ parsed_name, parsed_v = parse_name_and_version(string)
+ self.assertEqual(parsed_name, name)
+ self.assertEqual(parsed_v, Version(v))
+
+ # Make sure Spack formulates the right URL when we try to
+ # build one with a specific version.
+ self.assertEqual(string, substitute_version(string, v))
+
+ # Common Repositories
- def test_yet_another_erlang_version_style(self):
+ def test_github_downloads(self):
+ # name/archive/ver.ver
self.check(
- 'otp', 'R15B03-1',
- 'https://github.com/erlang/otp/tarball/OTP_R15B03-1')
+ 'nco', '4.6.2',
+ 'https://github.com/nco/nco/archive/4.6.2.tar.gz')
+ # name/archive/vver.ver
+ self.check(
+ 'vim', '8.0.0134',
+ 'https://github.com/vim/vim/archive/v8.0.0134.tar.gz')
+ # name/archive/name-ver.ver
+ self.check(
+ 'oce', '0.18',
+ 'https://github.com/tpaviot/oce/archive/OCE-0.18.tar.gz')
+ # name/releases/download/vver/name-ver.ver
+ self.check(
+ 'libmesh', '1.0.0',
+ 'https://github.com/libMesh/libmesh/releases/download/v1.0.0/libmesh-1.0.0.tar.bz2')
+ # name/tarball/vver.ver
+ self.check(
+ 'git', '2.7.1',
+ 'https://github.com/git/git/tarball/v2.7.1')
+ # name/zipball/vver.ver
+ self.check(
+ 'git', '2.7.1',
+ 'https://github.com/git/git/zipball/v2.7.1')
- def test_p7zip_version_style(self):
+ def test_gitlab_downloads(self):
+ # name/repository/archive.ext?ref=vver.ver
+ self.check(
+ 'swiftsim', '0.3.0',
+ 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0')
+ # name/repository/archive.ext?ref=name-ver.ver
self.check(
- 'p7zip', '9.04',
- 'http://kent.dl.sourceforge.net/sourceforge/p7zip/p7zip_9.04_src_all.tar.bz2')
+ 'icet', '1.2.3',
+ 'https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3')
- def test_new_github_style(self):
+ def test_bitbucket_downloads(self):
+ # name/get/ver.ver
self.check(
- 'libnet', '1.1.4',
- 'https://github.com/sam-github/libnet/tarball/libnet-1.1.4')
+ 'eigen', '3.2.7',
+ 'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2')
+ # name/get/vver.ver
+ self.check(
+ 'hoomd-blue', '1.3.3',
+ 'https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2')
+ # name/downloads/name-ver.ver
+ self.check(
+ 'dolfin', '2016.1.0',
+ 'https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-2016.1.0.tar.gz')
- def test_gloox_beta_style(self):
+ def test_sourceforge_downloads(self):
+ # name-ver.ver
self.check(
- 'gloox', '1.0-beta7',
- 'http://camaya.net/download/gloox-1.0-beta7.tar.bz2')
+ 'libpng', '1.6.27',
+ 'http://download.sourceforge.net/libpng/libpng-1.6.27.tar.gz')
+ self.check(
+ 'lcms2', '2.6',
+ 'http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz')
+ self.check(
+ 'modules', '3.2.10',
+ 'http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz')
+ # name-ver.ver.ext/download
+ self.check(
+ 'glew', '2.0.0',
+ 'https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download')
- def test_sphinx_beta_style(self):
+ def test_cran_downloads(self):
+ # name.name_ver.ver-ver.ver
self.check(
- 'sphinx', '1.10-beta',
- 'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz')
+ 'TH.data', '1.0-8',
+ 'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz')
+ self.check(
+ 'knitr', '1.14',
+ 'https://cran.rstudio.com/src/contrib/knitr_1.14.tar.gz')
+ self.check(
+ 'devtools', '1.12.0',
+ 'https://cloud.r-project.org/src/contrib/devtools_1.12.0.tar.gz')
- def test_astyle_verson_style(self):
+ def test_pypi_downloads(self):
+ # name.name_name-ver.ver
+ self.check(
+ '3to2', '1.1.1',
+ 'https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip')
self.check(
- 'astyle', '1.23',
- 'http://kent.dl.sourceforge.net/sourceforge/astyle/astyle_1.23_macosx.tar.gz')
+ 'mpmath', '0.19',
+ 'https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz')
+ self.check(
+ 'pandas', '0.16.0',
+ 'https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73')
+ self.check(
+ 'sphinx_rtd_theme', '0.1.10a0',
+ 'https://pypi.python.org/packages/da/6b/1b75f13d8aa3333f19c6cdf1f0bc9f52ea739cae464fbee050307c121857/sphinx_rtd_theme-0.1.10a0.tar.gz')
+ self.check(
+ 'backports.ssl_match_hostname', '3.5.0.1',
+ 'https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz')
- def test_version_dos2unix(self):
+ def test_bazaar_downloads(self):
self.check(
- 'dos2unix', '3.1',
- 'http://www.sfr-fresh.com/linux/misc/dos2unix-3.1.tar.gz')
+ 'libvterm', '681',
+ 'http://www.leonerd.org.uk/code/libvterm/libvterm-0+bzr681.tar.gz')
- def test_version_internal_dash(self):
+ # Common Tarball Formats
+
+ def test_version_only(self):
+ # ver.ver
+ self.check(
+ 'eigen', '3.2.7',
+ 'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2')
+ # ver.ver-ver
+ self.check(
+ 'ImageMagick', '7.0.2-7',
+ 'https://github.com/ImageMagick/ImageMagick/archive/7.0.2-7.tar.gz')
+ # vver.ver
self.check(
- 'foo-arse', '1.1-2',
- 'http://example.com/foo-arse-1.1-2.tar.gz')
+ 'CGNS', '3.3.0',
+ 'https://github.com/CGNS/CGNS/archive/v3.3.0.tar.gz')
+ # vver_ver
+ self.check(
+ 'luafilesystem', '1_6_3',
+ 'https://github.com/keplerproject/luafilesystem/archive/v1_6_3.tar.gz')
- def test_version_single_digit(self):
+ def test_no_separators(self):
+ # namever
+ self.check(
+ 'turbolinux', '702',
+ 'file://{0}/turbolinux702.tar.gz'.format(os.getcwd()))
self.check(
- 'foo-bar', '45',
- 'http://example.com/foo_bar.45.tar.gz')
+ 'nauty', '26r7',
+ 'http://pallini.di.uniroma1.it/nauty26r7.tar.gz')
- def test_noseparator_single_digit(self):
+ def test_dashes_only(self):
+ # name-name-ver-ver
+ self.check(
+ 'Trilinos', '12-10-1',
+ 'https://github.com/trilinos/Trilinos/archive/trilinos-release-12-10-1.tar.gz')
+ self.check(
+ 'panda', '2016-03-07',
+ 'http://comopt.ifi.uni-heidelberg.de/software/PANDA/downloads/panda-2016-03-07.tar')
self.check(
- 'foo-bar', '45',
- 'http://example.com/foo_bar45.tar.gz')
+ 'gts', '121130',
+ 'http://gts.sourceforge.net/tarballs/gts-snapshot-121130.tar.gz')
+ self.check(
+ 'cdd', '061a',
+ 'http://www.cs.mcgill.ca/~fukuda/download/cdd/cdd-061a.tar.gz')
- def test_version_developer_that_hates_us_format(self):
+ def test_underscores_only(self):
+ # name_name_ver_ver
+ self.check(
+ 'tinyxml', '2_6_2',
+ 'https://sourceforge.net/projects/tinyxml/files/tinyxml/2.6.2/tinyxml_2_6_2.tar.gz')
+ self.check(
+ 'boost', '1_55_0',
+ 'http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2')
self.check(
- 'foo-bar-la', '1.2.3',
- 'http://example.com/foo-bar-la.1.2.3.tar.gz')
+ 'yorick', '2_2_04',
+ 'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz')
+ # name_namever_ver
+ self.check(
+ 'tbb', '44_20160413',
+ 'https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz')
- def test_version_regular(self):
+ def test_dots_only(self):
+ # name.name.ver.ver
+ self.check(
+ 'prank', '150803',
+ 'http://wasabiapp.org/download/prank/prank.source.150803.tgz')
+ self.check(
+ 'jpeg', '9b',
+ 'http://www.ijg.org/files/jpegsrc.v9b.tar.gz')
+ self.check(
+ 'openjpeg', '2.1',
+ 'https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz')
+ # name.namever.ver
+ self.check(
+ 'atlas', '3.11.34',
+ 'http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2')
self.check(
- 'foo-bar', '1.21',
- 'http://example.com/foo_bar-1.21.tar.gz')
+ 'visit', '2.10.1',
+ 'http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz')
+ self.check(
+ 'geant', '4.10.01.p03',
+ 'http://geant4.cern.ch/support/source/geant4.10.01.p03.tar.gz')
+ self.check(
+ 'tcl', '8.6.5',
+ 'http://prdownloads.sourceforge.net/tcl/tcl8.6.5-src.tar.gz')
- def test_version_gitlab(self):
+ def test_dash_dot(self):
+ # name-name-ver.ver
+ # digit in name
self.check(
- 'vtk', '7.0.0',
- 'https://gitlab.kitware.com/vtk/vtk/repository/'
- 'archive.tar.bz2?ref=v7.0.0')
+ 'm4', '1.4.17',
+ 'https://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz')
+ # letter in version
self.check(
- 'icet', '1.2.3',
- 'https://gitlab.kitware.com/icet/icet/repository/'
- 'archive.tar.gz?ref=IceT-1.2.3')
+ 'gmp', '6.0.0a',
+ 'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2')
+ # version starts with 'v'
+ self.check(
+ 'LaunchMON', '1.0.2',
+ 'https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz')
+ # name-ver-ver.ver
self.check(
- 'foo', '42.1337',
- 'http://example.com/org/foo/repository/'
- 'archive.zip?ref=42.1337bar')
+ 'libedit', '20150325-3.1',
+ 'http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz')
- def test_version_github(self):
+ def test_dash_underscore(self):
+ # name-name-ver_ver
self.check(
- 'yajl', '1.0.5',
- 'http://github.com/lloyd/yajl/tarball/1.0.5')
+ 'icu4c', '57_1',
+ 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz')
- def test_version_github_with_high_patch_number(self):
+ def test_underscore_dot(self):
+ # name_name_ver.ver
self.check(
- 'yajl', '1.2.34',
- 'http://github.com/lloyd/yajl/tarball/v1.2.34')
+ 'superlu_dist', '4.1',
+ 'http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz')
+ self.check(
+ 'pexsi', '0.9.0',
+ 'https://math.berkeley.edu/~linlin/pexsi/download/pexsi_v0.9.0.tar.gz')
+ # name_name.ver.ver
+ self.check(
+ 'fer', '696',
+ 'ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.v696.tar.gz')
- def test_yet_another_version(self):
+ def test_dash_dot_dash_dot(self):
+ # name-name-ver.ver-ver.ver
+ self.check(
+ 'sowing', '1.1.23-p1',
+ 'http://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz')
self.check(
- 'mad', '0.15.1b',
- 'http://example.com/mad-0.15.1b.tar.gz')
+ 'bib2xhtml', '3.0-15-gf506',
+ 'http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz')
+ # namever.ver-ver.ver
+ self.check(
+ 'go', '1.4-bootstrap-20161024',
+ 'https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz')
- def test_lame_version_style(self):
+ def test_underscore_dash_dot(self):
+ # name_name-ver.ver
+ self.check(
+ 'the_silver_searcher', '0.32.0',
+ 'http://geoff.greer.fm/ag/releases/the_silver_searcher-0.32.0.tar.gz')
self.check(
- 'lame', '398-2',
- 'http://kent.dl.sourceforge.net/sourceforge/lame/lame-398-2.tar.gz')
+ 'sphinx_rtd_theme', '0.1.10a0',
+ 'https://pypi.python.org/packages/source/s/sphinx_rtd_theme/sphinx_rtd_theme-0.1.10a0.tar.gz')
- def test_ruby_version_style(self):
+ def test_dot_underscore_dot_dash_dot(self):
+ # name.name_ver.ver-ver.ver
self.check(
- 'ruby', '1.9.1-p243',
- 'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz')
+ 'TH.data', '1.0-8',
+ 'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz')
+ self.check(
+ 'XML', '3.98-1.4',
+ 'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz')
- def test_omega_version_style(self):
+ def test_dash_dot_underscore_dot(self):
+ # name-name-ver.ver_ver.ver
+ self.check(
+ 'pypar', '2.1.5_108',
+ 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pypar/pypar-2.1.5_108.tgz')
+ # name-namever.ver_ver.ver
self.check(
- 'omega', '0.80.2',
- 'http://www.alcyone.com/binaries/omega/omega-0.80.2-src.tar.gz')
+ 'STAR-CCM+', '11.06.010_02',
+ 'file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz'.format(os.getcwd()))
- def test_rc_style(self):
+ # Weird URLS
+
+ def test_version_in_path(self):
+ # github.com/repo/name/releases/download/name-vver/name
self.check(
- 'libvorbis', '1.2.2rc1',
- 'http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2')
+ 'nextflow', '0.20.1',
+ 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow')
- def test_dash_rc_style(self):
+ def test_suffix_queries(self):
self.check(
- 'js', '1.8.0-rc1',
- 'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz')
+ 'swiftsim', '0.3.0',
+ 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0')
+ self.check(
+ 'sionlib', '1.7.1',
+ 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
- def test_angband_version_style(self):
+ def test_stem_queries(self):
+ self.check(
+ 'slepc', '3.6.2',
+ 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz')
self.check(
- 'angband', '3.0.9b',
- 'http://rephial.org/downloads/3.0/angband-3.0.9b-src.tar.gz')
+ 'otf', '1.12.5salmon',
+ 'http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz')
- def test_stable_suffix(self):
+ def test_single_character_name(self):
self.check(
- 'libevent', '1.4.14b',
- 'http://www.monkey.org/~provos/libevent-1.4.14b-stable.tar.gz')
+ 'R', '3.3.2',
+ 'https://cloud.r-project.org/src/base/R-3/R-3.3.2.tar.gz')
+
+ def test_single_digit_version(self):
+ pass
- def test_debian_style_1(self):
+ def test_name_starts_with_digit(self):
self.check(
- 'sl', '3.03',
- 'http://ftp.de.debian.org/debian/pool/main/s/sl/sl_3.03.orig.tar.gz')
+ '3to2', '1.1.1',
+ 'https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip')
- def test_debian_style_2(self):
+ def plus_in_name(self):
self.check(
- 'mmv', '1.01b',
- 'http://ftp.de.debian.org/debian/pool/main/m/mmv/mmv_1.01b.orig.tar.gz')
+ 'gtk+', '2.24.31',
+ 'http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.31.tar.xz')
+ self.check(
+ 'voro++', '0.4.6',
+ 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz')
+
+ def test_no_version(self):
+ self.assert_not_detected('http://www.netlib.org/blas/blast-forum/cblas.tgz')
+ self.assert_not_detected('http://www.netlib.org/voronoi/triangle.zip')
- def test_imagemagick_style(self):
+ def test_download_php(self):
+ # Name comes before download.php
+ self.check(
+ 'sionlib', '1.7.1',
+ 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
+ # Ignore download.php
+ self.check(
+ 'slepc', '3.6.2',
+ 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz')
self.check(
- 'imagemagick', '6.7.5-7',
+ 'ScientificPython', '2.8.1',
+ 'https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz')
- 'http://downloads.sf.net/project/machomebrew/mirror/ImageMagick-6.7.5-7.tar.bz2')
+ def test_gloox_beta_style(self):
+ self.check(
+ 'gloox', '1.0-beta7',
+ 'http://camaya.net/download/gloox-1.0-beta7.tar.bz2')
- def test_dash_version_dash_style(self):
+ def test_sphinx_beta_style(self):
self.check(
- 'antlr', '3.4',
- 'http://www.antlr.org/download/antlr-3.4-complete.jar')
+ 'sphinx', '1.10-beta',
+ 'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz')
- def test_apache_version_style(self):
+ def test_ruby_version_style(self):
self.check(
- 'apache-cassandra', '1.2.0-rc2',
- 'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz')
+ 'ruby', '1.9.1-p243',
+ 'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz')
- def test_jpeg_style(self):
+ def test_rc_style(self):
self.check(
- 'jpegsrc', '8d',
- 'http://www.ijg.org/files/jpegsrc.v8d.tar.gz')
+ 'libvorbis', '1.2.2rc1',
+ 'http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2')
- def test_pypy_version(self):
+ def test_dash_rc_style(self):
self.check(
- 'pypy', '1.4.1',
- 'http://pypy.org/download/pypy-1.4.1-osx.tar.bz2')
+ 'js', '1.8.0-rc1',
+ 'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz')
- def test_openssl_version(self):
+ def test_apache_version_style(self):
self.check(
- 'openssl', '0.9.8s',
- 'http://www.openssl.org/source/openssl-0.9.8s.tar.gz')
+ 'apache-cassandra', '1.2.0-rc2',
+ 'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz')
def test_xaw3d_version(self):
self.check(
- 'xaw3d', '1.5E',
+ 'Xaw3d', '1.5E',
'ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz')
def test_fann_version(self):
@@ -269,16 +690,6 @@ class UrlParseTest(unittest.TestCase):
'fann', '2.1.0beta',
'http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip')
- def test_iges_version(self):
- self.check(
- 'grads', '2.0.1',
- 'ftp://iges.org/grads/2.0/grads-2.0.1-bin-darwin9.8-intel.tar.gz')
-
- def test_haxe_version(self):
- self.check(
- 'haxe', '2.08',
- 'http://haxe.org/file/haxe-2.08-osx.tar.gz')
-
def test_imap_version(self):
self.check(
'imap', '2007f',
@@ -289,26 +700,6 @@ class UrlParseTest(unittest.TestCase):
'suite3270', '3.3.12ga7',
'http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz')
- def test_synergy_version(self):
- self.check(
- 'synergy', '1.3.6p2',
- 'http://synergy.googlecode.com/files/synergy-1.3.6p2-MacOSX-Universal.zip')
-
- def test_mvapich2_19_version(self):
- self.check(
- 'mvapich2', '1.9',
- 'http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz')
-
- def test_mvapich2_20_version(self):
- self.check(
- 'mvapich2', '2.0',
- 'http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz')
-
- def test_hdf5_version(self):
- self.check(
- 'hdf5', '1.8.13',
- 'http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2')
-
def test_scalasca_version(self):
self.check(
'cube', '4.2.3',
@@ -317,55 +708,20 @@ class UrlParseTest(unittest.TestCase):
'cube', '4.3-TP1',
'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz')
- def test_mpileaks_version(self):
- self.check(
- 'mpileaks', '1.0',
- 'https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz')
- self.check(
- 'mpileaks', '1.0',
- 'https://github.com/hpc/mpileaks/releases/download/1.0/mpileaks-1.0.tar.gz')
-
- def test_gcc_version(self):
- self.check(
- 'gcc', '4.4.7',
- 'http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2')
-
- def test_gcc_version_precedence(self):
- # prefer the version in the tarball, not in the url prefix.
- self.check(
- 'gcc', '4.4.7',
- 'http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.4.7.tar.bz2')
-
def test_github_raw_url(self):
self.check(
- 'powerparser', '2.0.7',
+ 'CLAMR', '2.0.7',
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true')
- def test_r_xml_version(self):
+ def test_luaposix_version(self):
self.check(
- 'xml', '3.98-1.4',
- 'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz')
+ 'luaposix', '33.4.0',
+ 'https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz')
def test_nco_version(self):
self.check(
'nco', '4.6.2-beta03',
'https://github.com/nco/nco/archive/4.6.2-beta03.tar.gz')
-
self.check(
'nco', '4.6.3-alpha04',
'https://github.com/nco/nco/archive/4.6.3-alpha04.tar.gz')
-
- def test_yorick_version(self):
- self.check(
- 'yorick', '2_2_04',
- 'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz')
-
- def test_luaposix_version(self):
- self.check(
- 'luaposix', '33.4.0',
- 'https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz')
-
- def test_sionlib_version(self):
- self.check(
- 'sionlib', '1.7.1',
- 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py
index ea6374e3d2..449a3b29bf 100644
--- a/lib/spack/spack/test/url_substitution.py
+++ b/lib/spack/spack/test/url_substitution.py
@@ -22,44 +22,64 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-"""\
-This test does sanity checks on substituting new versions into URLs
-"""
+"""Tests Spack's ability to substitute a different version into a URL."""
+
+import os
import unittest
-import spack.url as url
+from spack.url import substitute_version
+
+
+class UrlSubstitutionTest(unittest.TestCase):
+ def check(self, base, version, new_url):
+ self.assertEqual(substitute_version(base, version), new_url)
-base = "https://comp.llnl.gov/linear_solvers/download/hypre-2.9.0b.tar.gz"
-stem = "https://comp.llnl.gov/linear_solvers/download/hypre-"
+ def test_same_version(self):
+ # Ensures that substituting the same version results in the same URL
+ self.check(
+ 'http://www.mr511.de/software/libelf-0.8.13.tar.gz', '0.8.13',
+ 'http://www.mr511.de/software/libelf-0.8.13.tar.gz')
+ def test_different_version(self):
+ # Test a completely different version syntax
+ self.check(
+ 'http://www.prevanders.net/libdwarf-20130729.tar.gz', '8.12',
+ 'http://www.prevanders.net/libdwarf-8.12.tar.gz')
-class PackageSanityTest(unittest.TestCase):
+ def test_double_version(self):
+ # Test a URL where the version appears twice
+ # It should get substituted both times
+ self.check(
+ 'https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz', '2.1.3',
+ 'https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz')
- def test_hypre_url_substitution(self):
- self.assertEqual(url.substitute_version(base, '2.9.0b'), base)
- self.assertEqual(
- url.substitute_version(base, '2.8.0b'), stem + "2.8.0b.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '2.7.0b'), stem + "2.7.0b.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '2.6.0b'), stem + "2.6.0b.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '1.14.0b'), stem + "1.14.0b.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '1.13.0b'), stem + "1.13.0b.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '2.0.0'), stem + "2.0.0.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '1.6.0'), stem + "1.6.0.tar.gz")
+ def test_partial_version_prefix(self):
+ # Test now with a partial prefix earlier in the URL
+ # This is hard to figure out so Spack only substitutes
+ # the last instance of the version
+ self.check(
+ 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2', '2.2.0',
+ 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.0.tar.bz2')
+ self.check(
+ 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2', '2.2',
+ 'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.tar.bz2')
- def test_otf2_url_substitution(self):
- base = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz"
+ def test_no_separator(self):
+ # No separator between the name and version of the package
+ self.check(
+ 'file://{0}/turbolinux702.tar.gz'.format(os.getcwd()), '703',
+ 'file://{0}/turbolinux703.tar.gz'.format(os.getcwd()))
- self.assertEqual(url.substitute_version(base, '1.4'), base)
+ def test_github_raw(self):
+ self.check(
+ 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '2.0.7',
+ 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true')
+ self.check(
+ 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '4.7',
+ 'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true')
- self.assertEqual(
- url.substitute_version(base, '1.3.1'),
- "http://www.vi-hps.org/upload/packages/otf2/otf2-1.3.1.tar.gz")
- self.assertEqual(
- url.substitute_version(base, '1.2.1'),
- "http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz")
+ def test_regex(self):
+ # Package name contains regex characters
+ self.check(
+ 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz', '1.2.3',
+ 'http://math.lbl.gov/voro++/download/dir/voro++-1.2.3.tar.gz')
diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py
index 9a7f4d9f8b..9fa95a8d18 100644
--- a/lib/spack/spack/test/web.py
+++ b/lib/spack/spack/test/web.py
@@ -23,7 +23,6 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""Tests for web.py."""
-import pytest
import os
import spack
@@ -141,7 +140,6 @@ def test_find_versions_of_archive_2():
assert ver('2.0.0') in versions
-@pytest.mark.xfail
def test_find_exotic_versions_of_archive_2():
versions = find_versions_of_archive(root_tarball, root, list_depth=2)
# up for grabs to make this better.
@@ -157,7 +155,6 @@ def test_find_versions_of_archive_3():
assert ver('4.5') in versions
-@pytest.mark.xfail
def test_find_exotic_versions_of_archive_3():
versions = find_versions_of_archive(root_tarball, root, list_depth=3)
assert ver('2.0.0b2') in versions
diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py
index 38ff74f7bc..174f7d0b3c 100644
--- a/lib/spack/spack/url.py
+++ b/lib/spack/spack/url.py
@@ -71,7 +71,7 @@ def find_list_url(url):
url_types = [
# e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz
- (r'^(https://github.com/[^/]+/[^/]+)/archive/',
+ (r'(.*github\.com/[^/]+/[^/]+)/archive/',
lambda m: m.group(1) + '/releases')]
for pattern, fun in url_types:
@@ -101,6 +101,177 @@ def strip_query_and_fragment(path):
return (path, '') # Ignore URL parse errors here
+def strip_version_suffixes(path):
+ """Some tarballs contain extraneous information after the version:
+
+ * ``bowtie2-2.2.5-source``
+ * ``libevent-2.0.21-stable``
+ * ``cuda_8.0.44_linux.run``
+
+ These strings are not part of the version number and should be ignored.
+ This function strips those suffixes off and returns the remaining string.
+ The goal is that the version is always the last thing in ``path``:
+
+ * ``bowtie2-2.2.5``
+ * ``libevent-2.0.21``
+ * ``cuda_8.0.44``
+
+ :param str path: The filename or URL for the package
+ :return: The ``path`` with any extraneous suffixes removed
+ :rtype: str
+ """
+ # NOTE: This could be done with complicated regexes in parse_version_offset
+ # NOTE: The problem is that we would have to add these regexes to the end
+ # NOTE: of every single version regex. Easier to just strip them off
+ # NOTE: permanently
+
+ suffix_regexes = [
+ # Download type
+ '[Ii]nstall',
+ 'all',
+ 'src(_0)?',
+ '[Ss]ources?',
+ 'file',
+ 'full',
+ 'single',
+ 'public',
+ 'with[a-zA-Z_-]+',
+ 'bin',
+ 'binary',
+ 'run',
+ '[Uu]niversal',
+ 'jar',
+ 'complete',
+ 'oss',
+ 'gem',
+ 'tar',
+ 'sh',
+
+ # Download version
+ 'stable',
+ '[Ff]inal',
+ 'rel',
+ 'orig',
+ 'dist',
+ '\+',
+
+ # License
+ 'gpl',
+
+ # Arch
+ # Needs to come before and after OS, appears in both orders
+ 'ia32',
+ 'intel',
+ 'amd64',
+ 'x64',
+ 'x86_64',
+ 'x86',
+ 'i[36]86',
+ 'ppc64(le)?',
+ 'armv?(7l|6l|64)',
+
+ # OS
+ '[Ll]inux(_64)?',
+ '[Uu]ni?x',
+ '[Ss]un[Oo][Ss]',
+ '[Mm]ac[Oo][Ss][Xx]?',
+ '[Oo][Ss][Xx]',
+ '[Dd]arwin(64)?',
+ '[Aa]pple',
+ '[Ww]indows',
+ '[Ww]in(64|32)?',
+ '[Cc]ygwin(64|32)?',
+ '[Mm]ingw',
+
+ # Arch
+ # Needs to come before and after OS, appears in both orders
+ 'ia32',
+ 'intel',
+ 'amd64',
+ 'x64',
+ 'x86_64',
+ 'x86',
+ 'i[36]86',
+ 'ppc64(le)?',
+ 'armv?(7l|6l|64)?',
+
+ # PyPI
+ '[._-]py[23].*\.whl',
+ '[._-]cp[23].*\.whl',
+ '[._-]win.*\.exe',
+ ]
+
+ for regex in suffix_regexes:
+ # Remove the suffix from the end of the path
+ # This may be done multiple times
+ path = re.sub(r'[._-]?' + regex + '$', '', path)
+
+ return path
+
+
+def strip_name_suffixes(path, version):
+ """Most tarballs contain a package name followed by a version number.
+ However, some also contain extraneous information in-between the name
+ and version:
+
+ * ``rgb-1.0.6``
+ * ``converge_install_2.3.16``
+ * ``jpegsrc.v9b``
+
+ These strings are not part of the package name and should be ignored.
+ This function strips the version number and any extraneous suffixes
+ off and returns the remaining string. The goal is that the name is
+ always the last thing in ``path``:
+
+ * ``rgb``
+ * ``converge``
+ * ``jpeg``
+
+ :param str path: The filename or URL for the package
+ :param str version: The version detected for this URL
+ :return: The ``path`` with any extraneous suffixes removed
+ :rtype: str
+ """
+ # NOTE: This could be done with complicated regexes in parse_name_offset
+ # NOTE: The problem is that we would have to add these regexes to every
+ # NOTE: single name regex. Easier to just strip them off permanently
+
+ suffix_regexes = [
+ # Strip off the version and anything after it
+
+ # name-ver
+ # name_ver
+ # name.ver
+ r'[._-]v?' + str(version) + '.*',
+
+ # namever
+ str(version) + '.*',
+
+ # Download type
+ 'install',
+ 'src',
+ '(open)?[Ss]ources?',
+ '[._-]std',
+
+ # Download version
+ 'snapshot',
+ 'distrib',
+
+ # VCS
+ '0\+bzr',
+
+ # License
+ 'gpl',
+ ]
+
+ for regex in suffix_regexes:
+ # Remove the suffix from the end of the path
+ # This may be done multiple times
+ path = re.sub('[._-]?' + regex + '$', '', path)
+
+ return path
+
+
def split_url_extension(path):
"""Some URLs have a query string, e.g.:
@@ -125,7 +296,7 @@ def split_url_extension(path):
prefix, ext, suffix = path, '', ''
# Strip off sourceforge download suffix.
- match = re.search(r'((?:sourceforge.net|sf.net)/.*)(/download)$', path)
+ match = re.search(r'((?:sourceforge\.net|sf\.net)/.*)(/download)$', path)
if match:
prefix, suffix = match.groups()
@@ -189,8 +360,20 @@ def parse_version_offset(path):
path, ext, suffix = split_url_extension(path)
# stem: Everything from path after the final '/'
- stem = os.path.basename(path)
- offset = len(path) - len(stem)
+ original_stem = os.path.basename(path)
+
+ # Try to strip off anything after the version number
+ stem = strip_version_suffixes(original_stem)
+
+ # Assumptions:
+ #
+ # 1. version always comes after the name
+ # 2. separators include '-', '_', and '.'
+ # 3. names can contain A-Z, a-z, 0-9, '+', separators
+ # 4. versions can contain A-Z, a-z, 0-9, separators
+ # 5. versions always start with a digit
+ # 6. versions are often prefixed by a 'v' character
+ # 7. separators are most reliable to determine name/version boundaries
# List of the following format:
#
@@ -202,87 +385,118 @@ def parse_version_offset(path):
# The first regex that matches string will be used to determine
# the version of the package. Thefore, hyperspecific regexes should
# come first while generic, catch-all regexes should come last.
+ # With that said, regular expressions are slow, so if possible, put
+ # ones that only catch one or two URLs at the bottom.
version_regexes = [
- # GitHub tarballs, e.g. v1.2.3
- (r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+)$', path),
+ # 1st Pass: Simplest case
+ # Assume name contains no digits and version contains no letters
+ # e.g. libpng-1.6.27
+ (r'^[a-zA-Z+._-]+[._-]v?(\d[\d._-]*)$', stem),
- # e.g. https://github.com/sam-github/libnet/tarball/libnet-1.1.4
- (r'github.com/.+/(?:zip|tar)ball/.*-((\d+\.)+\d+)$', path),
+ # 2nd Pass: Version only
+ # Assume version contains no letters
- # e.g. https://github.com/isaacs/npm/tarball/v0.2.5-1
- (r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+-(\d+))$', path),
+ # ver
+ # e.g. 3.2.7, 7.0.2-7, v3.3.0, v1_6_3
+ (r'^v?(\d[\d._-]*)$', stem),
- # e.g. https://github.com/petdance/ack/tarball/1.93_02
- (r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+_(\d+))$', path),
+ # 3rd Pass: No separator characters are used
+ # Assume name contains no digits
- # Yorick is very special.
- # e.g. https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz
- (r'github.com/[^/]+/yorick/archive/y_(\d+(?:_\d+)*)$', path),
+ # namever
+ # e.g. turbolinux702, nauty26r7
+ (r'^[a-zA-Z+]*(\d[\da-zA-Z]*)$', stem),
- # e.g. https://github.com/hpc/lwgrp/archive/v1.0.1.tar.gz
- (r'github.com/[^/]+/[^/]+/archive/(?:release-)?v?(\w+(?:[.-]\w+)*)$', path), # noqa
+ # 4th Pass: A single separator character is used
+ # Assume name contains no digits
- # e.g. https://github.com/erlang/otp/tarball/OTP_R15B01 (erlang style)
- (r'[-_](R\d+[AB]\d*(-\d+)?)', path),
+ # name-name-ver-ver
+ # e.g. panda-2016-03-07, gts-snapshot-121130, cdd-061a
+ (r'^[a-zA-Z+-]*(\d[\da-zA-Z-]*)$', stem),
- # e.g., https://github.com/hpc/libcircle/releases/download/0.2.1-rc.1/libcircle-0.2.1-rc.1.tar.gz
- # e.g.,
- # https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
- (r'github.com/[^/]+/[^/]+/releases/download/v?([^/]+)/.*$', path),
+ # name_name_ver_ver
+ # e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128, v1_6_3
+ (r'^[a-zA-Z+_]*(\d[\da-zA-Z_]*)$', stem),
- # GitLab syntax:
- # {baseUrl}{/organization}{/projectName}/repository/archive.{fileEnding}?ref={gitTag}
- # as with github releases, we hope a version can be found in the
- # git tag
- # Search dotted versions:
- # e.g., https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0
- # e.g., https://example.com/org/repo/repository/archive.tar.bz2?ref=SomePrefix-2.1.1
- # e.g., http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1
- (r'\?ref=(?:.*-|v)*((\d+\.)+\d+).*$', suffix),
- (r'\?version=((\d+\.)+\d+)', suffix),
+ # name.name.ver.ver
+ # e.g. prank.source.150803, jpegsrc.v9b, atlas3.11.34, geant4.10.01.p03
+ (r'^[a-zA-Z+.]*(\d[\da-zA-Z.]*)$', stem),
- # e.g. boost_1_39_0
- (r'((\d+_)+\d+)$', stem),
+ # 5th Pass: Two separator characters are used
+ # Name may contain digits, version may contain letters
- # e.g. foobar-4.5.1-1
- # e.g. ruby-1.9.1-p243
- (r'-((\d+\.)*\d\.\d+-(p|rc|RC)?\d+)(?:[-._](?:bin|dist|stable|src|sources))?$', stem), # noqa
+ # name-name-ver.ver
+ # e.g. m4-1.4.17, gmp-6.0.0a, launchmon-v1.0.2
+ (r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem),
- # e.g. lame-398-1
- (r'-((\d)+-\d)', stem),
+ # name-name-ver_ver
+ # e.g. icu4c-57_1
+ (r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z_]*)$', stem),
- # e.g. foobar_1.2-3 or 3.98-1.4
- (r'_((\d+\.)+\d+(-(\d+(\.\d+)?))?[a-z]?)', stem),
+ # name_name_ver.ver
+ # e.g. superlu_dist_4.1, pexsi_v0.9.0
+ (r'^[a-zA-Z\d+_]+_v?(\d[\da-zA-Z.]*)$', stem),
- # e.g. foobar-4.5.1
- (r'-((\d+\.)*\d+)$', stem),
+ # name_name.ver.ver
+ # e.g. fer_source.v696
+ (r'^[a-zA-Z\d+_]+\.v?(\d[\da-zA-Z.]*)$', stem),
- # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b
- (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem),
+ # name-name-ver.ver-ver.ver
+ # e.g. sowing-1.1.23-p1, bib2xhtml-v3.0-15-gf506, 4.6.3-alpha04
+ (r'^(?:[a-zA-Z\d+-]+-)?v?(\d[\da-zA-Z.-]*)$', stem),
- # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta
- (r'-((\d+\.)*\d+-beta(\d+)?)$', stem),
+ # namever.ver-ver.ver
+ # e.g. go1.4-bootstrap-20161024
+ (r'^[a-zA-Z+]+v?(\d[\da-zA-Z.-]*)$', stem),
- # e.g. foobar4.5.1
- (r'((\d+\.)*\d+)$', stem),
+ # 6th Pass: All three separator characters are used
+ # Name may contain digits, version may contain letters
- # e.g. foobar-4.5.0-bin
- (r'-((\d+\.)+\d+[a-z]?)[-._](bin|dist|stable|src|sources?)$', stem),
+ # name_name-ver.ver
+ # e.g. the_silver_searcher-0.32.0, sphinx_rtd_theme-0.1.10a0
+ (r'^[a-zA-Z\d+_]+-v?(\d[\da-zA-Z.]*)$', stem),
- # e.g. dash_0.5.5.1.orig.tar.gz (Debian style)
- (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem),
+ # name.name_ver.ver-ver.ver
+ # e.g. TH.data_1.0-8, XML_3.98-1.4
+ (r'^[a-zA-Z\d+.]+_v?(\d[\da-zA-Z.-]*)$', stem),
- # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz
- (r'-v?([^-]+(-alpha|-beta)?)', stem),
+ # name-name-ver.ver_ver.ver
+ # e.g. pypar-2.1.5_108
+ (r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z._]*)$', stem),
- # e.g. astyle_1.23_macosx.tar.gz
- (r'_([^_]+(_alpha|_beta)?)', stem),
+ # name.name_name-ver.ver
+ # e.g. tap.py-1.6, backports.ssl_match_hostname-3.5.0.1
+ (r'^[a-zA-Z\d+._]+-v?(\d[\da-zA-Z.]*)$', stem),
- # e.g. http://mirrors.jenkins-ci.org/war/1.486/jenkins.war
- (r'\/(\d\.\d+)\/', path),
+ # name-namever.ver_ver.ver
+ # e.g. STAR-CCM+11.06.010_02
+ (r'^[a-zA-Z+-]+(\d[\da-zA-Z._]*)$', stem),
- # e.g. http://www.ijg.org/files/jpegsrc.v8d.tar.gz
- (r'\.v(\d+[a-z]?)', stem)
+ # 7th Pass: Specific VCS
+
+ # bazaar
+ # e.g. libvterm-0+bzr681
+ (r'bzr(\d[\da-zA-Z._-]*)$', stem),
+
+ # 8th Pass: Version in path
+
+ # github.com/repo/name/releases/download/vver/name
+ # e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow
+ (r'github\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/', path), # noqa
+
+ # 9th Pass: Query strings
+
+ # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0
+ (r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix),
+
+ # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1
+ (r'\?version=v?(\d[\da-zA-Z._-]*)$', suffix),
+
+ # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz
+ (r'\?filename=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem),
+
+ # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz
+ (r'\?package=[a-zA-Z\d+-]+&get=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), # noqa
]
for i, version_regex in enumerate(version_regexes):
@@ -292,9 +506,15 @@ def parse_version_offset(path):
version = match.group(1)
start = match.start(1)
- # if we matched from the basename, then add offset in.
+ # If we matched from the stem or suffix, we need to add offset
+ offset = 0
if match_string is stem:
- start += offset
+ offset = len(path) - len(original_stem)
+ elif match_string is suffix:
+ offset = len(path)
+ if ext:
+ offset += len(ext) + 1 # .tar.gz is converted to tar.gz
+ start += offset
return version, start, len(version), i, regex
@@ -342,7 +562,7 @@ def parse_name_offset(path, v=None):
except UndetectableVersionError:
# Not all URLs contain a version. We still want to be able
# to determine a name if possible.
- v = ''
+ v = 'unknown'
# path: The prefix of the URL, everything before the ext and suffix
# ext: The file extension
@@ -350,8 +570,10 @@ def parse_name_offset(path, v=None):
path, ext, suffix = split_url_extension(path)
# stem: Everything from path after the final '/'
- stem = os.path.basename(path)
- offset = len(path) - len(stem)
+ original_stem = os.path.basename(path)
+
+ # Try to strip off anything after the package name
+ stem = strip_name_suffixes(original_stem, v)
# List of the following format:
#
@@ -363,26 +585,45 @@ def parse_name_offset(path, v=None):
# The first regex that matches string will be used to determine
# the name of the package. Thefore, hyperspecific regexes should
# come first while generic, catch-all regexes should come last.
+ # With that said, regular expressions are slow, so if possible, put
+ # ones that only catch one or two URLs at the bottom.
name_regexes = [
- (r'/sourceforge/([^/]+)/', path),
- (r'github.com/[^/]+/[^/]+/releases/download/%s/(.*)-%s$' %
- (v, v), path),
- (r'/([^/]+)/(tarball|zipball)/', path),
- (r'/([^/]+)[_.-](bin|dist|stable|src|sources)[_.-]%s' % v, path),
- (r'github.com/[^/]+/([^/]+)/archive', path),
- (r'github.com/[^/]+/([^/]+)/releases', path),
- (r'[^/]+/([^/]+)/repository/archive', path), # gitlab
- (r'([^/]+)/download.php', path),
-
- (r'([^/]+)[_.-]v?%s' % v, stem), # prefer the stem
- (r'([^/]+)%s' % v, stem),
-
- # accept the path if name is not in stem.
- (r'/([^/]+)[_.-]v?%s' % v, path),
- (r'/([^/]+)%s' % v, path),
-
- (r'^([^/]+)[_.-]v?%s' % v, path),
- (r'^([^/]+)%s' % v, path)
+ # 1st Pass: Common repositories
+
+ # GitHub: github.com/repo/name/
+ # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz
+ (r'github\.com/[^/]+/([^/]+)', path),
+
+ # GitLab: gitlab.*/repo/name/
+ # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0
+ (r'gitlab[^/]+/[^/]+/([^/]+)', path),
+
+ # Bitbucket: bitbucket.org/repo/name/
+ # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2
+ (r'bitbucket\.org/[^/]+/([^/]+)', path),
+
+ # PyPI: pypi.(python.org|io)/packages/source/first-letter/name/
+ # e.g. https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz
+ # e.g. https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz
+ (r'pypi\.(?:python\.org|io)/packages/source/[A-Za-z\d]/([^/]+)', path),
+
+ # 2nd Pass: Query strings
+
+ # ?filename=name-ver.ver
+ # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz
+ (r'\?filename=([A-Za-z\d+-]+)$', stem),
+
+ # ?package=name
+ # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz
+ (r'\?package=([A-Za-z\d+-]+)', stem),
+
+ # download.php
+ # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1
+ (r'([^/]+)/download.php$', path),
+
+ # 3rd Pass: Name followed by version in archive
+
+ (r'^([A-Za-z\d+\._-]+)$', stem),
]
for i, name_regex in enumerate(name_regexes):
@@ -392,13 +633,15 @@ def parse_name_offset(path, v=None):
name = match.group(1)
start = match.start(1)
- # if we matched from the basename, then add offset in.
+ # If we matched from the stem or suffix, we need to add offset
+ offset = 0
if match_string is stem:
- start += offset
-
- # package names should be lowercase and separated by dashes.
- name = name.lower()
- name = re.sub('[_.]', '-', name)
+ offset = len(path) - len(original_stem)
+ elif match_string is suffix:
+ offset = len(path)
+ if ext:
+ offset += len(ext) + 1 # .tar.gz is converted to tar.gz
+ start += offset
return name, start, len(name), i, regex
@@ -431,6 +674,9 @@ def parse_name_and_version(path):
The version of the package
:rtype: tuple
+
+ :raises UndetectableVersionError: If the URL does not match any regexes
+ :raises UndetectableNameError: If the URL does not match any regexes
"""
ver = parse_version(path)
name = parse_name(path, ver)
@@ -457,6 +703,22 @@ def cumsum(elts, init=0, fn=lambda x: x):
return sums
+def find_all(substring, string):
+ """Returns a list containing the indices of
+ every occurrence of substring in string."""
+
+ occurrences = []
+ index = 0
+ while index < len(string):
+ index = string.find(substring, index)
+ if index == -1:
+ break
+ occurrences.append(index)
+ index += len(substring)
+
+ return occurrences
+
+
def substitution_offsets(path):
"""This returns offsets for substituting versions and names in the
provided path. It is a helper for :func:`substitute_version`.
@@ -468,65 +730,34 @@ def substitution_offsets(path):
except UndetectableNameError:
return (None, -1, -1, (), ver, vs, vl, (vs,))
except UndetectableVersionError:
- return (None, -1, -1, (), None, -1, -1, ())
-
- # protect extensions like bz2 from getting inadvertently
- # considered versions.
- path = comp.strip_extension(path)
-
- # Construct a case-insensitive regular expression for the package name.
- name_re = '(%s)' % insensitize(name)
-
- # Split the string apart by things that match the name so that if the
- # name contains numbers or things that look like versions, we don't
- # accidentally substitute them with a version.
- name_parts = re.split(name_re, path)
-
- offsets = cumsum(name_parts, 0, len)
- name_offsets = offsets[1::2]
+ try:
+ name, ns, nl, ni, nregex = parse_name_offset(path)
+ return (name, ns, nl, (ns,), None, -1, -1, ())
+ except UndetectableNameError:
+ return (None, -1, -1, (), None, -1, -1, ())
- ver_offsets = []
- for i in range(0, len(name_parts), 2):
- vparts = re.split(ver, name_parts[i])
- voffsets = cumsum(vparts, offsets[i], len)
- ver_offsets.extend(voffsets[1::2])
+ # Find the index of every occurrence of name and ver in path
+ name_offsets = find_all(name, path)
+ ver_offsets = find_all(ver, path)
- return (name, ns, nl, tuple(name_offsets),
- ver, vs, vl, tuple(ver_offsets))
+ return (name, ns, nl, name_offsets,
+ ver, vs, vl, ver_offsets)
def wildcard_version(path):
"""Find the version in the supplied path, and return a regular expression
that will match this path with any version in its place.
"""
- # Get name and version, so we can treat them specially
- name, v = parse_name_and_version(path)
+ # Get version so we can replace it with a wildcard
+ version = parse_version(path)
- path, ext, suffix = split_url_extension(path)
+ # Split path by versions
+ vparts = path.split(str(version))
+
+ # Replace each version with a generic capture group to find versions
+ # and escape everything else so it's not interpreted as a regex
+ result = '(\d.*)'.join(re.escape(vp) for vp in vparts)
- # Construct a case-insensitive regular expression for the package name.
- name_re = '(%s)' % insensitize(name)
-
- # Split the string apart by things that match the name so that if the
- # name contains numbers or things that look like versions, we don't
- # catch them with the version wildcard.
- name_parts = re.split(name_re, path)
-
- # Even elements in the array did *not* match the name
- for i in range(0, len(name_parts), 2):
- # Split each part by things that look like versions.
- vparts = re.split(v.wildcard(), name_parts[i])
-
- # Replace each version with a generic capture group to find versions.
- # And escape everything else so it's not interpreted as a regex
- vgroup = '(%s)' % v.wildcard()
- name_parts[i] = vgroup.join(re.escape(vp) for vp in vparts)
-
- # Put it all back together with original name matches intact.
- result = ''.join(name_parts)
- if ext:
- result += '.' + ext
- result += suffix
return result
diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py
index 1f2bfa88cf..cd35008aed 100644
--- a/lib/spack/spack/util/naming.py
+++ b/lib/spack/spack/util/naming.py
@@ -39,6 +39,7 @@ __all__ = [
'validate_fully_qualified_module_name',
'validate_module_name',
'possible_spack_module_names',
+ 'simplify_name',
'NamespaceTrie']
# Valid module names can contain '-' but can't start with it.
@@ -108,6 +109,50 @@ def possible_spack_module_names(python_mod_name):
return results
+def simplify_name(name):
+ """Simplifies a name which may include uppercase letters, periods,
+ underscores, and pluses. In general, we want our package names to
+ only contain lowercase letters, digits, and dashes.
+
+ :param str name: The original name of the package
+ :return: The new name of the package
+ :rtype: str
+ """
+ # Convert CamelCase to Dashed-Names
+ # e.g. ImageMagick -> Image-Magick
+ # e.g. SuiteSparse -> Suite-Sparse
+ # name = re.sub('([a-z])([A-Z])', r'\1-\2', name)
+
+ # Rename Intel downloads
+ # e.g. l_daal, l_ipp, l_mkl -> daal, ipp, mkl
+ if name.startswith('l_'):
+ name = name[2:]
+
+ # Convert UPPERCASE to lowercase
+ # e.g. SAMRAI -> samrai
+ name = name.lower()
+
+ # Replace '_' and '.' with '-'
+ # e.g. backports.ssl_match_hostname -> backports-ssl-match-hostname
+ name = name.replace('_', '-')
+ name = name.replace('.', '-')
+
+ # Replace "++" with "pp" and "+" with "-plus"
+ # e.g. gtk+ -> gtk-plus
+ # e.g. voro++ -> voropp
+ name = name.replace('++', 'pp')
+ name = name.replace('+', '-plus')
+
+ # Simplify Lua package names
+ # We don't want "lua" to occur multiple times in the name
+ name = re.sub('^(lua)([^-])', r'\1-\2', name)
+
+ # Simplify Bio++ package names
+ name = re.sub('^(bpp)([^-])', r'\1-\2', name)
+
+ return name
+
+
def valid_module_name(mod_name):
"""Return whether mod_name is valid for use in Spack."""
return bool(re.match(_valid_module_re, mod_name))
diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py
index 8e2dd34635..f803c6cea3 100644
--- a/lib/spack/spack/util/web.py
+++ b/lib/spack/spack/util/web.py
@@ -268,6 +268,14 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0):
# part, not the full path.
url_regex = os.path.basename(url_regex)
+ # We need to add a / to the beginning of the regex to prevent
+ # Spack from picking up similarly named packages like:
+ # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz
+ # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz
+ # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz
+ # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz
+ url_regex = '/' + url_regex
+
# We need to add a $ anchor to the end of the regex to prevent
# Spack from picking up signature files like:
# .asc
@@ -275,7 +283,9 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0):
# .sha256
# .sig
# However, SourceForge downloads still need to end in '/download'.
- regexes.append(url_regex + '(\/download)?$')
+ url_regex += '(\/download)?$'
+
+ regexes.append(url_regex)
# Build a dict version -> URL from any links that match the wildcards.
versions = {}
diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py
index c8395aeb29..89fcc9aaa7 100644
--- a/lib/spack/spack/version.py
+++ b/lib/spack/spack/version.py
@@ -194,35 +194,6 @@ class Version(object):
nother = len(other.version)
return nother <= nself and self.version[:nother] == other.version
- def wildcard(self):
- """Create a regex that will match variants of this version string."""
- def a_or_n(seg):
- if type(seg) == int:
- return r'[0-9]+'
- else:
- return r'[a-zA-Z]+'
-
- version = self.version
-
- # Use a wildcard for separators, in case a version is written
- # two different ways (e.g., boost writes 1_55_0 and 1.55.0)
- sep_re = '[_.-]'
- separators = ('',) + (sep_re,) * len(self.separators)
-
- version += (version[-1],) * 2
- separators += (sep_re,) * 2
-
- segments = [a_or_n(seg) for seg in version]
-
- wc = segments[0]
- for i in range(1, len(separators)):
- wc += '(?:' + separators[i] + segments[i]
-
- # Add possible alpha or beta indicator at the end of each segemnt
- # We treat these specially b/c they're so common.
- wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1)
- return wc
-
def __iter__(self):
return iter(self.version)
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-16 08:25:50 +0000