Better version substitution and wildcard URLs.

- Previously, URLs like this wouldn't work with spack create:

	http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2

The '5' in hdf5 would interfere with version wildcard substitution beacuse
the wildcard regex would subsume it.

We now take the name of the package OUT of the URL before splitting it up
and adding version wildcards.  This prevents names with numbers from breaking
url.wildcard_version.

Also added a package sanity check test that ensures all builtin packages
work with wildcard_version.

6 files changed, 156 insertions, 63 deletions

diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index 8653fafa5f..cc2e0dd384 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -123,6 +123,18 @@ def make_version_dict(ver_hash_tuples):
                            for v, h in ver_hash_tuples) + ' }'
 
 
+def get_name():
+    """Prompt user to input a package name."""
+    name = ""
+    while not name:
+        new_name = raw_input("Name: ")
+        if spack.db.valid_name(name):
+            name = new_name
+        else:
+            print "Package name can only contain A-Z, a-z, 0-9, '_' and '-'"
+    return name
+
+
 def create(parser, args):
     url = args.url
 
@@ -130,18 +142,15 @@ def create(parser, args):
     name, version = spack.url.parse_name_and_version(url)
     if not name:
         tty.msg("Couldn't guess a name for this package.")
-        while not name:
-            new_name = raw_input("Name: ")
-            if spack.db.valid_name(name):
-                name = new_name
-            else:
-                print "Package name can only contain A-Z, a-z, 0-9, '_' and '-'"
+        name = get_name()
 
     if not version:
         tty.die("Couldn't guess a version string from %s." % url)
 
+    tty.msg("This looks like a URL for %s version %s." % (name, version))
     tty.msg("Creating template for package %s" % name)
 
+    # Create a directory for the new package.
     pkg_path = spack.db.filename_for_package_name(name)
     if os.path.exists(pkg_path) and not args.force:
         tty.die("%s already exists." % pkg_path)
diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py
index 732ced9bf2..08ded5cdbb 100644
--- a/lib/spack/spack/packages.py
+++ b/lib/spack/spack/packages.py
@@ -171,7 +171,7 @@ class PackageDB(object):
 
     def all_packages(self):
         for name in self.all_package_names():
-            yield get(name)
+            yield self.get(name)
 
 
     def exists(self, pkg_name):
@@ -228,7 +228,7 @@ class PackageDB(object):
                 pkg._dependents = []
 
             for name, dep in pkg.dependencies.iteritems():
-                dpkg = get(name)
+                dpkg = self.get(name)
                 if dpkg._dependents is None:
                     dpkg._dependents = []
                 dpkg._dependents.append(pkg.name)
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index 271b915479..5aac710119 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -43,7 +43,8 @@ test_names = ['versions',
               'spec_dag',
               'concretize',
               'multimethod',
-              'install']
+              'install',
+              'package_sanity']
 
 
 def list_tests():
diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py
new file mode 100644
index 0000000000..a0bf4ceb4a
--- /dev/null
+++ b/lib/spack/spack/test/package_sanity.py
@@ -0,0 +1,40 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""\
+This test does sanity checks on Spack's builtin package database.
+"""
+import unittest
+
+import spack
+import spack.url as url
+
+class PackageSanityTest(unittest.TestCase):
+    def test_url_versions(self):
+        """Ensure that url_for_version does the right thing for at least the
+           default version of each package.
+        """
+        for pkg in spack.db.all_packages():
+            v = url.parse_version(pkg.url)
+            self.assertEqual(pkg.url, pkg.url_for_version(v))
diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py
index 109402608b..a03d6098f1 100644
--- a/lib/spack/spack/test/url_parse.py
+++ b/lib/spack/spack/test/url_parse.py
@@ -36,21 +36,33 @@ class UrlParseTest(unittest.TestCase):
         self.assertRaises(
             url.UndetectableVersionError, url.parse_name_and_version, string)
 
-    def assert_detected(self, name, v, string):
+
+    def check(self, name, v, string, **kwargs):
+        # Make sure correct name and version are extracted.
         parsed_name, parsed_v = url.parse_name_and_version(string)
         self.assertEqual(parsed_name, name)
         self.assertEqual(parsed_v, url.Version(v))
 
+        # Some URLs (like boost) are special and need to override the
+        # built-in functionality.
+        if kwargs.get('no_check_url', False):
+            return
+
+        # Make sure Spack formulates the right URL when we try to
+        # build one with a specific version.
+        self.assertEqual(string, url.substitute_version(string, v))
+
+
     def test_wwwoffle_version(self):
-        self.assert_detected(
+        self.check(
             'wwwoffle', '2.9h',
             'http://www.gedanken.demon.co.uk/download-wwwoffle/wwwoffle-2.9h.tgz')
 
     def test_version_sourceforge_download(self):
-        self.assert_detected(
+        self.check(
             'foo_bar', '1.21',
             'http://sourceforge.net/foo_bar-1.21.tar.gz/download')
-        self.assert_detected(
+        self.check(
             'foo_bar', '1.21',
             'http://sf.net/foo_bar-1.21.tar.gz/download')
 
@@ -59,216 +71,222 @@ class UrlParseTest(unittest.TestCase):
         self.assert_not_detected('foo')
 
     def test_version_all_dots(self):
-        self.assert_detected(
+        self.check(
             'foo.bar.la', '1.14','http://example.com/foo.bar.la.1.14.zip')
 
     def test_version_underscore_separator(self):
-        self.assert_detected(
+        self.check(
             'grc', '1.1',
             'http://example.com/grc_1.1.tar.gz')
 
     def test_boost_version_style(self):
-        self.assert_detected(
+        self.check(
             'boost', '1.39.0',
-            'http://example.com/boost_1_39_0.tar.bz2')
+            'http://example.com/boost_1_39_0.tar.bz2',
+            no_check_url=True)
 
     def test_erlang_version_style(self):
-        self.assert_detected(
+        self.check(
             'otp', 'R13B',
             'http://erlang.org/download/otp_src_R13B.tar.gz')
 
     def test_another_erlang_version_style(self):
-        self.assert_detected(
+        self.check(
             'otp', 'R15B01',
             'https://github.com/erlang/otp/tarball/OTP_R15B01')
 
     def test_yet_another_erlang_version_style(self):
-        self.assert_detected(
+        self.check(
             'otp', 'R15B03-1',
             'https://github.com/erlang/otp/tarball/OTP_R15B03-1')
 
     def test_p7zip_version_style(self):
-        self.assert_detected(
+        self.check(
             'p7zip', '9.04',
             'http://kent.dl.sourceforge.net/sourceforge/p7zip/p7zip_9.04_src_all.tar.bz2')
 
     def test_new_github_style(self):
-        self.assert_detected(
+        self.check(
             'libnet', '1.1.4',
             'https://github.com/sam-github/libnet/tarball/libnet-1.1.4')
 
     def test_gloox_beta_style(self):
-        self.assert_detected(
+        self.check(
             'gloox', '1.0-beta7',
             'http://camaya.net/download/gloox-1.0-beta7.tar.bz2')
 
     def test_sphinx_beta_style(self):
-        self.assert_detected(
+        self.check(
             'sphinx', '1.10-beta',
             'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz')
 
     def test_astyle_verson_style(self):
-        self.assert_detected(
+        self.check(
             'astyle', '1.23',
             'http://kent.dl.sourceforge.net/sourceforge/astyle/astyle_1.23_macosx.tar.gz')
 
     def test_version_dos2unix(self):
-        self.assert_detected(
+        self.check(
             'dos2unix', '3.1',
             'http://www.sfr-fresh.com/linux/misc/dos2unix-3.1.tar.gz')
 
     def test_version_internal_dash(self):
-        self.assert_detected(
+        self.check(
             'foo-arse', '1.1-2',
             'http://example.com/foo-arse-1.1-2.tar.gz')
 
     def test_version_single_digit(self):
-        self.assert_detected(
+        self.check(
             'foo_bar', '45',
             'http://example.com/foo_bar.45.tar.gz')
 
     def test_noseparator_single_digit(self):
-        self.assert_detected(
+        self.check(
             'foo_bar', '45',
             'http://example.com/foo_bar45.tar.gz')
 
     def test_version_developer_that_hates_us_format(self):
-        self.assert_detected(
+        self.check(
             'foo-bar-la', '1.2.3',
             'http://example.com/foo-bar-la.1.2.3.tar.gz')
 
     def test_version_regular(self):
-        self.assert_detected(
+        self.check(
             'foo_bar', '1.21',
             'http://example.com/foo_bar-1.21.tar.gz')
 
     def test_version_github(self):
-        self.assert_detected(
+        self.check(
             'yajl', '1.0.5',
             'http://github.com/lloyd/yajl/tarball/1.0.5')
 
     def test_version_github_with_high_patch_number(self):
-        self.assert_detected(
+        self.check(
             'yajl', '1.2.34',
             'http://github.com/lloyd/yajl/tarball/v1.2.34')
 
     def test_yet_another_version(self):
-        self.assert_detected(
+        self.check(
             'mad', '0.15.1b',
             'http://example.com/mad-0.15.1b.tar.gz')
 
     def test_lame_version_style(self):
-        self.assert_detected(
+        self.check(
             'lame', '398-2',
             'http://kent.dl.sourceforge.net/sourceforge/lame/lame-398-2.tar.gz')
 
     def test_ruby_version_style(self):
-        self.assert_detected(
+        self.check(
             'ruby', '1.9.1-p243',
             'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz')
 
     def test_omega_version_style(self):
-        self.assert_detected(
+        self.check(
             'omega', '0.80.2',
             'http://www.alcyone.com/binaries/omega/omega-0.80.2-src.tar.gz')
 
     def test_rc_style(self):
-        self.assert_detected(
+        self.check(
             'libvorbis', '1.2.2rc1',
             'http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2')
 
     def test_dash_rc_style(self):
-        self.assert_detected(
+        self.check(
             'js', '1.8.0-rc1',
             'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz')
 
     def test_angband_version_style(self):
-        self.assert_detected(
+        self.check(
             'angband', '3.0.9b',
             'http://rephial.org/downloads/3.0/angband-3.0.9b-src.tar.gz')
 
     def test_stable_suffix(self):
-        self.assert_detected(
+        self.check(
             'libevent', '1.4.14b',
             'http://www.monkey.org/~provos/libevent-1.4.14b-stable.tar.gz')
 
     def test_debian_style_1(self):
-        self.assert_detected(
+        self.check(
             'sl', '3.03',
             'http://ftp.de.debian.org/debian/pool/main/s/sl/sl_3.03.orig.tar.gz')
 
     def test_debian_style_2(self):
-        self.assert_detected(
+        self.check(
             'mmv', '1.01b',
             'http://ftp.de.debian.org/debian/pool/main/m/mmv/mmv_1.01b.orig.tar.gz')
 
     def test_imagemagick_style(self):
-        self.assert_detected(
+        self.check(
             'ImageMagick', '6.7.5-7',
 
             'http://downloads.sf.net/project/machomebrew/mirror/ImageMagick-6.7.5-7.tar.bz2')
 
     def test_dash_version_dash_style(self):
-        self.assert_detected(
+        self.check(
             'antlr', '3.4',
             'http://www.antlr.org/download/antlr-3.4-complete.jar')
 
     def test_apache_version_style(self):
-        self.assert_detected(
+        self.check(
             'apache-cassandra', '1.2.0-rc2',
             'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz')
 
     def test_jpeg_style(self):
-        self.assert_detected(
+        self.check(
             'jpegsrc', '8d',
             'http://www.ijg.org/files/jpegsrc.v8d.tar.gz')
 
     def test_pypy_version(self):
-        self.assert_detected(
+        self.check(
             'pypy', '1.4.1',
             'http://pypy.org/download/pypy-1.4.1-osx.tar.bz2')
 
     def test_openssl_version(self):
-        self.assert_detected(
+        self.check(
             'openssl', '0.9.8s',
             'http://www.openssl.org/source/openssl-0.9.8s.tar.gz')
 
     def test_xaw3d_version(self):
-        self.assert_detected(
+        self.check(
             'Xaw3d', '1.5E',
             'ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz')
 
     def test_fann_version(self):
-        self.assert_detected(
+        self.check(
             'fann', '2.1.0beta',
             'http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip')
 
     def test_iges_version(self):
-        self.assert_detected(
+        self.check(
             'grads', '2.0.1',
             'ftp://iges.org/grads/2.0/grads-2.0.1-bin-darwin9.8-intel.tar.gz')
 
     def test_haxe_version(self):
-        self.assert_detected(
+        self.check(
             'haxe', '2.08',
             'http://haxe.org/file/haxe-2.08-osx.tar.gz')
 
     def test_imap_version(self):
-        self.assert_detected(
+        self.check(
             'imap', '2007f',
             'ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz')
 
     def test_suite3270_version(self):
-        self.assert_detected(
+        self.check(
             'suite3270', '3.3.12ga7',
             'http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz')
 
     def test_synergy_version(self):
-        self.assert_detected(
+        self.check(
             'synergy', '1.3.6p2',
             'http://synergy.googlecode.com/files/synergy-1.3.6p2-MacOSX-Universal.zip')
 
     def test_mvapich2_version(self):
-        self.assert_detected(
+        self.check(
             'mvapich2', '1.9',
             'http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz')
+
+    def test_hdf5_version(self):
+        self.check(
+            'hdf5', '1.8.13',
+            'http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2')
diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py
index deac156571..1b8120168f 100644
--- a/lib/spack/spack/url.py
+++ b/lib/spack/spack/url.py
@@ -191,6 +191,16 @@ def parse_name_and_version(path):
     return (name, ver)
 
 
+def insensitize(string):
+    """Chagne upper and lowercase letters to be case insensitive in
+       the provided string.  e.g., 'a' because '[Aa]', 'B' becomes
+       '[bB]', etc.  Use for building regexes."""
+    def to_ins(match):
+        char = match.group(1)
+        return '[%s%s]' % (char.lower(), char.upper())
+    return re.sub(r'([a-zA-Z])', to_ins, string)
+
+
 def substitute_version(path, new_version):
     """Given a URL or archive name, find the version in the path and substitute
        the new version for it.
@@ -203,11 +213,26 @@ def wildcard_version(path):
     """Find the version in the supplied path, and return a regular expression
        that will match this path with any version in its place.
     """
-    ver, start, end = parse_version_string_with_indices(path)
+    # Get name and version, so we can treat them specially
+    name, v = parse_name_and_version(path)
+
+    # Construct a case-insensitive regular expression for the package name.
+    name_re = '(%s)' % insensitize(name)
+
+    # Split the string apart by things that match the name so that if the
+    # name contains numbers or things that look like versions, we don't
+    # catch them with the version wildcard.
+    name_parts = re.split(name_re, path)
+
+    # Even elements in the array did *not* match the name
+    for i in xrange(0, len(name_parts), 2):
+        # Split each part by things that look like versions.
+        vparts = re.split(v.wildcard(), name_parts[i])
 
-    v = Version(ver)
-    parts = [re.escape(p) for p in re.split(v.wildcard(), path)]
+        # Replace each version with a generic capture group to find versions.
+        # And escape everything else so it's not interpreted as a regex
+        vgroup = '(%s)' % v.wildcard()
+        name_parts[i] = vgroup.join(re.escape(vp) for vp in vparts)
 
-    # Make a group for the wildcard, so it will be captured by the regex.
-    version_group = '(%s)' % v.wildcard()
-    return version_group.join(parts)
+    # Put it all back together with original name matches intact.
+    return ''.join(name_parts)