From 57ef3b8a807dc79b234cb0bd5cfad1676d5e4ee4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 17 May 2013 16:25:00 -0700 Subject: Ability to list versions from web page with spack list -v PACKAGE Experimental feature automatically parses versions out of web pages and prints what it thinks are avaialble versions of a package, e.g.: $ spack list -v libunwind 1.1 1.0 0.98.6 0.98.4 0.98.2 0.98 0.96 0.93 0.91 0.1 1.0.1 0.99 0.98.5 0.98.3 0.98.1 0.97 0.95 0.92 0.9 0.0 --- lib/spack/spack/cmd/list.py | 36 +++++++++++++++++++++++++++++++++++- lib/spack/spack/colify.py | 7 +++++-- lib/spack/spack/package.py | 33 +++++++++++++++++++++++++++++---- lib/spack/spack/packages/__init__.py | 28 +++++++++++++--------------- lib/spack/spack/packages/libdwarf.py | 2 ++ lib/spack/spack/url.py | 18 ++++++++++++++---- lib/spack/spack/version.py | 32 +++++++++++++++++++++++++++++++- 7 files changed, 129 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index f4f36220c6..c6f9e4d951 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -1,10 +1,19 @@ +import os +import re + import spack import spack.packages as packages +from spack.version import ver from spack.colify import colify +import spack.url as url +import spack.tty as tty + description ="List spack packages" def setup_parser(subparser): + subparser.add_argument('-v', '--versions', metavar="PACKAGE", dest='version_package', + help='List available versions of a package (experimental).') subparser.add_argument('-i', '--installed', action='store_true', dest='installed', help='List installed packages for each platform along with versions.') @@ -16,8 +25,33 @@ def list(parser, args): print "%s:" % sys_type package_vers = [] for pkg in pkgs[sys_type]: - pv = [pkg.name + "/" + v for v in pkg.installed_versions] + pv = [pkg.name + "@" + v for v in pkg.installed_versions] package_vers.extend(pv) colify(sorted(package_vers), indent=4) + + elif args.version_package: + pkg = packages.get(args.version_package) + + try: + # Run curl but grab the mime type from the http headers + listing = spack.curl('-s', '-L', pkg.list_url, return_output=True) + url_regex = os.path.basename(url.wildcard_version(pkg.url)) + strings = re.findall(url_regex, listing) + + versions = [] + wildcard = pkg.version.wildcard() + for s in strings: + match = re.search(wildcard, s) + if match: + versions.append(ver(match.group(0))) + + colify(str(v) for v in reversed(sorted(set(versions)))) + + except: + tty.die("Listing versions for %s failed" % pkg.name, + "Listing versions is experimental. You may need to add the list_url", + "attribute to the package to tell Spack where to look for versions.") + raise + else: colify(packages.all_package_names()) diff --git a/lib/spack/spack/colify.py b/lib/spack/spack/colify.py index 46a905eb77..0016b708b6 100644 --- a/lib/spack/spack/colify.py +++ b/lib/spack/spack/colify.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# # colify # By Todd Gamblin, tgamblin@llnl.gov # @@ -100,6 +98,11 @@ def colify(elts, **options): if not type(elts) == list: elts = list(elts) + if not output.isatty(): + for elt in elts: + output.write("%s\n" % elt) + return + console_cols = options.get("cols", None) if not console_cols: console_cols, console_rows = get_terminal_size() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 81b64ccc4e..dbe54fa765 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -25,6 +25,8 @@ import validate import url import arch +from spec import Compiler +from version import Version from multi_function import platform from stage import Stage from dependency import * @@ -241,6 +243,11 @@ class Package(object): """Controls whether install and uninstall check deps before running.""" ignore_dependencies = False + # TODO: multi-compiler support + """Default compiler for this package""" + compiler = Compiler('gcc') + + def __init__(self, sys_type = arch.sys_type()): # Check for attributes that derived classes must set. attr.required(self, 'homepage') @@ -261,10 +268,14 @@ class Package(object): validate.url(self.url) # Set up version - attr.setdefault(self, 'version', url.parse_version(self.url)) - if not self.version: - tty.die("Couldn't extract version from %s. " + - "You must specify it explicitly for this URL." % self.url) + if not hasattr(self, 'version'): + try: + self.version = url.parse_version(self.url) + except UndetectableVersionError: + tty.die("Couldn't extract a default version from %s. You " + + "must specify it explicitly in the package." % self.url) + elif type(self.version) == string: + self.version = Version(self.version) # This adds a bunch of convenience commands to the package's module scope. self.add_commands_to_module() @@ -275,6 +286,10 @@ class Package(object): # stage used to build this package. self.stage = Stage(self.stage_name, self.url) + # Set a default list URL (place to find lots of versions) + if not hasattr(self, 'list_url'): + self.list_url = os.path.dirname(self.url) + def add_commands_to_module(self): """Populate the module scope of install() with some useful functions. @@ -395,6 +410,16 @@ class Package(object): return new_path(self.package_path, self.version) + def url_version(self, version): + """Given a version, this returns a string that should be substituted into the + package's URL to download that version. + By default, this just returns the version string. Subclasses may need to + override this, e.g. for boost versions where you need to ensure that there + are _'s in the download URL. + """ + return version.string + + def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" if self.dirty: diff --git a/lib/spack/spack/packages/__init__.py b/lib/spack/spack/packages/__init__.py index d74bedb3a0..16cf556621 100644 --- a/lib/spack/spack/packages/__init__.py +++ b/lib/spack/spack/packages/__init__.py @@ -6,11 +6,10 @@ import inspect import glob import spack +import spack.error from spack.utils import * import spack.arch as arch -import spack.version as version -import spack.attr as attr -import spack.error as serr + # Valid package names -- can contain - but can't start with it. valid_package = r'^\w[\w-]*$' @@ -20,7 +19,16 @@ invalid_package = r'[_-][_-]+' instances = {} -class InvalidPackageNameError(serr.SpackError): + +def get(pkg, arch=arch.sys_type()): + key = (pkg, arch) + if not key in instances: + package_class = get_class(pkg) + instances[key] = package_class(arch) + return instances[key] + + +class InvalidPackageNameError(spack.error.SpackError): """Raised when we encounter a bad package name.""" def __init__(self, name): super(InvalidPackageNameError, self).__init__( @@ -34,7 +42,7 @@ def valid_name(pkg): def validate_name(pkg): if not valid_name(pkg): - raise spack.InvalidPackageNameError(pkg) + raise InvalidPackageNameError(pkg) def filename_for(pkg): @@ -45,8 +53,6 @@ def filename_for(pkg): def installed_packages(**kwargs): """Returns a dict from systype strings to lists of Package objects.""" - list_installed = kwargs.get('installed', False) - pkgs = {} if not os.path.isdir(spack.install_path): return pkgs @@ -108,14 +114,6 @@ def get_class(pkg): return klass -def get(pkg, arch=arch.sys_type()): - key = (pkg, arch) - if not key in instances: - package_class = get_class(pkg) - instances[key] = package_class(arch) - return instances[key] - - def compute_dependents(): """Reads in all package files and sets dependence information on Package objects in memory. diff --git a/lib/spack/spack/packages/libdwarf.py b/lib/spack/spack/packages/libdwarf.py index b8a2f3e052..bae701b38b 100644 --- a/lib/spack/spack/packages/libdwarf.py +++ b/lib/spack/spack/packages/libdwarf.py @@ -9,6 +9,8 @@ class Libdwarf(Package): url = "http://reality.sgiweb.org/davea/libdwarf-20130207.tar.gz" md5 = "64b42692e947d5180e162e46c689dfbf" + list_url = "http://reality.sgiweb.org/davea/dwarf.html" + depends_on("libelf") diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index b98c470923..f1cf3e78f9 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -163,9 +163,19 @@ def parse_name_and_version(path): return (name, ver) -def version_format(path): - """Given a URL or archive name, find the version and create a format string - that will allow another version to be substituted. +def substitute_version(path, new_version): + """Given a URL or archive name, find the version in the path and substitute + the new version for it. """ ver, start, end = parse_version_string_with_indices(path) - return path[:start] + '%s' + path[end:] + return path[:start] + new_version + path[end:] + + +def wildcard_version(path): + """Find the version in the supplied path, and return a regular expression + that will match this path with any version in its place. + """ + ver, start, end = parse_version_string_with_indices(path) + v = Version(ver) + + return re.escape(path[:start]) + v.wildcard() + re.escape(path[end:]) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 1dfd3718b5..e905756c9e 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -41,6 +41,10 @@ class Version(object): segments = re.findall(segment_regex, string) self.version = tuple(int_if_int(seg) for seg in segments) + # Store the separators from the original version string as well. + # last element of separators is '' + self.separators = tuple(re.split(segment_regex, string)[1:-1]) + def up_to(self, index): """Return a version string up to the specified component, exclusive. @@ -48,6 +52,29 @@ class Version(object): """ return '.'.join(str(x) for x in self[:index]) + def wildcard(self): + """Create a regex that will match variants of this version string.""" + def a_or_n(seg): + if type(seg) == int: + return r'[0-9]+' + else: + return r'[a-zA-Z]+' + + version = self.version + separators = ('',) + self.separators + + version += (version[-1],) * 2 + separators += (separators[-1],) * 2 + + sep_res = [re.escape(sep) for sep in separators] + seg_res = [a_or_n(seg) for seg in version] + + wc = seg_res[0] + for i in xrange(1, len(sep_res)): + wc += '(?:' + sep_res[i] + seg_res[i] + wc += ')?' * (len(seg_res) - 1) + return wc + def __iter__(self): for v in self.version: yield v @@ -96,13 +123,16 @@ class Version(object): def __eq__(self, other): """Implemented to match __lt__. See __lt__.""" - if type(other) == VersionRange: + if type(other) != Version: return False return self.version == other.version def __ne__(self, other): return not (self == other) + def __hash__(self): + return hash(self.version) + @total_ordering class VersionRange(object): -- cgit v1.2.3-70-g09d2