diff options
| -rw-r--r-- | LICENSE | 48 | ||||
| -rw-r--r-- | lib/spack/spack/fetch_strategy.py | 208 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 27 | ||||
| -rw-r--r-- | lib/spack/spack/stage.py | 6 | ||||
| -rw-r--r-- | var/spack/packages/callpath/package.py | 2 |
5 files changed, 198 insertions, 93 deletions
@@ -1,4 +1,4 @@ -Copyright (c) 2013, Lawrence Livermore National Security, LLC. +Copyright (c) 2013-2014, Lawrence Livermore National Security, LLC. Produced at the Lawrence Livermore National Laboratory. This file is part of Spack. @@ -55,22 +55,22 @@ Modification 0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called “this License”). Each -licensee is addressed as “you”. +this Lesser General Public License (also called "this License"). Each +licensee is addressed as "you". -A “library” means a collection of software functions and/or data +A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs (which use some of those functions and data) to form executables. -The “Library”, below, refers to any such software library or work -which has been distributed under these terms. A “work based on the -Library” means either the Library or any derivative work under +The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term “modification”.) +included without limitation in the term "modification".) -“Source code” for a work means the preferred form of the work for +"Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control @@ -83,7 +83,7 @@ covered only if its contents constitute a work based on the Library it). Whether that is true depends on what the Library does and what the program that uses the Library does. -1. You may copy and distribute verbatim copies of the Library’s +1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact @@ -170,17 +170,17 @@ source along with the object code. 5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or -linked with it, is called a “work that uses the Library”. Such a work, +linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and therefore falls outside the scope of this License. -However, linking a “work that uses the Library” with the Library +However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a “work that uses the -library”. The executable is therefore covered by this License. Section +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. Section 6 states terms for distribution of such executables. -When a “work that uses the Library” uses material from a header file +When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be @@ -200,10 +200,10 @@ distribute the object code for the work under the terms of Section whether or not they are linked directly with the Library itself. 6. As an exception to the Sections above, you may also combine or link -a “work that uses the Library” with the Library to produce a work +a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit modification of -the work for the customer’s own use and reverse engineering for +the work for the customer's own use and reverse engineering for debugging such modifications. You must give prominent notice with each copy of the work that the @@ -218,7 +218,7 @@ a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable liked with the Library, with the -complete machine-readable “work that uses the Library”, as object code +complete machine-readable "work that uses the Library", as object code and/or source code, so that the user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood that the user who changes the contents of @@ -227,7 +227,7 @@ recompile the application to use the modified definitions.) b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a copy -of the library already present on the user’s computer system, rather +of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) will operate properly with a modified version of the library, if the user installs one, as long as the modified version is interface- compatible @@ -245,8 +245,8 @@ specified materials from the same place. e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. -For an executable, the required form of the “work that uses the -Library” must include any data and utility programs needed for +For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is normally distributed (in either source or binary form) with the major @@ -296,7 +296,7 @@ the Library or works based on it. Library), the recipient automatically receives a license from the original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further -restrictions on the recipients’ exercise of the rights granted +restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. @@ -347,7 +347,7 @@ differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and -“any later version”, you have the option of following the terms and +"any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by @@ -367,7 +367,7 @@ NO WARRANTY 1 BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER -PARTIES PROVIDE THE LIBRARY “AS IS” WITHOUT WARRANTY OF ANY KIND, +PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b0845700b6..cba0ace6d3 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -47,12 +47,27 @@ import llnl.util.tty as tty import spack import spack.error import spack.util.crypto as crypto +from spack.util.executable import * +from spack.util.string import * from spack.version import Version, ver from spack.util.compression import decompressor_for +"""List of all fetch strategies, created by FetchStrategy metaclass.""" +all_strategies = [] class FetchStrategy(object): + """Superclass of all fetch strategies.""" + enabled = False # Non-abstract subclasses should be enabled. + required_attributes = None # Attributes required in version() args. + + class __metaclass__(type): + """This metaclass registers all fetch strategies in a list.""" + def __init__(cls, name, bases, dict): + type.__init__(cls, name, bases, dict) + if cls.enabled: all_strategies.append(cls) + + def __init__(self): # The stage is initialized late, so that fetch strategies can be constructed # at package construction time. This is where things will be fetched. @@ -76,12 +91,16 @@ class FetchStrategy(object): # This method is used to match fetch strategies to version() # arguments in packages. @classmethod - def match(kwargs): - return any(k in kwargs for k in self.attributes) + def matches(cls, args): + return any(k in args for k in cls.required_attributes) class URLFetchStrategy(FetchStrategy): - attributes = ('url', 'md5') + """FetchStrategy that pulls source code from a URL for an archive, + checks the archive against a checksum,and decompresses the archive. + """ + enabled = True + required_attributes = ['url'] def __init__(self, url=None, digest=None, **kwargs): super(URLFetchStrategy, self).__init__() @@ -172,11 +191,11 @@ class URLFetchStrategy(FetchStrategy): assert(self.stage) if not self.digest: raise NoDigestError("Attempt to check URLFetchStrategy with no digest.") - checker = crypto.Checker(digest) + checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (digest, checker.sum)) + "Expected %s but got %s." % (self.digest, checker.sum)) def reset(self): @@ -191,46 +210,73 @@ class URLFetchStrategy(FetchStrategy): def __str__(self): - if self.url: - return self.url - else: - return "URLFetchStrategy <no url>" + url = self.url if self.url else "no url" + return "URLFetchStrategy<%s>" % url class VCSFetchStrategy(FetchStrategy): - def __init__(self, name): + def __init__(self, name, *rev_types, **kwargs): super(VCSFetchStrategy, self).__init__() self.name = name + # Set a URL based on the type of fetch strategy. + self.url = kwargs.get(name, None) + if not self.url: raise ValueError( + "%s requires %s argument." % (self.__class__, name)) + + # Ensure that there's only one of the rev_types + if sum((k in kwargs for k in rev_types)) > 1: + raise FetchStrategyError( + "Supply only one of %s to fetch with %s." % ( + comma_or(rev_types), name)) + + # Set attributes for each rev type. + for rt in rev_types: + setattr(self, rt, getattr(kwargs, rt, None)) + def check(self): assert(self.stage) - tty.msg("No check needed when fetching with %s." % self.name) + tty.msg("No checksum needed when fetching with %s." % self.name) + def expand(self): assert(self.stage) tty.debug("Source fetched with %s is already expanded." % self.name) + def __str__(self): + return "%s<%s>" % (self.__class__, self.url) + + class GitFetchStrategy(VCSFetchStrategy): - attributes = ('git', 'ref', 'tag', 'branch') + """Fetch strategy that gets source code from a git repository. + Use like this in a package: - def __init__(self, **kwargs): - super(GitFetchStrategy, self).__init__("git") - self.url = kwargs.get('git', None) - if not self.url: - raise ValueError("GitFetchStrategy requires git argument.") + version('name', git='https://github.com/project/repo.git') - if sum((k in kwargs for k in ('ref', 'tag', 'branch'))) > 1: - raise FetchStrategyError( - "Git requires exactly one ref, branch, or tag.") + Optionally, you can provide a branch, or commit to check out, e.g.: + + version('1.1', git='https://github.com/project/repo.git', tag='v1.1') + + You can use these three optional attributes in addition to ``git``: + + * ``branch``: Particular branch to build from (default is master) + * ``tag``: Particular tag to check out + * ``commit``: Particular commit hash in the repo + """ + enabled = True + required_attributes = ('git',) + def __init__(self, **kwargs): + super(GitFetchStrategy, self).__init__( + 'git', 'tag', 'branch', 'commit', **kwargs) self._git = None - self.ref = kwargs.get('ref', None) - self.branch = kwargs.get('branch', None) + + # For git fetch branches and tags the same way. if not self.branch: - self.branch = kwargs.get('tag', None) + self.branch = self.tag @property @@ -252,21 +298,20 @@ class GitFetchStrategy(VCSFetchStrategy): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.source_path) + tty.msg("Already fetched %s." % self.stage.source_path) return tty.msg("Trying to clone git repository: %s" % self.url) - - if self.ref: + if self.commit: # Need to do a regular clone and check out everything if - # they asked for a particular ref. - git('clone', self.url) - self.chdir_to_source() - git('checkout', self.ref) + # they asked for a particular commit. + self.git('clone', self.url) + self.stage.chdir_to_source() + self.git('checkout', self.commit) else: - # Can be more efficient if not checking out a specific ref. + # Can be more efficient if not checking out a specific commit. args = ['clone'] # If we want a particular branch ask for it. @@ -279,26 +324,77 @@ class GitFetchStrategy(VCSFetchStrategy): args.append('--single-branch') args.append(self.url) - git(*args) - self.chdir_to_source() + self.git(*args) + self.stage.chdir_to_source() def reset(self): assert(self.stage) - git = which('git', required=True) + self.stage.chdir_to_source() + self.git('checkout', '.') + self.git('clean', '-f') + +class SvnFetchStrategy(VCSFetchStrategy): + """Fetch strategy that gets source code from a subversion repository. + Use like this in a package: + + version('name', svn='http://www.example.com/svn/trunk') + + Optionally, you can provide a revision for the URL: + + version('name', svn='http://www.example.com/svn/trunk', + revision='1641') + """ + enabled = True + required_attributes = ['svn'] + + def __init__(self, **kwargs): + super(SvnFetchStrategy, self).__init__( + 'svn', 'revision', **kwargs) + self._svn = None + + + @property + def svn(self): + if not self._svn: + self._svn = which('svn', required=True) + return self._svn + + + def fetch(self): + assert(self.stage) + self.stage.chdir() + + if self.stage.source_path: + tty.msg("Already fetched %s." % self.stage.source_path) + return + + tty.msg("Trying to check out svn repository: %s" % self.url) + + args = ['checkout', '--force'] + if self.revision: + args += ['-r', self.revision] + + self.svn(*args) self.stage.chdir_to_source() - git('checkout', '.') - git('clean', '-f') - def __str__(self): - return self.url + def _remove_untracked_files(self): + """Removes untracked files in an svn repository.""" + status = self.svn('status', '--no-ignore', check_output=True) + for line in status.split('\n'): + if not re.match('^[I?]'): + continue + path = line[8:].strip() + shutil.rmtree(path, ignore_errors=True) -class SvnFetchStrategy(FetchStrategy): - attributes = ('svn', 'rev', 'revision') - pass + def reset(self): + assert(self.stage) + self.stage.chdir_to_source() + self._remove_untracked_files() + self.svn('revert', '.', '-R') def from_url(url): @@ -312,25 +408,31 @@ def from_url(url): def args_are_for(args, fetcher): - return any(arg in args for arg in fetcher.attributes) + fetcher.matches(args) def from_args(args, pkg): """Determine a fetch strategy based on the arguments supplied to version() in the package description.""" - fetchers = (URLFetchStrategy, GitFetchStrategy) - for fetcher in fetchers: - if args_are_for(args, fetcher): - attrs = {} - for attr in fetcher.attributes: - default = getattr(pkg, attr, None) - if default: - attrs[attr] = default - - attrs.update(args) + + # Test all strategies against per-version arguments. + for fetcher in all_strategies: + if fetcher.matches(args): + return fetcher(**args) + + # If nothing matched for a *specific* version, test all strategies + # against + for fetcher in all_strategies: + attrs = dict((attr, getattr(pkg, attr, None)) + for attr in fetcher.required_attributes) + attrs.update(args) + if fetcher.matches(attrs): return fetcher(**attrs) - return None + raise InvalidArgsError( + "Could not construct fetch strategy for package %s", + pkg.spec.format("%_%@")) + class FetchStrategyError(spack.error.SpackError): def __init__(self, msg, long_msg): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 553e0118e3..e9fca9ec49 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -453,19 +453,18 @@ class Package(object): raise ValueError("Can only get a stage for a concrete package.") if self._stage is None: - if not self.url: - raise PackageVersionError(self.version) - - # TODO: move this logic into a mirror module. - # TODO: get rid of dependence on extension. - mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % ( - self.name, self.version, extension(self.url))) - self._stage = Stage( - self.fetcher, mirror_path=mirror_path, name=self.spec.short_spec) + self.fetcher, mirror_path=self.mirror_path(), name=self.spec.short_spec) return self._stage + def mirror_path(self): + """Get path to this package's archive in a mirror.""" + filename = "%s-%s." % (self.name, self.version) + filename += extension(self.url) if self.has_url() else "tar.gz" + return "%s/%s" % (self.name, filename) + + def preorder_traversal(self, visited=None, **kwargs): """This does a preorder traversal of the package's dependence DAG.""" virtual = kwargs.get("virtual", False) @@ -617,9 +616,7 @@ class Package(object): self.stage.fetch() if spack.do_checksum and self.version in self.versions: - digest = self.versions[self.version].checksum - self.stage.check(digest) - tty.msg("Checksum passed for %s@%s" % (self.name, self.version)) + self.stage.check() def do_stage(self): @@ -645,8 +642,14 @@ class Package(object): if not self.spec.concrete: raise ValueError("Can only patch concrete packages.") + # Kick off the stage first. self.do_stage() + # If there are no patches, note it. + if not self.patches: + tty.msg("No patches needed for %s." % self.name) + return + # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. archive_dir = self.stage.source_path diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5dc6eac488..ed92fb17f7 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -84,14 +84,12 @@ class Stage(object): """ if isinstance(url_or_fetch_strategy, basestring): self.fetcher = fetch_strategy.from_url(url_or_fetch_strategy) - self.fetcher.set_stage(self) - elif isinstance(url_or_fetch_strategy, fetch_strategy.FetchStrategy): self.fetcher = url_or_fetch_strategy - else: raise ValueError("Can't construct Stage without url or fetch strategy") + self.fetcher.set_stage(self) self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') @@ -260,7 +258,7 @@ class Stage(object): continue - def check(self, digest): + def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" self.fetcher.check() diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py index 84170d9c9e..6102458291 100644 --- a/var/spack/packages/callpath/package.py +++ b/var/spack/packages/callpath/package.py @@ -33,6 +33,8 @@ class Callpath(Package): version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325') + depends_on("libelf") + depends_on("libdwarf") depends_on("dyninst") depends_on("adept-utils") depends_on("mpi") |