diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/llnl/util/filesystem.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/create.py | 43 | ||||
| -rw-r--r-- | lib/spack/spack/spec.py | 53 | ||||
| -rw-r--r-- | lib/spack/spack/stage.py | 112 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/test_install.py | 39 |
5 files changed, 145 insertions, 104 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6e4cd338fe..553ec1e4b5 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -189,7 +189,7 @@ def install(src, dest): def install_tree(src, dest, **kwargs): - """Manually install a file to a particular location.""" + """Manually install a directory tree to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) shutil.copytree(src, dest, **kwargs) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 2c440096d1..da74ceb2f6 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -96,8 +96,7 @@ class ${class_name}(Package): ${versions} - # FIXME: Add additional dependencies if required. - ${dependencies} +${dependencies} def install(self, spec, prefix): ${install} @@ -105,13 +104,39 @@ ${install} # Build dependencies and extensions dependencies_dict = { - 'autotools': "# depends_on('foo')", - 'cmake': "depends_on('cmake')", - 'scons': "depends_on('scons')", - 'python': "extends('python')", - 'R': "extends('R')", - 'octave': "extends('octave')", - 'unknown': "# depends_on('foo')" + 'autotools': """\ + # FIXME: Add dependencies if required. + # depends_on('foo')""", + + 'cmake': """\ + # FIXME: Add additional dependencies if required. + depends_on('cmake', type='build')""", + + 'scons': """\ + # FIXME: Add additional dependencies if required. + depends_on('scons', type='build')""", + + 'python': """\ + extends('python') + + # FIXME: Add additional dependencies if required. + # depends_on('py-foo', type=nolink)""", + + 'R': """\ + extends('R') + + # FIXME: Add additional dependencies if required. + # depends_on('r-foo', type=nolink)""", + + 'octave': """\ + extends('octave') + + # FIXME: Add additional dependencies if required. + # depends_on('octave-foo', type=nolink)""", + + 'unknown': """\ + # FIXME: Add dependencies if required. + # depends_on('foo')""" } # Default installation instructions diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 8bdae0445e..c6277fc8d2 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -95,32 +95,30 @@ thing. Spack uses ~variant in directory names and in the canonical form of specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ -import sys -import hashlib import base64 +import hashlib import imp +import sys from StringIO import StringIO from operator import attrgetter -import yaml -from yaml.error import MarkedYAMLError import llnl.util.tty as tty -from llnl.util.lang import * -from llnl.util.tty.color import * -from llnl.util.filesystem import join_path - import spack import spack.architecture -import spack.parse -import spack.error import spack.compilers as compilers - -from spack.version import * -from spack.util.string import * -from spack.util.prefix import Prefix +import spack.error +import spack.parse +import yaml +from llnl.util.filesystem import join_path +from llnl.util.lang import * +from llnl.util.tty.color import * +from spack.build_environment import get_path_from_module, load_module from spack.util.naming import mod_to_class +from spack.util.prefix import Prefix +from spack.util.string import * +from spack.version import * from spack.virtual import ProviderIndex -from spack.build_environment import get_path_from_module, load_module +from yaml.error import MarkedYAMLError # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -161,19 +159,6 @@ special_types = { } -def index_specs(specs): - """Take a list of specs and return a dict of lists. Dict is - keyed by spec name and lists include all specs with the - same name. - """ - spec_dict = {} - for spec in specs: - if spec.name not in spec_dict: - spec_dict[spec.name] = [] - spec_dict[spec.name].append(spec) - return spec_dict - - def colorize_spec(spec): """Returns a spec colorized according to the colors specified in color_formats.""" @@ -1036,7 +1021,7 @@ class Spec(object): """ try: yfile = yaml.load(stream) - except MarkedYAMLError, e: + except MarkedYAMLError as e: raise SpackYAMLError("error parsing YAML spec:", str(e)) nodes = yfile['spec'] @@ -1350,7 +1335,7 @@ class Spec(object): return flat_deps - except UnsatisfiableSpecError, e: + except UnsatisfiableSpecError as e: # Here, the DAG contains two instances of the same package # with inconsistent constraints. Users cannot produce # inconsistent specs like this on the command line: the @@ -1385,7 +1370,7 @@ class Spec(object): dep = Spec(name) try: dep.constrain(dep_spec) - except UnsatisfiableSpecError, e: + except UnsatisfiableSpecError as e: e.message = ("Conflicting conditional dependencies on" "package %s for spec %s" % (self.name, self)) raise e @@ -1471,7 +1456,7 @@ class Spec(object): try: changed |= spec_deps[dep.name].spec.constrain(dep) - except UnsatisfiableSpecError, e: + except UnsatisfiableSpecError as e: e.message = "Invalid spec: '%s'. " e.message += "Package %s requires %s %s, but spec asked for %s" e.message %= (spec_deps[dep.name].spec, dep.name, @@ -2405,7 +2390,7 @@ class SpecParser(spack.parse.Parser): # errors now? specs.append(self.spec(None, True)) - except spack.parse.ParseError, e: + except spack.parse.ParseError as e: raise SpecParseError(e) # If the spec has an os or a target and no platform, give it @@ -2850,4 +2835,4 @@ class AmbiguousHashError(SpecError): def __init__(self, msg, *specs): super(AmbiguousHashError, self).__init__(msg) for spec in specs: - print ' ', spec.format('$.$@$%@+$+$=$#') + print(' ', spec.format('$.$@$%@+$+$=$#')) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b08cce43b8..1a8b1a169a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -37,6 +37,7 @@ import spack import spack.config import spack.fetch_strategy as fs import spack.error +from spack.version import * STAGE_PREFIX = 'spack-stage-' @@ -51,10 +52,12 @@ class Stage(object): lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory + with Stage() as stage: # Context manager creates and destroys the + # stage directory stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by user of Stage) + <install> # Build and install the archive. (handled by + # user of Stage) ``` When used as a context manager, the stage is automatically @@ -71,7 +74,8 @@ class Stage(object): stage.create() # Explicitly create the stage directory. stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by user of Stage) + <install> # Build and install the archive. (handled by + # user of Stage) finally: stage.destroy() # Explicitly destroy the stage directory. ``` @@ -120,13 +124,17 @@ class Stage(object): elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: - raise ValueError("Can't construct Stage without url or fetch strategy") + raise ValueError( + "Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) - self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + # self.fetcher can change with mirrors. + self.default_fetcher = self.fetcher + # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + # TODO : this uses a protected member of tempfile, but seemed the only + # TODO : way to get a temporary name besides, the temporary link name + # TODO : won't be the same as the temporary stage area in tmp_root self.name = name if name is None: self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) @@ -143,7 +151,6 @@ class Stage(object): # Flag to decide whether to delete the stage folder on exit or not self.keep = keep - def __enter__(self): """ Entering a stage context will create the stage directory @@ -154,7 +161,6 @@ class Stage(object): self.create() return self - def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -173,12 +179,10 @@ class Stage(object): if exc_type is None and not self.keep: self.destroy() - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. - Returns False if path needs to be created. - """ + Returns False if path needs to be created.""" # Path doesn't exist yet. Will need to create it. if not os.path.exists(self.path): return True @@ -196,7 +200,8 @@ class Stage(object): if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the # right spot, then keep it. - if (real_path.startswith(real_tmp) and os.path.exists(real_path)): + if (real_path.startswith(real_tmp) and + os.path.exists(real_path)): return False else: # otherwise, just unlink it and start over. @@ -204,7 +209,8 @@ class Stage(object): return True else: - # If we're not tmp mode, then it's a link and we want a directory. + # If we're not tmp mode, then it's a link and we want a + # directory. os.unlink(self.path) return True @@ -215,10 +221,12 @@ class Stage(object): """Possible archive file paths.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) return paths @@ -227,10 +235,12 @@ class Stage(object): """Path to the source archive within this stage directory.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): @@ -262,7 +272,8 @@ class Stage(object): return None def chdir(self): - """Changes directory to the stage path. Or dies if it is not set up.""" + """Changes directory to the stage path. Or dies if it is not set + up.""" if os.path.isdir(self.path): os.chdir(self.path) else: @@ -306,6 +317,19 @@ class Stage(object): fetchers.insert(0, fs.URLFetchStrategy(url, digest)) fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) + # Look for the archive in list_url + archive_version = spack.url.parse_version(self.default_fetcher.url) + package_name = os.path.dirname(self.mirror_path) + pkg = spack.repo.get(package_name) + if pkg.list_url is not None and pkg.url is not None: + versions = pkg.fetch_remote_versions() + try: + url_from_list = versions[Version(archive_version)] + fetchers.append(fs.URLFetchStrategy(url_from_list, digest)) + except KeyError: + tty.msg("Can not find version %s in url_list" % + archive_version) + for fetcher in fetchers: try: fetcher.set_stage(self) @@ -321,11 +345,11 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + if self.fetcher is not self.default_fetcher and \ + self.skip_checksum_for_mirror: tty.warn("Fetching from mirror without a checksum!", "This package is normally checked out from a version " "control system, but it has been archived on a spack " @@ -335,16 +359,13 @@ class Stage(object): else: self.fetcher.check() - def cache_local(self): spack.cache.store(self.fetcher, self.mirror_path) - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded - archive. Fail if the stage is not set up or if the archive is not yet - downloaded. - """ + archive. Fail if the stage is not set up or if the archive is not yet + downloaded.""" archive_dir = self.source_path if not archive_dir: self.fetcher.expand() @@ -386,8 +407,8 @@ class Stage(object): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path + # If a tmp_root exists then create a directory there and then link it + # in the stage area, otherwise create the stage directory in self.path if self._need_to_create_path(): if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) @@ -409,6 +430,7 @@ class Stage(object): class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) self.root_stage = root @@ -418,12 +440,15 @@ class ResourceStage(Stage): super(ResourceStage, self).expand_archive() root_stage = self.root_stage resource = self.resource - placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + placement = os.path.basename(self.source_path) \ + if resource.placement is None \ + else resource.placement if not isinstance(placement, dict): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - target_path = join_path(root_stage.source_path, resource.destination) + target_path = join_path( + root_stage.source_path, resource.destination) destination_path = join_path(target_path, value) source_path = join_path(self.source_path, key) @@ -437,21 +462,23 @@ class ResourceStage(Stage): if not os.path.exists(destination_path): # Create a symlink - tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( - stage=source_path, destination=destination_path - )) + tty.info('Moving resource stage\n\tsource : ' + '{stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', + 'restage', 'destroy', 'cache_local']) class StageComposite: - """ - Composite for Stage type objects. The first item in this composite is considered to be the root package, and - operations that return a value are forwarded to it. - """ + """Composite for Stage type objects. The first item in this composite is + considered to be the root package, and operations that return a value are + forwarded to it.""" # # __enter__ and __exit__ delegate to all stages in the composite. # + def __enter__(self): for item in self: item.__enter__() @@ -496,8 +523,11 @@ class DIYStage(object): raise ChdirError("Setup failed: no such directory: " + self.path) # DIY stages do nothing as context managers. - def __enter__(self): pass - def __exit__(self, exc_type, exc_val, exc_tb): pass + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass def chdir_to_source(self): self.chdir() diff --git a/lib/spack/spack/test/cmd/test_install.py b/lib/spack/spack/test/cmd/test_install.py index a94d3c8bba..39287d5d6d 100644 --- a/lib/spack/spack/test/cmd/test_install.py +++ b/lib/spack/spack/test/cmd/test_install.py @@ -22,18 +22,24 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import StringIO import collections -from contextlib import contextmanager +import os +import unittest +import contextlib -import StringIO +import spack +import spack.cmd FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) + # Monkey-patch open to write module files to a StringIO instance -@contextmanager +@contextlib.contextmanager def mock_open(filename, mode): if not mode == 'wb': - raise RuntimeError('test.test_install : unexpected opening mode for monkey-patched open') + message = 'test.test_install : unexpected opening mode for mock_open' + raise RuntimeError(message) FILE_REGISTRY[filename] = StringIO.StringIO() @@ -44,19 +50,14 @@ def mock_open(filename, mode): FILE_REGISTRY[filename] = handle.getvalue() handle.close() -import os -import itertools -import unittest - -import spack -import spack.cmd - -# The use of __import__ is necessary to maintain a name with hyphen (which cannot be an identifier in python) +# The use of __import__ is necessary to maintain a name with hyphen (which +# cannot be an identifier in python) test_install = __import__("spack.cmd.test-install", fromlist=['test_install']) class MockSpec(object): + def __init__(self, name, version, hashStr=None): self._dependencies = {} self.name = name @@ -88,10 +89,6 @@ class MockSpec(object): for _, spec in self._dependencies.items(): yield spec.spec yield self - #from_iterable = itertools.chain.from_iterable - #allDeps = from_iterable(i.traverse() - # for i in self.dependencies()) - #return set(itertools.chain([self], allDeps)) def dag_hash(self): return self.hash @@ -102,6 +99,7 @@ class MockSpec(object): class MockPackage(object): + def __init__(self, spec, buildLogPath): self.name = spec.name self.spec = spec @@ -113,6 +111,7 @@ class MockPackage(object): class MockPackageDb(object): + def __init__(self, init=None): self.specToPkg = {} if init: @@ -133,6 +132,7 @@ pkgY = MockPackage(specY, 'logY') class MockArgs(object): + def __init__(self, package): self.package = package self.jobs = None @@ -168,7 +168,7 @@ class TestInstallTest(unittest.TestCase): test_install.open = mock_open # Clean FILE_REGISTRY - FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) + FILE_REGISTRY.clear() pkgX.installed = False pkgY.installed = False @@ -194,7 +194,7 @@ class TestInstallTest(unittest.TestCase): spack.repo = self.saved_db def test_installing_both(self): - test_install.test_install(None, MockArgs('X') ) + test_install.test_install(None, MockArgs('X')) self.assertEqual(len(FILE_REGISTRY), 1) for _, content in FILE_REGISTRY.items(): self.assertTrue('tests="2"' in content) @@ -210,4 +210,5 @@ class TestInstallTest(unittest.TestCase): self.assertTrue('tests="2"' in content) self.assertTrue('failures="0"' in content) self.assertTrue('errors="0"' in content) - self.assertEqual(sum('skipped' in line for line in content.split('\n')), 2) + self.assertEqual( + sum('skipped' in line for line in content.split('\n')), 2) |