diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/spack/cmd/spec.py | 3 | ||||
-rw-r--r-- | lib/spack/spack/cmd/test.py | 12 | ||||
-rw-r--r-- | lib/spack/spack/concretize.py | 10 | ||||
-rw-r--r-- | lib/spack/spack/package.py | 12 | ||||
-rw-r--r-- | lib/spack/spack/spec.py | 133 | ||||
-rw-r--r-- | lib/spack/spack/test/concretize.py | 61 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/__init__.py | 1 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/callpath.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/dyninst.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/libdwarf.py | 1 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/libelf.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/mpich.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages/mpileaks.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/mock_packages_test.py | 55 | ||||
-rw-r--r-- | lib/spack/spack/test/spec_dag.py | 66 | ||||
-rw-r--r-- | lib/spack/spack/tty.py | 6 | ||||
-rw-r--r-- | lib/spack/spack/util/lang.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/version.py | 15 |
18 files changed, 268 insertions, 119 deletions
diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index f87e94dae6..36bfc5e154 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -14,3 +14,6 @@ def spec(parser, args): for spec in specs: spec.normalize() print spec.tree() + + spec.concretize() + print spec.tree() diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index c9752fbbac..b1c8df4eb3 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -17,13 +17,21 @@ def setup_parser(subparser): help="verbose output") +def find_test_modules(): + """Include all the modules under test, unless they set skip_test=True""" + for name in list_modules(spack.test_path): + module = __import__('spack.test.' + name, fromlist='skip_test') + if not getattr(module, 'skip_test', False): + yield name + + def test(parser, args): if args.list: print "Available tests:" - colify(list_modules(spack.test_path, directories=False)) + colify(find_test_modules()) elif not args.names: - for name in list_modules(spack.test_path, directories=False): + for name in find_test_modules(): print "Running Tests: %s" % name spack.test.run(name, verbose=args.verbose) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index ac1d4421da..6acae39039 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -24,12 +24,12 @@ def concretize_version(spec): if spec.versions.concrete: return - pkg = speck.package - available = pkg.available_versions + pkg = spec.package # If there are known avaialble versions, return the most recent - if versions: - spec.versions = ver([avaialble[-1]]) + available_versions = pkg.available_versions + if available_versions: + spec.versions = ver([available_versions[-1]]) else: spec.versions = ver([pkg.version]) @@ -73,7 +73,7 @@ def concretize_compiler(spec): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ - if spec.compiler.concrete: + if spec.compiler and spec.compiler.concrete: if spec.compiler != spack.compilers.default_compiler(): raise spack.spec.UnknownCompilerError(str(spec.compiler)) else: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3658277634..d61ab4620d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -298,6 +298,11 @@ class Package(object): # This is set by scraping a web page. self._available_versions = None + # This list overrides available_versions if set by the user. + attr.setdefault(self, 'versions', None) + if self.versions and type(self.versions) != VersionList: + self.versions = VersionList(self.versions) + # stage used to build this package. self.stage = Stage("%s-%s" % (self.name, self.version), self.url) @@ -637,6 +642,11 @@ class Package(object): @property def available_versions(self): + # If the package overrode available_versions, then use that. + if self.versions is not None: + return self.versions + + # If not, then try to fetch using list_url if not self._available_versions: self._available_versions = ver([self.version]) try: @@ -656,7 +666,7 @@ class Package(object): "to the package to tell Spack where to look for versions.") except subprocess.CalledProcessError: - tty.warn("Fetching %s failed." % self.list_url, + tty.warn("Could not connect to %s" % self.list_url, "Package.available_versions requires an internet connection.", "Version list may be incomplete.") diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f7f4299942..e4203e85fb 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -170,7 +170,7 @@ class Compiler(object): def __str__(self): out = self.name if self.versions: - vlist = ",".join(str(v) for v in sorted(self.versions)) + vlist = ",".join(str(v) for v in self.versions) out += "@%s" % vlist return out @@ -190,6 +190,10 @@ class Variant(object): return (self.name, self.enabled) + def copy(self): + return Variant(self.name, self.enabled) + + def __str__(self): out = '+' if self.enabled else '~' return out + self.name @@ -350,43 +354,58 @@ class Spec(object): yield spec - def _concretize(self): + def _concretize_helper(self, presets): + for name in sorted(self.dependencies.keys()): + self.dependencies[name]._concretize_helper(presets) + + if self.name in presets: + self.constrain(presets[self.name]) + else: + spack.concretize.concretize_architecture(self) + spack.concretize.concretize_compiler(self) + spack.concretize.concretize_version(self) + presets[self.name] = self + + + def concretize(self, *presets): """A spec is concrete if it describes one build of a package uniquely. This will ensure that this spec is concrete. If this spec could describe more than one version, variant, or build - of a package, this will resolve it to be concrete. - - Ensures that the spec is in canonical form. + of a package, this will add constraints to make it concrete. - This means: - 1. All dependencies of this package and of its dependencies are - in the dependencies list (transitive closure of deps). - 2. All dependencies in the dependencies list are canonicalized. - - This function also serves to validate the spec, in that it makes sure - that each package exists an that spec criteria don't violate package - criteria. + Some rigorous validation and checks are also performed on the spec. + Concretizing ensures that it is self-consistent and that it's consistent + with requirements of its pacakges. See flatten() and normalize() for + more details on this. """ - # TODO: modularize the process of selecting concrete versions. - # There should be a set of user-configurable policies for these decisions. - self.validate() - - # take the system's architecture for starters - if not self.architecture: - self.architecture = arch.sys_type() - - if self.compiler: - self.compiler._concretize() - - # TODO: handle variants. - - # Take the highest version in a range - if not self.versions.concrete: - preferred = self.versions.highest() or self.package.version - self.versions = VersionList([preferred]) - - # Ensure dependencies have right versions + # Build specs out of user-provided presets + specs = [Spec(p) for p in presets] + + # Concretize the presets first. They could be partial specs, like just + # a particular version that the caller wants. + for spec in specs: + if not spec.concrete: + try: + spec.concretize() + except UnsatisfiableSpecError, e: + e.message = ("Unsatisfiable preset in concretize: %s." + % e.message) + raise e + + # build preset specs into a map + preset_dict = {spec.name : spec for spec in specs} + + # Concretize bottom up, passing in presets to force concretization + # for certain specs. + self.normalize() + self._concretize_helper(preset_dict) + + + def concretized(self, *presets): + clone = self.copy() + clone.concretize(*presets) + return clone def flat_dependencies(self): @@ -406,7 +425,9 @@ class Spec(object): try: for spec in self.preorder_traversal(): if spec.name not in flat_deps: - flat_deps[spec.name] = spec + new_spec = spec.copy(dependencies=False) + flat_deps[spec.name] = new_spec + else: flat_deps[spec.name].constrain(spec) @@ -466,7 +487,7 @@ class Spec(object): # provided spec is sane, and that all dependency specs are in the # root node of the spec. flat_dependencies will do this for us. spec_deps = self.flat_dependencies() - self.dependencies = DependencyMap() + self.dependencies.clear() visited = set() self._normalize_helper(visited, spec_deps) @@ -523,30 +544,37 @@ class Spec(object): self.dependencies.satisfies(other.dependencies)) - def concretized(self): - clone = self.copy() - clone._concretize() - return clone - - - def _dup(self, other): + def _dup(self, other, **kwargs): """Copy the spec other into self. This is a - first-party, overwriting copy.""" + first-party, overwriting copy. This does not copy + parent; if the other spec has a parent, this one will not. + To duplicate an entire DAG, Duplicate the root of the DAG. + """ # TODO: this needs to handle DAGs. self.name = other.name + self.parent = None self.versions = other.versions.copy() self.variants = other.variants.copy() self.architecture = other.architecture self.compiler = None if other.compiler: self.compiler = other.compiler.copy() - self.dependencies = other.dependencies.copy() + copy_deps = kwargs.get('dependencies', True) + if copy_deps: + self.dependencies = other.dependencies.copy() + else: + self.dependencies = DependencyMap() - def copy(self): - """Return a deep copy of this spec.""" - return Spec(self) + def copy(self, **kwargs): + """Return a copy of this spec. + By default, returns a deep copy. Supply dependencies=False + to get a shallow copy. + """ + clone = Spec.__new__(Spec) + clone._dup(self, **kwargs) + return clone @property def version(self): @@ -564,7 +592,7 @@ class Spec(object): return colorize_spec(self) - def str_no_deps(self): + def str_no_deps(self, **kwargs): out = self.name # If the version range is entirely open, omit it @@ -579,12 +607,17 @@ class Spec(object): if self.architecture: out += "=%s" % self.architecture - return out + if kwargs.get('color', False): + return colorize_spec(out) + else: + return out - def tree(self): + def tree(self, **kwargs): """Prints out this spec and its dependencies, tree-formatted with indentation.""" + color = kwargs.get('color', False) + out = "" cur_id = 0 ids = {} @@ -594,7 +627,7 @@ class Spec(object): ids[id(node)] = cur_id out += str(ids[id(node)]) out += " "+ (" " * d) - out += node.str_no_deps() + "\n" + out += node.str_no_deps(color=color) + "\n" return out diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index f010d09c0e..c95db03609 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -1,17 +1,60 @@ import unittest from spack.spec import Spec +from spack.test.mock_packages_test import * +class ConcretizeTest(MockPackagesTest): -class ConcretizeTest(unittest.TestCase): + def check_spec(self, abstract, concrete): + if abstract.versions.concrete: + self.assertEqual(abstract.versions, concrete.versions) - def check_concretize(self, abstract_spec): + if abstract.variants: + self.assertEqual(abstract.versions, concrete.versions) + + if abstract.compiler and abstract.compiler.concrete: + self.assertEqual(abstract.compiler, concrete.compiler) + + if abstract.architecture and abstract.architecture.concrete: + self.assertEqual(abstract.architecture, concrete.architecture) + + + def check_concretize(self, abstract_spec, *presets): abstract = Spec(abstract_spec) - print abstract - print abstract.concretized() - print abstract.concretized().concrete - self.assertTrue(abstract.concretized().concrete) + concrete = abstract.concretized(*presets) + + self.assertFalse(abstract.concrete) + self.assertTrue(concrete.concrete) + self.check_spec(abstract, concrete) + + return concrete + + + def check_presets(self, abstract, *presets): + abstract = Spec(abstract) + concrete = self.check_concretize(abstract, *presets) + + flat_deps = concrete.flat_dependencies() + for preset in presets: + preset_spec = Spec(preset) + name = preset_spec.name + + self.assertTrue(name in flat_deps) + self.check_spec(preset_spec, flat_deps[name]) + + return concrete + + + def test_concretize_no_deps(self): + self.check_concretize('libelf') + self.check_concretize('libelf@0.8.13') + + + def test_concretize_dag(self): + self.check_concretize('mpileaks') + self.check_concretize('callpath') - def test_packages(self): - pass - #self.check_concretize("libelf") + def test_concretize_with_presets(self): + self.check_presets('mpileaks', 'callpath@0.8') + self.check_presets('mpileaks', 'callpath@0.9', 'dyninst@8.0+debug') + self.check_concretize('callpath', 'libelf@0.8.13+debug~foo', 'mpich@1.0') diff --git a/lib/spack/spack/test/mock_packages/__init__.py b/lib/spack/spack/test/mock_packages/__init__.py index e69de29bb2..e6a8f467e6 100644 --- a/lib/spack/spack/test/mock_packages/__init__.py +++ b/lib/spack/spack/test/mock_packages/__init__.py @@ -0,0 +1 @@ +skip_test = True diff --git a/lib/spack/spack/test/mock_packages/callpath.py b/lib/spack/spack/test/mock_packages/callpath.py index 958960e0ab..4ca1a57007 100644 --- a/lib/spack/spack/test/mock_packages/callpath.py +++ b/lib/spack/spack/test/mock_packages/callpath.py @@ -5,6 +5,8 @@ class Callpath(Package): url = "http://github.com/tgamblin/callpath-0.2.tar.gz" md5 = "foobarbaz" + versions = [0.8, 0.9, 1.0] + depends_on("dyninst") depends_on("mpich") diff --git a/lib/spack/spack/test/mock_packages/dyninst.py b/lib/spack/spack/test/mock_packages/dyninst.py index f550cde54f..99bbd1507b 100644 --- a/lib/spack/spack/test/mock_packages/dyninst.py +++ b/lib/spack/spack/test/mock_packages/dyninst.py @@ -5,6 +5,8 @@ class Dyninst(Package): url = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz" md5 = "bf03b33375afa66fe0efa46ce3f4b17a" + versions = '7.0, 7.0.1, 8.0, 8.1.1, 8.1.2' + depends_on("libelf") depends_on("libdwarf") diff --git a/lib/spack/spack/test/mock_packages/libdwarf.py b/lib/spack/spack/test/mock_packages/libdwarf.py index bae701b38b..a096eef0b8 100644 --- a/lib/spack/spack/test/mock_packages/libdwarf.py +++ b/lib/spack/spack/test/mock_packages/libdwarf.py @@ -10,6 +10,7 @@ class Libdwarf(Package): md5 = "64b42692e947d5180e162e46c689dfbf" list_url = "http://reality.sgiweb.org/davea/dwarf.html" + versions = '20111030, 20120410, 20130207' depends_on("libelf") diff --git a/lib/spack/spack/test/mock_packages/libelf.py b/lib/spack/spack/test/mock_packages/libelf.py index 7e3046b174..efc3ebc98b 100644 --- a/lib/spack/spack/test/mock_packages/libelf.py +++ b/lib/spack/spack/test/mock_packages/libelf.py @@ -5,6 +5,8 @@ class Libelf(Package): url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" md5 = "4136d7b4c04df68b686570afa26988ac" + versions = '0.8.10, 0.8.12, 0.8.13' + def install(self, prefix): configure("--prefix=%s" % prefix, "--enable-shared", diff --git a/lib/spack/spack/test/mock_packages/mpich.py b/lib/spack/spack/test/mock_packages/mpich.py index d8cd67d528..ab235b1e43 100644 --- a/lib/spack/spack/test/mock_packages/mpich.py +++ b/lib/spack/spack/test/mock_packages/mpich.py @@ -5,6 +5,8 @@ class Mpich(Package): url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" md5 = "9c5d5d4fe1e17dd12153f40bc5b6dbc0" + versions = '1.0.3, 1.3.2p1, 1.4.1p1, 3.0.4, 3.1b1' + def install(self, prefix): configure("--prefix=%s" % prefix) make() diff --git a/lib/spack/spack/test/mock_packages/mpileaks.py b/lib/spack/spack/test/mock_packages/mpileaks.py index 224557cc52..d006ff61ed 100644 --- a/lib/spack/spack/test/mock_packages/mpileaks.py +++ b/lib/spack/spack/test/mock_packages/mpileaks.py @@ -5,6 +5,8 @@ class Mpileaks(Package): url = "http://www.llnl.gov/mpileaks-1.0.tar.gz" md5 = "foobarbaz" + versions = [1.0, 2.1, 2.2, 2.3] + depends_on("mpich") depends_on("callpath") diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py new file mode 100644 index 0000000000..d58e1d1124 --- /dev/null +++ b/lib/spack/spack/test/mock_packages_test.py @@ -0,0 +1,55 @@ +import unittest + +import spack +import spack.packages as packages +from spack.spec import Spec +from spack.util.lang import new_path, list_modules + +mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages') +original_deps = None + + +def set_pkg_dep(pkg, spec): + """Alters dependence information for a pacakge. + Use this to mock up constraints. + """ + spec = Spec(spec) + packages.get(pkg).dependencies[spec.name] = spec + + +def restore_dependencies(): + # each time through restore original dependencies & constraints + global original_deps + for pkg_name, deps in original_deps.iteritems(): + packages.get(pkg_name).dependencies.clear() + for dep in deps: + set_pkg_dep(pkg_name, dep) + + +class MockPackagesTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Use a different packages directory for these tests. We want to use + # mocked up packages that don't interfere with the real ones. + cls.real_packages_path = spack.packages_path + spack.packages_path = mock_packages_path + + # First time through, record original relationships bt/w packages + global original_deps + original_deps = {} + for name in list_modules(mock_packages_path): + pkg = packages.get(name) + original_deps[name] = [ + spec for spec in pkg.dependencies.values()] + + + @classmethod + def tearDownClass(cls): + """Restore the real packages path after any test.""" + restore_dependencies() + spack.packages_path = cls.real_packages_path + + + def setUp(self): + """Before each test, restore deps between packages to original state.""" + restore_dependencies() diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index bdb64fee80..452884e539 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -6,61 +6,18 @@ packages directories that these tests use in: Each test validates conditions with the packages in those directories. """ -import unittest - import spack import spack.package import spack.packages as packages from spack.util.lang import new_path, list_modules from spack.spec import Spec +from spack.test.mock_packages_test import * mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages') -def set_pkg_dep(pkg, spec): - """Alters dependence information for a pacakge. - Use this to mock up constraints. - """ - spec = Spec(spec) - packages.get(pkg).dependencies[spec.name] = spec - - -class ValidationTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - # Use a different packages directory for these tests. We want to use - # mocked up packages that don't interfere with the real ones. - cls.real_packages_path = spack.packages_path - spack.packages_path = mock_packages_path - - # First time through, record original relationships bt/w packages - cls.original_deps = {} - for name in list_modules(mock_packages_path): - pkg = packages.get(name) - cls.original_deps[name] = [ - spec for spec in pkg.dependencies.values()] - - - @classmethod - def restore(cls): - # each time through restore original dependencies & constraints - for pkg_name, deps in cls.original_deps.iteritems(): - packages.get(pkg_name).dependencies.clear() - for dep in deps: - set_pkg_dep(pkg_name, dep) - - @classmethod - def tearDownClass(cls): - """Restore the real packages path after any test.""" - cls.restore() - spack.packages_path = cls.real_packages_path - - - def setUp(self): - """Before each test, restore deps between packages to original state.""" - ValidationTest.restore() - +class ValidationTest(MockPackagesTest): def test_conflicting_package_constraints(self): set_pkg_dep('mpileaks', 'mpich@1.0') @@ -87,6 +44,25 @@ class ValidationTest(unittest.TestCase): self.assertRaises(spack.spec.InconsistentSpecError, mpileaks.flatten) + def test_normalize_twice(self): + """Make sure normalize can be run twice on the same spec, + and that it is idempotent.""" + spec = Spec('mpileaks') + spec.normalize() + n1 = spec.copy() + + spec.normalize() + self.assertEqual(n1, spec) + + + def test_normalize_a_lot(self): + spec = Spec('mpileaks') + spec.normalize() + spec.normalize() + spec.normalize() + spec.normalize() + + def test_unsatisfiable_version(self): set_pkg_dep('mpileaks', 'mpich@1.0') spec = Spec('mpileaks ^mpich@2.0 ^callpath ^dyninst ^libelf ^libdwarf') diff --git a/lib/spack/spack/tty.py b/lib/spack/spack/tty.py index d6f64f7203..4cad0914e6 100644 --- a/lib/spack/spack/tty.py +++ b/lib/spack/spack/tty.py @@ -24,15 +24,15 @@ def verbose(message, *args): def debug(*args): if spack.debug: - info(message, *args, format='*c') + info("Debug: " + message, *args, format='*c') def error(message, *args): - info(message, *args, format='*r') + info("Error: " + message, *args, format='*r') def warn(message, *args): - info(message, *args, format='*Y') + info("Warning: " + message, *args, format='*Y') def die(message, *args): diff --git a/lib/spack/spack/util/lang.py b/lib/spack/spack/util/lang.py index 90546c3f2f..d983141045 100644 --- a/lib/spack/spack/util/lang.py +++ b/lib/spack/spack/util/lang.py @@ -97,5 +97,5 @@ class HashableMap(dict): # Copy everything from this dict into it. for key in self: - clone[key] = self[key] + clone[key] = self[key].copy() return clone diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 15b4027606..0be19e96ed 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -347,9 +347,16 @@ class VersionList(object): def __init__(self, vlist=None): self.versions = [] if vlist != None: - vlist = list(vlist) - for v in vlist: - self.add(ver(v)) + if type(vlist) == str: + vlist = _string_to_version(vlist) + if type(vlist) == VersionList: + self.versions = vlist.versions + else: + self.versions = [vlist] + else: + vlist = list(vlist) + for v in vlist: + self.add(ver(v)) def add(self, version): @@ -540,6 +547,8 @@ def ver(obj): return VersionList(obj) elif t == str: return _string_to_version(obj) + elif t in (int, float): + return _string_to_version(str(obj)) elif t in (Version, VersionRange, VersionList): return obj else: |