Fix how environments are read from lockfile

16 files changed, 833 insertions, 109 deletions

diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index 381a6af39f..3c3d7f9c5c 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -27,7 +27,6 @@ import spack.cmd
 import spack.config as config
 import spack.database as spack_db
 import spack.fetch_strategy as fs
-import spack.hash_types as ht
 import spack.hooks
 import spack.hooks.sbang
 import spack.mirror
@@ -806,10 +805,8 @@ def generate_package_index(cache_prefix):
                 spec_file_contents = codecs.getreader('utf-8')(spec_file).read()
                 # Need full spec.json name or this gets confused with index.json.
                 if spec_url.endswith('.json'):
-                    spec_dict = sjson.load(spec_file_contents)
                     s = Spec.from_json(spec_file_contents)
                 elif spec_url.endswith('.yaml'):
-                    spec_dict = syaml.load(spec_file_contents)
                     s = Spec.from_yaml(spec_file_contents)
                 if s:
                     db.add(s, None)
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index f15fd39655..358fcb8846 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -332,18 +332,15 @@ def _report_suite_results(test_suite, args, constraints):
                 .format(results_desc, test_suite.name, matching))
 
         results = {}
-        tty.msg('test results')
         with open(test_suite.results_file, 'r') as f:
             for line in f:
                 pkg_id, status = line.split()
                 results[pkg_id] = status
-                tty.msg('  {0}'.format(pkg_id))
 
         tty.msg('test specs:')
 
         failed, skipped, untested = 0, 0, 0
         for pkg_id in test_specs:
-            tty.msg('  {0}'.format(pkg_id))
             if pkg_id in results:
                 status = results[pkg_id]
                 if status == 'FAILED':
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 90e7962b0f..35e2956fdd 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -1812,78 +1812,34 @@ class Environment(object):
 
     def _read_lockfile_dict(self, d):
         """Read a lockfile dictionary into this environment."""
+        self.specs_by_hash = {}
+
         roots = d['roots']
         self.concretized_user_specs = [Spec(r['spec']) for r in roots]
         self.concretized_order = [r['hash'] for r in roots]
-
         json_specs_by_hash = d['concrete_specs']
-        root_hashes = set(self.concretized_order)
-
-        import pdb
-        pdb.set_trace()
 
         specs_by_hash = {}
-        for dag_hash, node_dict in json_specs_by_hash.items():
-            spec = Spec.from_node_dict(node_dict)
-            if d['_meta']['lockfile-version'] > 1:
-                # Build hash is stored as a key, but not as part of the node dict
-                # To ensure build hashes are not recomputed, we reattach here
-                setattr(spec, ht.dag_hash.attr, dag_hash)
-            specs_by_hash[dag_hash] = spec
-
-        for dag_hash, node_dict in json_specs_by_hash.items():
+
+        for lockfile_key, node_dict in json_specs_by_hash.items():
+            specs_by_hash[lockfile_key] = Spec.from_node_dict(node_dict)
+
+        for lockfile_key, node_dict in json_specs_by_hash.items():
             for _, dep_hash, deptypes, _ in (
                     Spec.dependencies_from_node_dict(node_dict)):
-                specs_by_hash[dag_hash]._add_dependency(
+                specs_by_hash[lockfile_key]._add_dependency(
                     specs_by_hash[dep_hash], deptypes)
 
-        # Current lockfile key: dag_hash() (dag_hash() == full_hash())
-        # Previous lockfile keys from most recent to least:
-        #   1. build_hash
-        #   2. dag_hash (computed *without* build deps)
-
-        # If we are reading an older lockfile format, the key may have been computed
-        # using a different hash type than the one spack uses currently (which
-        # includes build deps as well as the package hash).  If this is the case
-        # the following code updates the keys in in 'concretized_order' to be computed
-        # using the hash type spack currently uses, while maintaining the order of the
-        # list.
-        old_hash_to_new = {}
-        self.specs_by_hash = {}
-        for _, spec in specs_by_hash.items():
-            # - to get old dag_hash() (w/out build deps) use runtime_hash() now
-            # - dag_hash() now includes build deps and package hash
-            #     - i.e. dag_hash() == full_hash()
-            # - regardless of what hash type keyed the lockfile we're reading,
-            #   the dag_hash we read from the file may appear appear in install
-            #   trees and binary mirrors, and as such, must be considered the
-            #   permanent id of the spec.
-            dag_hash = spec.dag_hash()          # == full_hash()
-            runtime_hash = spec.runtime_hash()  # == old dag_hash()
-
-            if dag_hash in root_hashes:
-                # This spec's dag hash (now computed with build deps and pkg
-                # hash) is in the keys found in the file, so we're looking at
-                # the current format
-                pass
-            elif runtime_hash in root_hashes:
-                # This spec's runtime hash (the old dag hash w/out build deps,
-                # etc) is a key in this lockfile, so this is the oldest format
-                old_hash_to_new[runtime_hash] = dag_hash
-            else:
-                # Neither of this spec's hashes appeared as a key in the lock
-                # file, so
-                old_hash_to_new[build_hash] = dag_hash
-
-            if (runtime_hash in root_hashes or
-                    build_hash in root_hashes or dag_hash in root_hashes):
-                self.specs_by_hash[dag_hash] = spec
-
-        if old_hash_to_new:
-            # Replace any older hashes in concretized_order with hashes
-            # that include build deps
-            self.concretized_order = [
-                old_hash_to_new.get(h, h) for h in self.concretized_order]
+        # Now make sure concretized_order and our internal specs dict
+        # contains the keys used by modern spack (i.e. the dag_hash
+        # that includes build deps and package hash).
+        self.concretized_order = [specs_by_hash[h_key].dag_hash()
+                                  for h_key in self.concretized_order]
+
+        for _, env_spec in specs_by_hash.items():
+            spec_dag_hash = env_spec.dag_hash()
+            if spec_dag_hash in self.concretized_order:
+                self.specs_by_hash[spec_dag_hash] = env_spec
 
     def write(self, regenerate=True):
         """Writes an in-memory environment to its location on disk.
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 422227675e..ac9b034d07 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -184,7 +184,7 @@ default_format += '{%compiler.name}{@compiler.version}{compiler_flags}'
 default_format += '{variants}{arch=architecture}'
 
 #: specfile format version. Must increase monotonically
-specfile_format_version = 2
+specfile_format_version = 3
 
 
 def colorize_spec(spec):
diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py
index 864493c1e4..24214d6467 100644
--- a/lib/spack/spack/test/bindist.py
+++ b/lib/spack/spack/test/bindist.py
@@ -11,15 +11,12 @@ import sys
 import py
 import pytest
 
-import llnl.util.filesystem as fs
-
 import spack.binary_distribution as bindist
 import spack.config
 import spack.hooks.sbang as sbang
 import spack.main
 import spack.mirror
 import spack.repo
-import spack.spec as spec
 import spack.store
 import spack.util.gpg
 import spack.util.web as web_util
@@ -426,14 +423,14 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir):
     # Put installed package in the buildcache
     buildcache_cmd('create', '-u', '-a', '-d', mirror_dir.strpath, s.name)
 
-    rebuild = bindist.needs_rebuild(s, mirror_url, rebuild_on_errors=True)
+    rebuild = bindist.needs_rebuild(s, mirror_url)
 
     assert not rebuild
 
-    # Now monkey patch Spec to change the full hash on the package
+    # Now monkey patch Spec to change the hash on the package
     monkeypatch.setattr(spack.spec.Spec, 'dag_hash', fake_dag_hash)
 
-    rebuild = bindist.needs_rebuild(s, mirror_url, rebuild_on_errors=True)
+    rebuild = bindist.needs_rebuild(s, mirror_url)
 
     assert rebuild
 
diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py
index ae9dbd195a..a0608fecea 100644
--- a/lib/spack/spack/test/cmd/env.py
+++ b/lib/spack/spack/test/cmd/env.py
@@ -19,6 +19,7 @@ import spack.environment as ev
 import spack.environment.shell
 import spack.hash_types as ht
 import spack.modules
+import spack.paths
 import spack.repo
 import spack.util.spack_json as sjson
 from spack.cmd.env import _env_create
@@ -28,6 +29,7 @@ from spack.stage import stage_prefix
 from spack.util.executable import Executable
 from spack.util.mock_package import MockPackageMultiRepo
 from spack.util.path import substitute_path_variables
+from spack.version import Version
 
 # TODO-27021
 # everything here uses the mock_env_path
@@ -982,7 +984,8 @@ def create_v1_lockfile_dict(roots, all_specs):
         # Version one lockfiles use the dag hash without build deps as keys,
         # but they write out the full node dict (including build deps)
         "concrete_specs": dict(
-            (s.runtime_hash(), s.to_node_dict(hash=ht.dag_hash))
+            # (s.dag_hash(), s.to_node_dict(hash=ht.dag_hash))
+            (s.runtime_hash(), s.to_node_dict(hash=ht.build_hash))
             for s in all_specs
         )
     }
@@ -1044,8 +1047,9 @@ def test_read_old_lock_creates_backup(tmpdir):
         assert os.path.exists(e._lock_backup_v1_path)
         with open(e._lock_backup_v1_path, 'r') as backup_v1_file:
             lockfile_dict_v1 = sjson.load(backup_v1_file)
+
         # Make sure that the backup file follows the v1 hash scheme
-        assert y.dag_hash() in lockfile_dict_v1['concrete_specs']
+        assert y.runtime_hash() in lockfile_dict_v1['concrete_specs']
 
 
 @pytest.mark.usefixtures('config')
@@ -2861,6 +2865,54 @@ def test_environment_query_spec_by_hash(mock_stage, mock_fetch, install_mockery)
         assert e.matching_spec('libelf').installed
 
 
+def test_read_legacy_lockfile_and_reconcretize(mock_stage, mock_fetch, install_mockery):
+    """Make sure that when we read a legacy environment lock file with a hash
+    conflict (one from before we switched to full hash), the behavior as to
+    which of the conflicting specs we pick is deterministic.  When we read
+    the lockfile, we process root specs in the order they appear in 'roots',
+    so we expect the dependencies of the last root in that list to be the
+    ones that appear in the environment before we forcefully re-concretize
+    the environment.  After we force reconcretization, we should see all
+    the dependencies again."""
+    legacy_lockfile_path = os.path.join(
+        spack.paths.test_path, 'data', 'legacy_env', 'spack.lock'
+    )
+
+    env('create', 'test', legacy_lockfile_path)
+    test = ev.read('test')
+
+    # Before we forcefully reconcretize, we expect there will be only a
+    # single actual spec in the environment, and it should depend on
+    # dtbuild1@1.0, since that root appears last in the list.
+    assert len(test.specs_by_hash) == 1
+
+    single_root = next(iter(test.specs_by_hash.values()))
+
+    assert single_root['dtbuild1'].version == Version('1.0')
+
+    # Now forcefully reconcretize
+    with ev.read('test'):
+        concretize('-f')
+
+    test = ev.read('test')
+
+    # After reconcretizing, we should again see two roots, one depending on
+    # each of the dtbuild1 versions specified in the roots of the original
+    # lockfile.
+    assert len(test.specs_by_hash) == 2
+
+    expected_dtbuild1_versions = [Version('0.5'), Version('1.0')]
+
+    for s in test.specs_by_hash.values():
+        expected_dtbuild1_versions.remove(s['dtbuild1'].version)
+
+    assert len(expected_dtbuild1_versions) == 0
+
+    expected_versions = set([Version('0.5'), Version('1.0')])
+    current_versions = set(s['dtbuild1'].version for s in test.specs_by_hash.values())
+    assert current_versions == expected_versions
+
+
 def test_environment_depfile_makefile(tmpdir, mock_packages):
     env('create', 'test')
     make = Executable('make')
diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py
index f24f40c830..0278ca344c 100644
--- a/lib/spack/spack/test/cmd/test.py
+++ b/lib/spack/spack/test/cmd/test.py
@@ -9,9 +9,12 @@ import sys
 
 import pytest
 
+from llnl.util.filesystem import copy_tree
+
 import spack.cmd.install
 import spack.config
 import spack.package
+import spack.paths
 import spack.store
 from spack.main import SpackCommand
 
@@ -242,37 +245,28 @@ def test_has_test_method_fails(capsys):
     assert 'is not a class' in captured
 
 
-def test_hash_change(mock_test_stage, mock_packages, mock_archive, mock_fetch,
-                     install_mockery_mutable_config):
-    """Ensure output printed from pkgs is captured by output redirection."""
-    install('printing-package')
-    spack_test('run', '--alias', 'printpkg', 'printing-package')
+def test_read_old_results(mock_test_stage):
+    """Take test data generated before the switch to full hash everywhere
+    and make sure we can still read it in"""
+    # Test data was generated with:
+    #   spack install printing-package
+    #   spack test run --alias printpkg printing-package
 
-    stage_files = os.listdir(mock_test_stage)
+    test_data_src = os.path.join(
+        spack.paths.test_path, 'data', 'test', 'test_stage')
 
-    # Grab test stage directory contents
-    testdir = os.path.join(mock_test_stage, stage_files[0])
+    # Copy the old test data into the mock stage directory
+    copy_tree(test_data_src, mock_test_stage)
 
-    outfile = os.path.join(testdir, 'test_suite.lock')
-    with open(outfile, 'r') as f:
-        output = f.read()
-        val_replace = '"hash": "{0}"'.format(
-            spack.store.db.query('printing-package')[0].dag_hash())
-        changed_hash = output.replace(
-            val_replace,
-            '"hash": "fakehash492ucwhwvzhxfbmcc45x49ha"')
-    with open(outfile, 'w') as f:
-        f.write(changed_hash)
-
-    # The find command should show the contents
+    # The find command should print info about the old test, under
+    # the alias used at test generation time
     find_output = spack_test('find')
     assert 'printpkg' in find_output
-    # The results should be obtainable
+
+    # The results command should still print the old test results
     results_output = spack_test('results')
     assert 'PASSED' in results_output
 
-    assert(False)
-
 
 def test_test_results_none(mock_packages, mock_test_stage):
     name = 'trivial'
diff --git a/lib/spack/spack/test/data/legacy_env/spack.lock b/lib/spack/spack/test/data/legacy_env/spack.lock
new file mode 100644
index 0000000000..794a3f263c
--- /dev/null
+++ b/lib/spack/spack/test/data/legacy_env/spack.lock
@@ -0,0 +1,672 @@
+{
+  "_meta": {
+    "file-type": "spack-lockfile",
+    "lockfile-version": 3,
+    "specfile-version": 2
+  },
+  "roots": [
+    {
+      "hash": "wci7a3aaaa4nj6ct6rj4aeltgbfojefy",
+      "spec": "dttop ^dtbuild1@0.5"
+    },
+    {
+      "hash": "5zg6wxwir2xien62soca6xaeilfzofz7",
+      "spec": "dttop ^dtbuild1@1.0"
+    }
+  ],
+  "concrete_specs": {
+    "wci7a3aaaa4nj6ct6rj4aeltgbfojefy": {
+      "name": "dttop",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtbuild1",
+          "build_hash": "qgpyperfcinui6o25aoglxzkdrlakf6b",
+          "type": [
+            "build"
+          ]
+        },
+        {
+          "name": "dtlink1",
+          "build_hash": "4skh62lxn6gra5li7sqaeunzgaxjkbns",
+          "type": [
+            "build",
+            "link"
+          ]
+        },
+        {
+          "name": "dtrun1",
+          "build_hash": "upfcexeb7zwzxdsimesyzo42yz35bw2s",
+          "type": [
+            "run"
+          ]
+        }
+      ],
+      "hash": "foya4e4rtwl5ep4mq463sdeslgaoc3qu"
+    },
+    "qgpyperfcinui6o25aoglxzkdrlakf6b": {
+      "name": "dtbuild1",
+      "version": "0.5",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtbuild2",
+          "build_hash": "pq6krl6alw7ic5ix4g5izvlzt2llbvcp",
+          "type": [
+            "build"
+          ]
+        },
+        {
+          "name": "dtlink2",
+          "build_hash": "5isttyk6zuekua2nqp23rrjpmcpo7y6a",
+          "type": [
+            "build",
+            "link"
+          ]
+        },
+        {
+          "name": "dtrun2",
+          "build_hash": "ypkyvbgxdzvverb7ami5rb6yxmp5ylmo",
+          "type": [
+            "run"
+          ]
+        }
+      ],
+      "hash": "lgcxyf3mkho6hpzymcvbetrvbujacufn"
+    },
+    "pq6krl6alw7ic5ix4g5izvlzt2llbvcp": {
+      "name": "dtbuild2",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "hash": "pq6krl6alw7ic5ix4g5izvlzt2llbvcp"
+    },
+    "5isttyk6zuekua2nqp23rrjpmcpo7y6a": {
+      "name": "dtlink2",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "hash": "5isttyk6zuekua2nqp23rrjpmcpo7y6a"
+    },
+    "ypkyvbgxdzvverb7ami5rb6yxmp5ylmo": {
+      "name": "dtrun2",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "hash": "ypkyvbgxdzvverb7ami5rb6yxmp5ylmo"
+    },
+    "4skh62lxn6gra5li7sqaeunzgaxjkbns": {
+      "name": "dtlink1",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtlink3",
+          "build_hash": "iq7m6ubgajdcnukktxolh7nc2z666h7r",
+          "type": [
+            "build",
+            "link"
+          ]
+        }
+      ],
+      "hash": "4oxug37ghalgpxyzuurftzdvlr2a7wrz"
+    },
+    "iq7m6ubgajdcnukktxolh7nc2z666h7r": {
+      "name": "dtlink3",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtbuild2",
+          "build_hash": "pq6krl6alw7ic5ix4g5izvlzt2llbvcp",
+          "type": [
+            "build"
+          ]
+        },
+        {
+          "name": "dtlink4",
+          "build_hash": "kdt2sgmlahmfyjt3rc3mdvuoh7wdyoe3",
+          "type": [
+            "build",
+            "link"
+          ]
+        }
+      ],
+      "hash": "n4j5jrvzgfnrvwwjwfycnk6n3ce2xk25"
+    },
+    "kdt2sgmlahmfyjt3rc3mdvuoh7wdyoe3": {
+      "name": "dtlink4",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "hash": "kdt2sgmlahmfyjt3rc3mdvuoh7wdyoe3"
+    },
+    "upfcexeb7zwzxdsimesyzo42yz35bw2s": {
+      "name": "dtrun1",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtlink5",
+          "build_hash": "bxpadcbd6xljttj5s5m3awlrt4zqztsh",
+          "type": [
+            "build",
+            "link"
+          ]
+        },
+        {
+          "name": "dtrun3",
+          "build_hash": "iqth4unmdwlv7zyw7joloh2lyuyvu6gb",
+          "type": [
+            "run"
+          ]
+        }
+      ],
+      "hash": "byqsjfa6hl27wpqbva5isxgbwdybgplb"
+    },
+    "bxpadcbd6xljttj5s5m3awlrt4zqztsh": {
+      "name": "dtlink5",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "hash": "bxpadcbd6xljttj5s5m3awlrt4zqztsh"
+    },
+    "iqth4unmdwlv7zyw7joloh2lyuyvu6gb": {
+      "name": "dtrun3",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtbuild3",
+          "build_hash": "t77enxfrcdrc6mumxzcdossrq4gvdliq",
+          "type": [
+            "build"
+          ]
+        }
+      ],
+      "hash": "beml5jys2cfxib6evml7ufn4wy2ak5by"
+    },
+    "t77enxfrcdrc6mumxzcdossrq4gvdliq": {
+      "name": "dtbuild3",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "hash": "t77enxfrcdrc6mumxzcdossrq4gvdliq"
+    },
+    "5zg6wxwir2xien62soca6xaeilfzofz7": {
+      "name": "dttop",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtbuild1",
+          "build_hash": "l7ikvcp4qgxtc4queb2kawhd267pylkn",
+          "type": [
+            "build"
+          ]
+        },
+        {
+          "name": "dtlink1",
+          "build_hash": "4skh62lxn6gra5li7sqaeunzgaxjkbns",
+          "type": [
+            "build",
+            "link"
+          ]
+        },
+        {
+          "name": "dtrun1",
+          "build_hash": "upfcexeb7zwzxdsimesyzo42yz35bw2s",
+          "type": [
+            "run"
+          ]
+        }
+      ],
+      "hash": "foya4e4rtwl5ep4mq463sdeslgaoc3qu"
+    },
+    "l7ikvcp4qgxtc4queb2kawhd267pylkn": {
+      "name": "dtbuild1",
+      "version": "1.0",
+      "arch": {
+        "platform": "test",
+        "platform_os": "debian6",
+        "target": {
+          "name": "core2",
+          "vendor": "GenuineIntel",
+          "features": [
+            "mmx",
+            "sse",
+            "sse2",
+            "ssse3"
+          ],
+          "generation": 0,
+          "parents": [
+            "nocona"
+          ]
+        }
+      },
+      "compiler": {
+        "name": "gcc",
+        "version": "4.5.0"
+      },
+      "namespace": "builtin.mock",
+      "parameters": {
+        "cflags": [],
+        "cppflags": [],
+        "cxxflags": [],
+        "fflags": [],
+        "ldflags": [],
+        "ldlibs": []
+      },
+      "dependencies": [
+        {
+          "name": "dtbuild2",
+          "build_hash": "pq6krl6alw7ic5ix4g5izvlzt2llbvcp",
+          "type": [
+            "build"
+          ]
+        },
+        {
+          "name": "dtlink2",
+          "build_hash": "5isttyk6zuekua2nqp23rrjpmcpo7y6a",
+          "type": [
+            "build",
+            "link"
+          ]
+        },
+        {
+          "name": "dtrun2",
+          "build_hash": "ypkyvbgxdzvverb7ami5rb6yxmp5ylmo",
+          "type": [
+            "run"
+          ]
+        }
+      ],
+      "hash": "4tldi4u3p35juizd5y5pqushwiddwmbm"
+    }
+  }
+}
\ No newline at end of file
diff --git a/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-test-out.txt b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-test-out.txt
new file mode 100755
index 0000000000..de19fa57e4
--- /dev/null
+++ b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-test-out.txt
@@ -0,0 +1,6 @@
+==> Testing package printing-package-1.0-hzgcoow
+BEFORE TEST
+==> [2022-02-28-20:21:46.510616] test: true: expect command status in [0]
+==> [2022-02-28-20:21:46.510937] '/bin/true'
+PASSED
+AFTER TEST
diff --git a/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-tested.txt b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-tested.txt
new file mode 100755
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-tested.txt
diff --git a/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/results.txt b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/results.txt
new file mode 100644
index 0000000000..135a68553e
--- /dev/null
+++ b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/results.txt
@@ -0,0 +1 @@
+printing-package-1.0-hzgcoow PASSED
diff --git a/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/test_suite.lock b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/test_suite.lock
new file mode 100644
index 0000000000..a1b76ddf00
--- /dev/null
+++ b/lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/test_suite.lock
@@ -0,0 +1,51 @@
+{
+  "specs": [
+    {
+      "spec": {
+        "_meta": {
+          "version": 2
+        },
+        "nodes": [
+          {
+            "name": "printing-package",
+            "version": "1.0",
+            "arch": {
+              "platform": "test",
+              "platform_os": "debian6",
+              "target": {
+                "name": "core2",
+                "vendor": "GenuineIntel",
+                "features": [
+                  "mmx",
+                  "sse",
+                  "sse2",
+                  "ssse3"
+                ],
+                "generation": 0,
+                "parents": [
+                  "nocona"
+                ]
+              }
+            },
+            "compiler": {
+              "name": "gcc",
+              "version": "4.5.0"
+            },
+            "namespace": "builtin.mock",
+            "parameters": {
+              "cflags": [],
+              "cppflags": [],
+              "cxxflags": [],
+              "fflags": [],
+              "ldflags": [],
+              "ldlibs": []
+            },
+            "hash": "hzgcoowzej2ftjj3v4nkdling63w2xcc",
+            "full_hash": "fakehash492ucwhwvzhxfbmcc45x49ha"
+          }
+        ]
+      }
+    }
+  ],
+  "alias": "printpkg"
+}
diff --git a/lib/spack/spack/test/env.py b/lib/spack/spack/test/env.py
index fc4cf031a8..6d16924cbd 100644
--- a/lib/spack/spack/test/env.py
+++ b/lib/spack/spack/test/env.py
@@ -27,7 +27,7 @@ def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
     # rewrite the hash
     old_hash = env.concretized_order[0]
     new_hash = 'abc'
-    env.specs_by_hash[old_hash]._build_hash = new_hash
+    env.specs_by_hash[old_hash]._hash = new_hash
     env.concretized_order[0] = new_hash
     env.specs_by_hash[new_hash] = env.specs_by_hash[old_hash]
     del env.specs_by_hash[old_hash]
@@ -39,7 +39,7 @@ def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
     # Ensure read hashes are used (rewritten hash seen on read)
     assert read_in.concretized_order
     assert read_in.concretized_order[0] in read_in.specs_by_hash
-    assert read_in.specs_by_hash[read_in.concretized_order[0]]._build_hash == new_hash
+    assert read_in.specs_by_hash[read_in.concretized_order[0]]._hash == new_hash
 
 
 def test_activate_should_require_an_env():
diff --git a/lib/spack/spack/test/test_activations.py b/lib/spack/spack/test/test_activations.py
index a832236c49..f8b3e15faa 100644
--- a/lib/spack/spack/test/test_activations.py
+++ b/lib/spack/spack/test/test_activations.py
@@ -215,9 +215,9 @@ def test_python_ignore_namespace_init_conflict(
     python_spec = spack.spec.Spec('python@2.7.12')
     python_spec._concrete = True
 
-    ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
+    ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec,
                                      monkeypatch, py_namespace)
-    ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
+    ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec,
                                      monkeypatch, py_namespace)
 
     view_dir = str(tmpdir.join('view'))
@@ -250,9 +250,9 @@ def test_python_keep_namespace_init(
     python_spec = spack.spec.Spec('python@2.7.12')
     python_spec._concrete = True
 
-    ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
+    ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec,
                                      monkeypatch, py_namespace)
-    ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
+    ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec,
                                      monkeypatch, py_namespace)
 
     view_dir = str(tmpdir.join('view'))
@@ -293,9 +293,9 @@ def test_python_namespace_conflict(tmpdir, namespace_extensions,
     python_spec = spack.spec.Spec('python@2.7.12')
     python_spec._concrete = True
 
-    ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
+    ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec,
                                      monkeypatch, py_namespace)
-    ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
+    ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec,
                                      monkeypatch, other_namespace)
 
     view_dir = str(tmpdir.join('view'))
diff --git a/lib/spack/spack/test/verification.py b/lib/spack/spack/test/verification.py
index ad7373a439..ea60dc1824 100644
--- a/lib/spack/spack/test/verification.py
+++ b/lib/spack/spack/test/verification.py
@@ -143,7 +143,7 @@ def test_check_prefix_manifest(tmpdir):
     prefix_path = tmpdir.join('prefix')
     prefix = str(prefix_path)
 
-    spec = spack.spec.Spec('libelf')
+    spec = spack.spec.Spec('libelf@0.8.13')
     spec._mark_concrete()
     spec.prefix = prefix
 
diff --git a/var/spack/repos/builtin.mock/packages/dtbuild1/package.py b/var/spack/repos/builtin.mock/packages/dtbuild1/package.py
index f4b2a469a5..36063bdd48 100644
--- a/var/spack/repos/builtin.mock/packages/dtbuild1/package.py
+++ b/var/spack/repos/builtin.mock/packages/dtbuild1/package.py
@@ -14,6 +14,7 @@ class Dtbuild1(Package):
     url = "http://www.example.com/dtbuild1-1.0.tar.gz"
 
     version('1.0', '0123456789abcdef0123456789abcdef')
+    version('0.5', 'fedcba9876543210fedcba9876543210')
 
     depends_on('dtbuild2', type='build')
     depends_on('dtlink2')