diff options
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 5 | ||||
| -rw-r--r-- | lib/spack/spack/database.py | 25 | ||||
| -rw-r--r-- | lib/spack/spack/test/bindist.py | 40 |
3 files changed, 66 insertions, 4 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 1ac89b615c..ba4921b677 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -156,7 +156,7 @@ class BinaryCacheIndex(object): with self._index_file_cache.read_transaction(cache_key): db._read_from_file(cache_path) - spec_list = db.query_local(installed=False) + spec_list = db.query_local(installed=False, in_buildcache=True) for indexed_spec in spec_list: dag_hash = indexed_spec.dag_hash() @@ -716,7 +716,7 @@ def generate_package_index(cache_prefix): db_root_dir = os.path.join(tmpdir, 'db_root') db = spack_db.Database(None, db_dir=db_root_dir, enable_transaction_locking=False, - record_fields=['spec', 'ref_count']) + record_fields=['spec', 'ref_count', 'in_buildcache']) try: file_list = ( @@ -748,6 +748,7 @@ def generate_package_index(cache_prefix): # s = Spec.from_yaml(yaml_obj) s = Spec.from_yaml(yaml_contents) db.add(s, None) + db.mark(s, 'in_buildcache', True) except (URLError, web_util.SpackWebError) as url_err: tty.error('Error reading spec.yaml: {0}'.format(file_path)) tty.error(url_err) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 650ab48492..dee9a90d10 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -187,7 +187,8 @@ class InstallRecord(object): ref_count=0, explicit=False, installation_time=None, - deprecated_for=None + deprecated_for=None, + in_buildcache=False, ): self.spec = spec self.path = str(path) if path else None @@ -196,6 +197,7 @@ class InstallRecord(object): self.explicit = explicit self.installation_time = installation_time or _now() self.deprecated_for = deprecated_for + self.in_buildcache = in_buildcache def install_type_matches(self, installed): installed = InstallStatuses.canonicalize(installed) @@ -282,6 +284,11 @@ _query_docstring = """ hashes (container): list or set of hashes that we can use to restrict the search + in_buildcache (bool or any, optional): Specs that are marked in + this database as part of an associated binary cache are + ``in_buildcache``. All other specs are not. This field is used + for querying mirror indices. Default is ``any``. + Returns: list of specs that match the query @@ -1256,6 +1263,16 @@ class Database(object): self._data[spec_key] = spec_rec @_autospec + def mark(self, spec, key, value): + """Mark an arbitrary record on a spec.""" + with self.write_transaction(): + return self._mark(spec, key, value) + + def _mark(self, spec, key, value): + record = self._data[self._get_matching_spec_key(spec)] + setattr(record, key, value) + + @_autospec def deprecate(self, spec, deprecator): """Marks a spec as deprecated in favor of its deprecator""" with self.write_transaction(): @@ -1415,7 +1432,8 @@ class Database(object): explicit=any, start_date=None, end_date=None, - hashes=None + hashes=None, + in_buildcache=any, ): """Run a query on the database.""" @@ -1447,6 +1465,9 @@ class Database(object): if not rec.install_type_matches(installed): continue + if in_buildcache is not any and rec.in_buildcache != in_buildcache: + continue + if explicit is not any and rec.explicit != explicit: continue diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py index 1868d74ccc..fb2db23365 100644 --- a/lib/spack/spack/test/bindist.py +++ b/lib/spack/spack/test/bindist.py @@ -2,6 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import glob import os import sys import platform @@ -435,6 +436,45 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir): assert rebuild +@pytest.mark.usefixtures( + 'install_mockery_mutable_config', 'mock_packages', 'mock_fetch', +) +def test_generate_index_missing(monkeypatch, tmpdir, mutable_config): + """Ensure spack buildcache index only reports available packages""" + + # Create a temp mirror directory for buildcache usage + mirror_dir = tmpdir.join('mirror_dir') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + spack.config.set('mirrors', {'test': mirror_url}) + + s = Spec('libdwarf').concretized() + + # Install a package + install_cmd('--no-cache', s.name) + + # Create a buildcache and update index + buildcache_cmd('create', '-uad', mirror_dir.strpath, s.name) + buildcache_cmd('update-index', '-d', mirror_dir.strpath) + + # Check package and dependency in buildcache + cache_list = buildcache_cmd('list', '--allarch') + assert 'libdwarf' in cache_list + assert 'libelf' in cache_list + + # Remove dependency from cache + libelf_files = glob.glob( + os.path.join(mirror_dir.join('build_cache').strpath, '*libelf*')) + os.remove(*libelf_files) + + # Update index + buildcache_cmd('update-index', '-d', mirror_dir.strpath) + + # Check dependency not in buildcache + cache_list = buildcache_cmd('list', '--allarch') + assert 'libdwarf' in cache_list + assert 'libelf' not in cache_list + + def test_generate_indices_key_error(monkeypatch, capfd): def mock_list_url(url, recursive=False): |