summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorScott Wittenburg <scott.wittenburg@kitware.com>2020-10-02 11:00:42 -0600
committerGitHub <noreply@github.com>2020-10-02 11:00:42 -0600
commita44135dccf8d7386597513c5dea6f71f7c12a9b4 (patch)
tree8fa0848e706a3f011efb05c926d6b5130733f6c2
parent0e8be35c25cdd51543faba53578e34286b728fc3 (diff)
downloadspack-a44135dccf8d7386597513c5dea6f71f7c12a9b4.tar.gz
spack-a44135dccf8d7386597513c5dea6f71f7c12a9b4.tar.bz2
spack-a44135dccf8d7386597513c5dea6f71f7c12a9b4.tar.xz
spack-a44135dccf8d7386597513c5dea6f71f7c12a9b4.zip
Update buildcache key index when we update the package index (#19117)
This changes makes sure that when we run the pipeline job that updates the buildcache package index on the remote mirror, we also update the key index. The public keys corresponding to the signing keys used to sign the package was pushed to the mirror as a part of creating the buildcache index, so this is just ensuring those keys are reflected in the key index. Also, this change makes sure the "spack buildcache update-index" job runs even when there may have been pipeline failures, since we would like the index always to reflect the true state of the mirror.
-rw-r--r--lib/spack/spack/binary_distribution.py4
-rw-r--r--lib/spack/spack/ci.py5
-rw-r--r--lib/spack/spack/cmd/buildcache.py10
-rw-r--r--lib/spack/spack/test/cmd/buildcache.py41
-rwxr-xr-xshare/spack/spack-completion.bash2
5 files changed, 59 insertions, 3 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index c14c84e346..7e156ba998 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -400,6 +400,10 @@ def build_cache_relative_path():
return _build_cache_relative_path
+def build_cache_keys_relative_path():
+ return _build_cache_keys_relative_path
+
+
def build_cache_prefix(prefix):
return os.path.join(prefix, build_cache_relative_path())
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index f231bea9ce..6c7a54a26a 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -806,9 +806,10 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
final_stage = 'stage-rebuild-index'
final_job = {
'stage': final_stage,
- 'script': 'spack buildcache update-index -d {0}'.format(
+ 'script': 'spack buildcache update-index --keys -d {0}'.format(
mirror_urls[0]),
- 'tags': final_job_config['tags']
+ 'tags': final_job_config['tags'],
+ 'when': 'always'
}
if 'image' in final_job_config:
final_job['image'] = final_job_config['image']
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py
index 1e0ccb27d4..ff93646ffd 100644
--- a/lib/spack/spack/cmd/buildcache.py
+++ b/lib/spack/spack/cmd/buildcache.py
@@ -231,6 +231,9 @@ def setup_parser(subparser):
'update-index', help=buildcache_update_index.__doc__)
update_index.add_argument(
'-d', '--mirror-url', default=None, help='Destination mirror url')
+ update_index.add_argument(
+ '-k', '--keys', default=False, action='store_true',
+ help='If provided, key index will be updated as well as package index')
update_index.set_defaults(func=buildcache_update_index)
@@ -777,6 +780,13 @@ def buildcache_update_index(args):
bindist.generate_package_index(
url_util.join(outdir, bindist.build_cache_relative_path()))
+ if args.keys:
+ keys_url = url_util.join(outdir,
+ bindist.build_cache_relative_path(),
+ bindist.build_cache_keys_relative_path())
+
+ bindist.generate_key_index(keys_url)
+
def buildcache(parser, args):
if args.func:
diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py
index 6cd60db55f..cbf8a76051 100644
--- a/lib/spack/spack/test/cmd/buildcache.py
+++ b/lib/spack/spack/test/cmd/buildcache.py
@@ -19,6 +19,9 @@ buildcache = spack.main.SpackCommand('buildcache')
install = spack.main.SpackCommand('install')
env = spack.main.SpackCommand('env')
add = spack.main.SpackCommand('add')
+gpg = spack.main.SpackCommand('gpg')
+mirror = spack.main.SpackCommand('mirror')
+uninstall = spack.main.SpackCommand('uninstall')
@pytest.fixture()
@@ -133,3 +136,41 @@ def test_buildcache_create_fail_on_perm_denied(
'--unsigned', 'trivial-install-test-package')
assert error.value.errno == errno.EACCES
tmpdir.chmod(0o700)
+
+
+@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
+ reason='This test requires gpg')
+def test_update_key_index(tmpdir, mutable_mock_env_path,
+ install_mockery, mock_packages, mock_fetch,
+ mock_stage, mock_gnupghome):
+ """Test the update-index command with the --keys option"""
+ working_dir = tmpdir.join('working_dir')
+
+ mirror_dir = working_dir.join('mirror')
+ mirror_url = 'file://{0}'.format(mirror_dir.strpath)
+
+ mirror('add', 'test-mirror', mirror_url)
+
+ gpg('create', 'Test Signing Key', 'nobody@nowhere.com')
+
+ s = Spec('libdwarf').concretized()
+
+ # Install a package
+ install(s.name)
+
+ # Put installed package in the buildcache, which, because we're signing
+ # it, should result in the public key getting pushed to the buildcache
+ # as well.
+ buildcache('create', '-a', '-d', mirror_dir.strpath, s.name)
+
+ # Now make sure that when we pass the "--keys" argument to update-index
+ # it causes the index to get update.
+ buildcache('update-index', '--keys', '-d', mirror_dir.strpath)
+
+ key_dir_list = os.listdir(os.path.join(
+ mirror_dir.strpath, 'build_cache', '_pgp'))
+
+ uninstall('-y', s.name)
+ mirror('rm', 'test-mirror')
+
+ assert 'index.json' in key_dir_list
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 4ecec82bd9..c73bdffcb7 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -434,7 +434,7 @@ _spack_buildcache_copy() {
}
_spack_buildcache_update_index() {
- SPACK_COMPREPLY="-h --help -d --mirror-url"
+ SPACK_COMPREPLY="-h --help -d --mirror-url -k --keys"
}
_spack_cd() {