summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorOmar Padron <omar.padron@kitware.com>2020-09-25 12:54:24 -0400
committerGitHub <noreply@github.com>2020-09-25 12:54:24 -0400
commit2d9315411900d19487b1b216e1717ed83a65ace8 (patch)
treefbdc0272ed52c15a2bca21f7eb3eb71783eb6759 /lib
parent421f4e12a79a4ad2361970c285ed896a476e8bbf (diff)
downloadspack-2d9315411900d19487b1b216e1717ed83a65ace8.tar.gz
spack-2d9315411900d19487b1b216e1717ed83a65ace8.tar.bz2
spack-2d9315411900d19487b1b216e1717ed83a65ace8.tar.xz
spack-2d9315411900d19487b1b216e1717ed83a65ace8.zip
Streamline key management for build caches (#17792)
* Rework spack.util.web.list_url() list_url() now accepts an optional recursive argument (default: False) for controlling whether to only return files within the prefix url or to return all files whose path starts with the prefix url. Allows for the most effecient implementation for the given prefix url scheme. For example, only recursive queries are supported for S3 prefixes, so the returned list is trimmed down if recursive == False, but the native search is returned as-is when recursive == True. Suitable implementations for each case are also used for file system URLs. * Switch to using an explicit index for public keys Switches to maintaining a build cache's keys under build_cache/_pgp. Within this directory is an index.json file listing all the available keys and a <fingerprint>.pub file for each such key. - Adds spack.binary_distribution.generate_key_index() - (re)generates a build cache's key index - Modifies spack.binary_distribution.build_tarball() - if tarball is signed, automatically pushes the key used for signing along with the tarball - if regenerate_index == True, automatically (re)generates the build cache's key index along with the build cache's package index; as in spack.binary_distribution.generate_key_index() - Modifies spack.binary_distribution.get_keys() - a build cache's key index is now used instead of programmatic listing - Adds spack.binary_distribution.push_keys() - publishes keys from Spack's keyring to a given list of mirrors - Adds new spack subcommand: spack gpg publish - publishes keys from Spack's keyring to a given list of mirrors - Modifies spack.util.gpg.Gpg.signing_keys() - Accepts optional positional arguments for filtering the set of keys returned - Adds spack.util.gpg.Gpg.public_keys() - As spack.util.gpg.Gpg.signing_keys(), except public keys are returned - Modifies spack.util.gpg.Gpg.export_keys() - Fixes an issue where GnuPG would prompt for user input if trying to overwrite an existing file - Modifies spack.util.gpg.Gpg.untrust() - Fixes an issue where GnuPG would fail for input that were not key fingerprints - Modifies spack.util.web.url_exists() - Fixes an issue where url_exists() would throw instead of returning False * rework gpg module/fix error with very long GNUPGHOME dir * add a shim for functools.cached_property * handle permission denied error in gpg util * fix tests/make gpgconf optional if no socket dir is available
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/binary_distribution.py214
-rw-r--r--lib/spack/spack/cmd/gpg.py68
-rw-r--r--lib/spack/spack/test/bindist.py41
-rw-r--r--lib/spack/spack/test/ci.py11
-rw-r--r--lib/spack/spack/test/cmd/ci.py11
-rw-r--r--lib/spack/spack/test/cmd/gpg.py36
-rw-r--r--lib/spack/spack/test/conftest.py6
-rw-r--r--lib/spack/spack/test/packaging.py25
-rw-r--r--lib/spack/spack/test/stage.py3
-rw-r--r--lib/spack/spack/test/util/util_gpg.py32
-rw-r--r--lib/spack/spack/test/web.py28
-rw-r--r--lib/spack/spack/util/gpg.py468
-rw-r--r--lib/spack/spack/util/web.py18
13 files changed, 736 insertions, 225 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index 590f6e6710..3e70ddf792 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -26,17 +26,18 @@ import spack.cmd
import spack.config as config
import spack.database as spack_db
import spack.fetch_strategy as fs
-import spack.util.gpg
import spack.relocate as relocate
+import spack.util.gpg
+import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
import spack.mirror
import spack.util.url as url_util
import spack.util.web as web_util
from spack.spec import Spec
from spack.stage import Stage
-from spack.util.gpg import Gpg
_build_cache_relative_path = 'build_cache'
+_build_cache_keys_relative_path = '_pgp'
BUILD_CACHE_INDEX_TEMPLATE = '''
<html>
@@ -247,15 +248,9 @@ def checksum_tarball(file):
return hasher.hexdigest()
-def sign_tarball(key, force, specfile_path):
- # Sign the packages if keys available
- if spack.util.gpg.Gpg.gpg() is None:
- raise NoGpgException(
- "gpg2 is not available in $PATH .\n"
- "Use spack install gnupg and spack load gnupg.")
-
+def select_signing_key(key=None):
if key is None:
- keys = Gpg.signing_keys()
+ keys = spack.util.gpg.signing_keys()
if len(keys) == 1:
key = keys[0]
@@ -263,26 +258,30 @@ def sign_tarball(key, force, specfile_path):
raise PickKeyException(str(keys))
if len(keys) == 0:
- msg = "No default key available for signing.\n"
- msg += "Use spack gpg init and spack gpg create"
- msg += " to create a default key."
- raise NoKeyException(msg)
+ raise NoKeyException(
+ "No default key available for signing.\n"
+ "Use spack gpg init and spack gpg create"
+ " to create a default key.")
+ return key
+
+def sign_tarball(key, force, specfile_path):
if os.path.exists('%s.asc' % specfile_path):
if force:
os.remove('%s.asc' % specfile_path)
else:
raise NoOverwriteException('%s.asc' % specfile_path)
- Gpg.sign(key, specfile_path, '%s.asc' % specfile_path)
+ key = select_signing_key(key)
+ spack.util.gpg.sign(key, specfile_path, '%s.asc' % specfile_path)
def generate_package_index(cache_prefix):
"""Create the build cache index page.
Creates (or replaces) the "index.json" page at the location given in
- cache_prefix. This page contains a link for each binary package (*.yaml)
- and public key (*.key) under cache_prefix.
+ cache_prefix. This page contains a link for each binary package (.yaml)
+ under cache_prefix.
"""
tmpdir = tempfile.mkdtemp()
db_root_dir = os.path.join(tmpdir, 'db_root')
@@ -325,6 +324,45 @@ def generate_package_index(cache_prefix):
shutil.rmtree(tmpdir)
+def generate_key_index(key_prefix, tmpdir=None):
+ """Create the key index page.
+
+ Creates (or replaces) the "index.json" page at the location given in
+ key_prefix. This page contains an entry for each key (.pub) under
+ key_prefix.
+ """
+
+ tty.debug(' '.join(('Retrieving key.pub files from',
+ url_util.format(key_prefix),
+ 'to build key index')))
+
+ fingerprints = (
+ entry[:-4]
+ for entry in web_util.list_url(key_prefix, recursive=False)
+ if entry.endswith('.pub'))
+
+ keys_local = url_util.local_file_path(key_prefix)
+ if keys_local:
+ target = os.path.join(keys_local, 'index.json')
+ else:
+ target = os.path.join(tmpdir, 'index.json')
+
+ index = {
+ 'keys': dict(
+ (fingerprint, {}) for fingerprint
+ in sorted(set(fingerprints)))
+ }
+ with open(target, 'w') as f:
+ sjson.dump(index, f)
+
+ if not keys_local:
+ web_util.push_to_url(
+ target,
+ url_util.join(key_prefix, 'index.json'),
+ keep_original=False,
+ extra_args={'ContentType': 'application/json'})
+
+
def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
allow_root=False, key=None, regenerate_index=False):
"""
@@ -445,7 +483,9 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
# sign the tarball and spec file with gpg
if not unsigned:
+ key = select_signing_key(key)
sign_tarball(key, force, specfile_path)
+
# put tarball, spec and signature files in .spack archive
with closing(tarfile.open(spackfile_path, 'w')) as tar:
tar.add(name=tarfile_path, arcname='%s' % tarfile_name)
@@ -468,7 +508,15 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
.format(spec, remote_spackfile_path))
try:
- # create an index.html for the build_cache directory so specs can be
+ # push the key to the build cache's _pgp directory so it can be
+ # imported
+ if not unsigned:
+ push_keys(outdir,
+ keys=[key],
+ regenerate_index=regenerate_index,
+ tmpdir=tmpdir)
+
+ # create an index.json for the build_cache directory so specs can be
# found
if regenerate_index:
generate_package_index(url_util.join(
@@ -695,7 +743,8 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
if os.path.exists('%s.asc' % specfile_path):
try:
suppress = config.get('config:suppress_gpg_warnings', False)
- Gpg.verify('%s.asc' % specfile_path, specfile_path, suppress)
+ spack.util.gpg.verify(
+ '%s.asc' % specfile_path, specfile_path, suppress)
except Exception as e:
shutil.rmtree(tmpdir)
raise e
@@ -898,41 +947,46 @@ def get_specs():
return _cached_specs
-def get_keys(install=False, trust=False, force=False):
- """
- Get pgp public keys available on mirror
- with suffix .key or .pub
+def get_keys(install=False, trust=False, force=False, mirrors=None):
+ """Get pgp public keys available on mirror with suffix .pub
"""
- if not spack.mirror.MirrorCollection():
+ mirror_collection = (mirrors or spack.mirror.MirrorCollection())
+
+ if not mirror_collection:
tty.die("Please add a spack mirror to allow " +
"download of build caches.")
- keys = set()
+ for mirror in mirror_collection.values():
+ fetch_url = mirror.fetch_url
+ keys_url = url_util.join(fetch_url,
+ _build_cache_relative_path,
+ _build_cache_keys_relative_path)
+ keys_index = url_util.join(keys_url, 'index.json')
- for mirror in spack.mirror.MirrorCollection().values():
- fetch_url_build_cache = url_util.join(
- mirror.fetch_url, _build_cache_relative_path)
+ tty.debug('Finding public keys in {0}'.format(
+ url_util.format(fetch_url)))
- mirror_dir = url_util.local_file_path(fetch_url_build_cache)
- if mirror_dir:
- tty.debug('Finding public keys in {0}'.format(mirror_dir))
- files = os.listdir(str(mirror_dir))
- for file in files:
- if re.search(r'\.key', file) or re.search(r'\.pub', file):
- link = url_util.join(fetch_url_build_cache, file)
- keys.add(link)
- else:
- tty.debug('Finding public keys at {0}'
- .format(url_util.format(fetch_url_build_cache)))
- # For s3 mirror need to request index.html directly
- p, links = web_util.spider(
- url_util.join(fetch_url_build_cache, 'index.html'))
+ try:
+ _, _, json_file = web_util.read_from_url(keys_index)
+ json_index = sjson.load(codecs.getreader('utf-8')(json_file))
+ except (URLError, web_util.SpackWebError) as url_err:
+ if web_util.url_exists(keys_index):
+ err_msg = [
+ 'Unable to find public keys in {0},',
+ ' caught exception attempting to read from {1}.',
+ ]
- for link in links:
- if re.search(r'\.key', link) or re.search(r'\.pub', link):
- keys.add(link)
+ tty.error(''.join(err_msg).format(
+ url_util.format(fetch_url),
+ url_util.format(keys_index)))
+
+ tty.debug(url_err)
+
+ continue
+
+ for fingerprint, key_attributes in json_index['keys'].items():
+ link = os.path.join(keys_url, fingerprint + '.pub')
- for link in keys:
with Stage(link, name="build_cache", keep=True) as stage:
if os.path.exists(stage.save_filename) and force:
os.remove(stage.save_filename)
@@ -941,16 +995,80 @@ def get_keys(install=False, trust=False, force=False):
stage.fetch()
except fs.FetchError:
continue
- tty.debug('Found key {0}'.format(link))
+
+ tty.debug('Found key {0}'.format(fingerprint))
if install:
if trust:
- Gpg.trust(stage.save_filename)
+ spack.util.gpg.trust(stage.save_filename)
tty.debug('Added this key to trusted keys.')
else:
tty.debug('Will not add this key to trusted keys.'
'Use -t to install all downloaded keys')
+def push_keys(*mirrors, **kwargs):
+ """
+ Upload pgp public keys to the given mirrors
+ """
+ keys = kwargs.get('keys')
+ regenerate_index = kwargs.get('regenerate_index', False)
+ tmpdir = kwargs.get('tmpdir')
+ remove_tmpdir = False
+
+ keys = spack.util.gpg.public_keys(*(keys or []))
+
+ try:
+ for mirror in mirrors:
+ push_url = getattr(mirror, 'push_url', mirror)
+ keys_url = url_util.join(push_url,
+ _build_cache_relative_path,
+ _build_cache_keys_relative_path)
+ keys_local = url_util.local_file_path(keys_url)
+
+ verb = 'Writing' if keys_local else 'Uploading'
+ tty.debug('{0} public keys to {1}'.format(
+ verb, url_util.format(push_url)))
+
+ if keys_local: # mirror is local, don't bother with the tmpdir
+ prefix = keys_local
+ mkdirp(keys_local)
+ else:
+ # A tmp dir is created for the first mirror that is non-local.
+ # On the off-hand chance that all the mirrors are local, then
+ # we can avoid the need to create a tmp dir.
+ if tmpdir is None:
+ tmpdir = tempfile.mkdtemp()
+ remove_tmpdir = True
+ prefix = tmpdir
+
+ for fingerprint in keys:
+ tty.debug(' ' + fingerprint)
+ filename = fingerprint + '.pub'
+
+ export_target = os.path.join(prefix, filename)
+ spack.util.gpg.export_keys(export_target, fingerprint)
+
+ # If mirror is local, the above export writes directly to the
+ # mirror (export_target points directly to the mirror).
+ #
+ # If not, then export_target is a tmpfile that needs to be
+ # uploaded to the mirror.
+ if not keys_local:
+ spack.util.web.push_to_url(
+ export_target,
+ url_util.join(keys_url, filename),
+ keep_original=False)
+
+ if regenerate_index:
+ if keys_local:
+ generate_key_index(keys_url)
+ else:
+ generate_key_index(keys_url, tmpdir)
+ finally:
+ if remove_tmpdir:
+ shutil.rmtree(tmpdir)
+
+
def needs_rebuild(spec, mirror_url, rebuild_on_errors=False):
if not spec.concrete:
raise ValueError('spec must be concrete to check against mirror')
diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py
index 0a77812c12..59fba4a4df 100644
--- a/lib/spack/spack/cmd/gpg.py
+++ b/lib/spack/spack/cmd/gpg.py
@@ -6,9 +6,10 @@
import os
import argparse
+import spack.binary_distribution
import spack.cmd.common.arguments as arguments
import spack.paths
-from spack.util.gpg import Gpg
+import spack.util.gpg
description = "handle GPG actions for spack"
section = "packaging"
@@ -81,37 +82,64 @@ def setup_parser(subparser):
'all secret keys if unspecified')
export.set_defaults(func=gpg_export)
+ publish = subparsers.add_parser('publish', help=gpg_publish.__doc__)
+
+ output = publish.add_mutually_exclusive_group(required=True)
+ output.add_argument('-d', '--directory',
+ metavar='directory',
+ type=str,
+ help="local directory where " +
+ "keys will be published.")
+ output.add_argument('-m', '--mirror-name',
+ metavar='mirror-name',
+ type=str,
+ help="name of the mirror where " +
+ "keys will be published.")
+ output.add_argument('--mirror-url',
+ metavar='mirror-url',
+ type=str,
+ help="URL of the mirror where " +
+ "keys will be published.")
+ publish.add_argument('--rebuild-index', action='store_true',
+ default=False, help=(
+ "Regenerate buildcache key index "
+ "after publishing key(s)"))
+ publish.add_argument('keys', nargs='*',
+ help='the keys to publish; '
+ 'all public keys if unspecified')
+ publish.set_defaults(func=gpg_publish)
+
def gpg_create(args):
"""create a new key"""
if args.export:
- old_sec_keys = Gpg.signing_keys()
- Gpg.create(name=args.name, email=args.email,
- comment=args.comment, expires=args.expires)
+ old_sec_keys = spack.util.gpg.signing_keys()
+ spack.util.gpg.create(name=args.name, email=args.email,
+ comment=args.comment, expires=args.expires)
if args.export:
- new_sec_keys = set(Gpg.signing_keys())
+ new_sec_keys = set(spack.util.gpg.signing_keys())
new_keys = new_sec_keys.difference(old_sec_keys)
- Gpg.export_keys(args.export, *new_keys)
+ spack.util.gpg.export_keys(args.export, *new_keys)
def gpg_export(args):
"""export a secret key"""
keys = args.keys
if not keys:
- keys = Gpg.signing_keys()
- Gpg.export_keys(args.location, *keys)
+ keys = spack.util.gpg.signing_keys()
+ spack.util.gpg.export_keys(args.location, *keys)
def gpg_list(args):
"""list keys available in the keyring"""
- Gpg.list(args.trusted, args.signing)
+ spack.util.gpg.list(args.trusted, args.signing)
def gpg_sign(args):
"""sign a package"""
key = args.key
if key is None:
- keys = Gpg.signing_keys()
+ keys = spack.util.gpg.signing_keys()
if len(keys) == 1:
key = keys[0]
elif not keys:
@@ -123,12 +151,12 @@ def gpg_sign(args):
if not output:
output = args.spec[0] + '.asc'
# TODO: Support the package format Spack creates.
- Gpg.sign(key, ' '.join(args.spec), output, args.clearsign)
+ spack.util.gpg.sign(key, ' '.join(args.spec), output, args.clearsign)
def gpg_trust(args):
"""add a key to the keyring"""
- Gpg.trust(args.keyfile)
+ spack.util.gpg.trust(args.keyfile)
def gpg_init(args):
@@ -141,12 +169,12 @@ def gpg_init(args):
for filename in filenames:
if not filename.endswith('.key'):
continue
- Gpg.trust(os.path.join(root, filename))
+ spack.util.gpg.trust(os.path.join(root, filename))
def gpg_untrust(args):
"""remove a key from the keyring"""
- Gpg.untrust(args.signing, *args.keys)
+ spack.util.gpg.untrust(args.signing, *args.keys)
def gpg_verify(args):
@@ -155,7 +183,17 @@ def gpg_verify(args):
signature = args.signature
if signature is None:
signature = args.spec[0] + '.asc'
- Gpg.verify(signature, ' '.join(args.spec))
+ spack.util.gpg.verify(signature, ' '.join(args.spec))
+
+
+def gpg_publish(args):
+ """publish public keys to a build cache"""
+
+ # TODO(opadron): switch to using the mirror args once #17547 is merged
+ mirror = args.directory
+
+ spack.binary_distribution.push_keys(
+ mirror, keys=args.keys, regenerate_index=args.rebuild_index)
def gpg(parser, args):
diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py
index a7995ca43b..9b2f1b1408 100644
--- a/lib/spack/spack/test/bindist.py
+++ b/lib/spack/spack/test/bindist.py
@@ -19,8 +19,10 @@ import spack.cmd.buildcache as buildcache
import spack.cmd.install as install
import spack.cmd.uninstall as uninstall
import spack.cmd.mirror as mirror
-from spack.spec import Spec
+import spack.mirror
+import spack.util.gpg
from spack.directory_layout import YamlDirectoryLayout
+from spack.spec import Spec
def_install_path_scheme = '${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}' # noqa: E501
@@ -469,3 +471,40 @@ def test_relative_rpaths_install_nondefault(tmpdir,
margs = mparser.parse_args(
['rm', '--scope', 'site', 'test-mirror-rel'])
mirror.mirror(mparser, margs)
+
+
+def test_push_and_fetch_keys(mock_gnupghome):
+ testpath = str(mock_gnupghome)
+
+ mirror = os.path.join(testpath, 'mirror')
+ mirrors = {'test-mirror': mirror}
+ mirrors = spack.mirror.MirrorCollection(mirrors)
+ mirror = spack.mirror.Mirror('file://' + mirror)
+
+ gpg_dir1 = os.path.join(testpath, 'gpg1')
+ gpg_dir2 = os.path.join(testpath, 'gpg2')
+
+ # dir 1: create a new key, record its fingerprint, and push it to a new
+ # mirror
+ with spack.util.gpg.gnupg_home_override(gpg_dir1):
+ spack.util.gpg.create(name='test-key',
+ email='fake@test.key',
+ expires='0',
+ comment=None)
+
+ keys = spack.util.gpg.public_keys()
+ assert len(keys) == 1
+ fpr = keys[0]
+
+ bindist.push_keys(mirror, keys=[fpr], regenerate_index=True)
+
+ # dir 2: import the key from the mirror, and confirm that its fingerprint
+ # matches the one created above
+ with spack.util.gpg.gnupg_home_override(gpg_dir2):
+ assert len(spack.util.gpg.public_keys()) == 0
+
+ bindist.get_keys(mirrors=mirrors, install=True, trust=True, force=True)
+
+ new_keys = spack.util.gpg.public_keys()
+ assert len(new_keys) == 1
+ assert new_keys[0] == fpr
diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py
index a2ad2708a1..f1986b34d0 100644
--- a/lib/spack/spack/test/ci.py
+++ b/lib/spack/spack/test/ci.py
@@ -53,15 +53,8 @@ def test_urlencode_string():
assert(s_enc == 'Spack+Test+Project')
-def has_gpg():
- try:
- gpg = spack.util.gpg.Gpg.gpg()
- except spack.util.gpg.SpackGPGError:
- gpg = None
- return bool(gpg)
-
-
-@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg')
+@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
+ reason='This test requires gpg')
def test_import_signing_key(mock_gnupghome):
signing_key_dir = spack_paths.mock_gpg_keys_path
signing_key_path = os.path.join(signing_key_dir, 'package-signing-key')
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index 82f7fd86cc..abb9f5c5ad 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -35,14 +35,6 @@ buildcache_cmd = spack.main.SpackCommand('buildcache')
git = exe.which('git', required=True)
-def has_gpg():
- try:
- gpg = spack.util.gpg.Gpg.gpg()
- except spack.util.gpg.SpackGPGError:
- gpg = None
- return bool(gpg)
-
-
@pytest.fixture()
def env_deactivate():
yield
@@ -690,7 +682,8 @@ spack:
@pytest.mark.disable_clean_stage_check
-@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg')
+@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
+ reason='This test requires gpg')
def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages, mock_fetch,
mock_stage, mock_gnupghome):
diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py
index 56d178f449..751790182b 100644
--- a/lib/spack/spack/test/cmd/gpg.py
+++ b/lib/spack/spack/test/cmd/gpg.py
@@ -29,39 +29,35 @@ gpg = SpackCommand('gpg')
('gpg2', 'gpg (GnuPG) 2.2.19'), # gpg2 command
])
def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
+ TEMPLATE = ('#!/bin/sh\n'
+ 'echo "{version}"\n')
+
with tmpdir.as_cwd():
- with open(cmd_name, 'w') as f:
- f.write("""\
-#!/bin/sh
-echo "{version}"
-""".format(version=version))
- fs.set_executable(cmd_name)
+ for fname in (cmd_name, 'gpgconf'):
+ with open(fname, 'w') as f:
+ f.write(TEMPLATE.format(version=version))
+ fs.set_executable(fname)
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
if version == 'undetectable' or version.endswith('1.3.4'):
with pytest.raises(spack.util.gpg.SpackGPGError):
- exe = spack.util.gpg.Gpg.gpg()
+ spack.util.gpg.ensure_gpg(reevaluate=True)
else:
- exe = spack.util.gpg.Gpg.gpg()
- assert isinstance(exe, spack.util.executable.Executable)
+ spack.util.gpg.ensure_gpg(reevaluate=True)
+ gpg_exe = spack.util.gpg.get_global_gpg_instance().gpg_exe
+ assert isinstance(gpg_exe, spack.util.executable.Executable)
+ gpgconf_exe = spack.util.gpg.get_global_gpg_instance().gpgconf_exe
+ assert isinstance(gpgconf_exe, spack.util.executable.Executable)
def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch):
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
with pytest.raises(spack.util.gpg.SpackGPGError):
- spack.util.gpg.Gpg.gpg()
-
-
-def has_gpg():
- try:
- gpg = spack.util.gpg.Gpg.gpg()
- except spack.util.gpg.SpackGPGError:
- gpg = None
- return bool(gpg)
+ spack.util.gpg.ensure_gpg(reevaluate=True)
@pytest.mark.maybeslow
-@pytest.mark.skipif(not has_gpg(),
+@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
reason='These tests require gnupg2')
def test_gpg(tmpdir, mock_gnupghome):
# Verify a file with an empty keyring.
@@ -103,7 +99,7 @@ def test_gpg(tmpdir, mock_gnupghome):
'--export', str(keypath),
'Spack testing 1',
'spack@googlegroups.com')
- keyfp = spack.util.gpg.Gpg.signing_keys()[0]
+ keyfp = spack.util.gpg.signing_keys()[0]
# List the keys.
# TODO: Test the output here.
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index a8cdce90ee..152c2c846e 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -781,10 +781,8 @@ def mock_gnupghome(monkeypatch):
# This comes up because tmp paths on macOS are already long-ish, and
# pytest makes them longer.
short_name_tmpdir = tempfile.mkdtemp()
- monkeypatch.setattr(spack.util.gpg, 'GNUPGHOME', short_name_tmpdir)
- monkeypatch.setattr(spack.util.gpg.Gpg, '_gpg', None)
-
- yield
+ with spack.util.gpg.gnupg_home_override(short_name_tmpdir):
+ yield short_name_tmpdir
# clean up, since we are doing this manually
shutil.rmtree(short_name_tmpdir)
diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py
index bee0d01b16..e2cef22577 100644
--- a/lib/spack/spack/test/packaging.py
+++ b/lib/spack/spack/test/packaging.py
@@ -20,6 +20,7 @@ import spack.repo
import spack.store
import spack.binary_distribution as bindist
import spack.cmd.buildcache as buildcache
+import spack.util.gpg
from spack.spec import Spec
from spack.paths import mock_gpg_keys_path
from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite
@@ -31,14 +32,6 @@ from spack.relocate import _placeholder, macho_find_paths
from spack.relocate import file_is_relocatable
-def has_gpg():
- try:
- gpg = spack.util.gpg.Gpg.gpg()
- except spack.util.gpg.SpackGPGError:
- gpg = None
- return bool(gpg)
-
-
def fake_fetchify(url, pkg):
"""Fake the URL for a package so it downloads from a file."""
fetcher = FetchStrategyComposite()
@@ -46,7 +39,8 @@ def fake_fetchify(url, pkg):
pkg.fetcher = fetcher
-@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg')
+@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
+ reason='This test requires gpg')
@pytest.mark.usefixtures('install_mockery', 'mock_gnupghome')
def test_buildcache(mock_archive, tmpdir):
# tweak patchelf to only do a download
@@ -101,12 +95,9 @@ echo $PATH"""
create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash]
# Create a private key to sign package with if gpg2 available
- if spack.util.gpg.Gpg.gpg():
- spack.util.gpg.Gpg.create(name='test key 1', expires='0',
- email='spack@googlegroups.com',
- comment='Spack test key')
- else:
- create_args.insert(create_args.index('-a'), '-u')
+ spack.util.gpg.create(name='test key 1', expires='0',
+ email='spack@googlegroups.com',
+ comment='Spack test key')
create_args.insert(create_args.index('-a'), '--rebuild-index')
@@ -119,8 +110,6 @@ echo $PATH"""
pkg.do_uninstall(force=True)
install_args = ['install', '-a', '-f', pkghash]
- if not spack.util.gpg.Gpg.gpg():
- install_args.insert(install_args.index('-a'), '-u')
args = parser.parse_args(install_args)
# Test install
buildcache.buildcache(parser, args)
@@ -144,8 +133,6 @@ echo $PATH"""
# Uninstall the package
pkg.do_uninstall(force=True)
- if not spack.util.gpg.Gpg.gpg():
- install_args.insert(install_args.index('-a'), '-u')
args = parser.parse_args(install_args)
buildcache.buildcache(parser, args)
diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py
index 204a6e6a3e..9dea091730 100644
--- a/lib/spack/spack/test/stage.py
+++ b/lib/spack/spack/test/stage.py
@@ -839,7 +839,8 @@ class TestStage(object):
assert 'spack' in path.split(os.path.sep)
# Make sure cached stage path value was changed appropriately
- assert spack.stage._stage_root == test_path
+ assert spack.stage._stage_root in (
+ test_path, os.path.join(test_path, getpass.getuser()))
# Make sure the directory exists
assert os.path.isdir(spack.stage._stage_root)
diff --git a/lib/spack/spack/test/util/util_gpg.py b/lib/spack/spack/test/util/util_gpg.py
index 4a243985db..83fad508f9 100644
--- a/lib/spack/spack/test/util/util_gpg.py
+++ b/lib/spack/spack/test/util/util_gpg.py
@@ -3,7 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import spack.util.gpg as gpg
+import os
+import pytest
+
+import spack.util.gpg
def test_parse_gpg_output_case_one():
@@ -17,7 +20,7 @@ fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:
uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>:
ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
"""
- keys = gpg.parse_keys_output(output)
+ keys = spack.util.gpg.parse_secret_keys_output(output)
assert len(keys) == 2
assert keys[0] == 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
@@ -34,7 +37,7 @@ ssb:-:2048:1:AAAAAAAAA::::::esa:::+:::23:
fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:
grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
"""
- keys = gpg.parse_keys_output(output)
+ keys = spack.util.gpg.parse_secret_keys_output(output)
assert len(keys) == 1
assert keys[0] == 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
@@ -53,8 +56,29 @@ uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>:
ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
fpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:"""
- keys = gpg.parse_keys_output(output)
+ keys = spack.util.gpg.parse_secret_keys_output(output)
assert len(keys) == 2
assert keys[0] == 'WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW'
assert keys[1] == 'YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY'
+
+
+@pytest.mark.skipif(not spack.util.gpg.GpgConstants.user_run_dir,
+ reason='This test requires /var/run/user/$(id -u)')
+def test_really_long_gnupg_home_dir(tmpdir):
+ N = 960
+
+ tdir = str(tmpdir)
+ while len(tdir) < N:
+ tdir = os.path.join(tdir, 'filler')
+
+ tdir = tdir[:N].rstrip(os.sep)
+ tdir += '0' * (N - len(tdir))
+
+ with spack.util.gpg.gnupg_home_override(tdir):
+ spack.util.gpg.create(name='Spack testing 1',
+ email='test@spack.io',
+ comment='Spack testing key',
+ expires='0')
+
+ spack.util.gpg.list(True, True)
diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py
index dfca41c95b..472ab0dab7 100644
--- a/lib/spack/spack/test/web.py
+++ b/lib/spack/spack/test/web.py
@@ -167,3 +167,31 @@ def test_get_header():
# If there isn't even a fuzzy match, raise KeyError
with pytest.raises(KeyError):
spack.util.web.get_header(headers, 'ContentLength')
+
+
+def test_list_url(tmpdir):
+ testpath = str(tmpdir)
+
+ os.mkdir(os.path.join(testpath, 'dir'))
+
+ with open(os.path.join(testpath, 'file-0.txt'), 'w'):
+ pass
+ with open(os.path.join(testpath, 'file-1.txt'), 'w'):
+ pass
+ with open(os.path.join(testpath, 'file-2.txt'), 'w'):
+ pass
+
+ with open(os.path.join(testpath, 'dir', 'another-file.txt'), 'w'):
+ pass
+
+ list_url = lambda recursive: list(sorted(
+ spack.util.web.list_url(testpath, recursive=recursive)))
+
+ assert list_url(False) == ['file-0.txt',
+ 'file-1.txt',
+ 'file-2.txt']
+
+ assert list_url(True) == ['dir/another-file.txt',
+ 'file-0.txt',
+ 'file-1.txt',
+ 'file-2.txt']
diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py
index e6b7f56741..eaa199e417 100644
--- a/lib/spack/spack/util/gpg.py
+++ b/lib/spack/spack/util/gpg.py
@@ -3,20 +3,73 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import contextlib
+import errno
+import functools
import os
import re
+import llnl.util.lang
+
import spack.error
import spack.paths
+import spack.util.executable
import spack.version
-from spack.util.executable import which
-_gnupg_version_re = r"^gpg \(GnuPG\) (.*)$"
-GNUPGHOME = os.getenv('SPACK_GNUPGHOME', spack.paths.gpg_path)
+_gnupg_version_re = r"^gpg(conf)? \(GnuPG\) (.*)$"
+_gnupg_home_override = None
+_global_gpg_instance = None
+
+
+def get_gnupg_home(gnupg_home=None):
+ """Returns the directory that should be used as the GNUPGHOME environment
+ variable when calling gpg.
+
+ If a [gnupg_home] is passed directly (and not None), that value will be
+ used.
+
+ Otherwise, if there is an override set (and it is not None), then that
+ value will be used.
+
+ Otherwise, if the environment variable "SPACK_GNUPGHOME" is set, then that
+ value will be used.
+
+ Otherwise, the default gpg path for Spack will be used.
+
+ See also: gnupg_home_override()
+ """
+ return (gnupg_home or
+ _gnupg_home_override or
+ os.getenv('SPACK_GNUPGHOME') or
+ spack.paths.gpg_path)
+
+
+@contextlib.contextmanager
+def gnupg_home_override(new_gnupg_home):
+ global _gnupg_home_override
+ global _global_gpg_instance
+
+ old_gnupg_home_override = _gnupg_home_override
+ old_global_gpg_instance = _global_gpg_instance
+
+ _gnupg_home_override = new_gnupg_home
+ _global_gpg_instance = None
+
+ yield
+
+ _gnupg_home_override = old_gnupg_home_override
+ _global_gpg_instance = old_global_gpg_instance
-def parse_keys_output(output):
+def get_global_gpg_instance():
+ global _global_gpg_instance
+ if _global_gpg_instance is None:
+ _global_gpg_instance = Gpg()
+ return _global_gpg_instance
+
+
+def parse_secret_keys_output(output):
keys = []
found_sec = False
for line in output.split('\n'):
@@ -31,43 +84,230 @@ def parse_keys_output(output):
return keys
+def parse_public_keys_output(output):
+ keys = []
+ found_pub = False
+ for line in output.split('\n'):
+ if found_pub:
+ if line.startswith('fpr'):
+ keys.append(line.split(':')[9])
+ found_pub = False
+ elif line.startswith('ssb'):
+ found_pub = False
+ elif line.startswith('pub'):
+ found_pub = True
+ return keys
+
+
+cached_property = getattr(functools, 'cached_property', None)
+
+# If older python version has no cached_property, emulate it here.
+# TODO(opadron): maybe this shim should be moved to llnl.util.lang?
+if not cached_property:
+ def cached_property(*args, **kwargs):
+ result = property(llnl.util.lang.memoized(*args, **kwargs))
+ attr = result.fget.__name__
+
+ @result.deleter
+ def result(self):
+ getattr(type(self), attr).fget.cache.pop((self,), None)
+
+ return result
+
+
+class GpgConstants(object):
+ @cached_property
+ def target_version(self):
+ return spack.version.Version('2')
+
+ @cached_property
+ def gpgconf_string(self):
+ exe_str = spack.util.executable.which_string(
+ 'gpgconf', 'gpg2conf', 'gpgconf2')
+
+ no_gpgconf_msg = (
+ 'Spack requires gpgconf version >= 2\n'
+ ' To install a suitable version using Spack, run\n'
+ ' spack install gnupg@2:\n'
+ ' and load it by running\n'
+ ' spack load gnupg@2:')
+
+ if not exe_str:
+ raise SpackGPGError(no_gpgconf_msg)
+
+ exe = spack.util.executable.Executable(exe_str)
+ output = exe('--version', output=str)
+ match = re.search(_gnupg_version_re, output, re.M)
+
+ if not match:
+ raise SpackGPGError('Could not determine gpgconf version')
+
+ if spack.version.Version(match.group(2)) < self.target_version:
+ raise SpackGPGError(no_gpgconf_msg)
+
+ # ensure that the gpgconf we found can run "gpgconf --create-socketdir"
+ try:
+ exe('--dry-run', '--create-socketdir')
+ except spack.util.executable.ProcessError:
+ # no dice
+ exe_str = None
+
+ return exe_str
+
+ @cached_property
+ def gpg_string(self):
+ exe_str = spack.util.executable.which_string('gpg2', 'gpg')
+
+ no_gpg_msg = (
+ 'Spack requires gpg version >= 2\n'
+ ' To install a suitable version using Spack, run\n'
+ ' spack install gnupg@2:\n'
+ ' and load it by running\n'
+ ' spack load gnupg@2:')
+
+ if not exe_str:
+ raise SpackGPGError(no_gpg_msg)
+
+ exe = spack.util.executable.Executable(exe_str)
+ output = exe('--version', output=str)
+ match = re.search(_gnupg_version_re, output, re.M)
+
+ if not match:
+ raise SpackGPGError('Could not determine gpg version')
+
+ if spack.version.Version(match.group(2)) < self.target_version:
+ raise SpackGPGError(no_gpg_msg)
+
+ return exe_str
+
+ @cached_property
+ def user_run_dir(self):
+ # Try to ensure that (/var)/run/user/$(id -u) exists so that
+ # `gpgconf --create-socketdir` can be run later.
+ #
+ # NOTE(opadron): This action helps prevent a large class of
+ # "file-name-too-long" errors in gpg.
+
+ try:
+ has_suitable_gpgconf = bool(GpgConstants.gpgconf_string)
+ except SpackGPGError:
+ has_suitable_gpgconf = False
+
+ # If there is no suitable gpgconf, don't even bother trying to
+ # precreate a user run dir.
+ if not has_suitable_gpgconf:
+ return None
+
+ result = None
+ for var_run in ('/run', '/var/run'):
+ if not os.path.exists(var_run):
+ continue
+
+ var_run_user = os.path.join(var_run, 'user')
+ try:
+ if not os.path.exists(var_run_user):
+ os.mkdir(var_run_user)
+ os.chmod(var_run_user, 0o777)
+
+ user_dir = os.path.join(var_run_user, str(os.getuid()))
+
+ if not os.path.exists(user_dir):
+ os.mkdir(user_dir)
+ os.chmod(user_dir, 0o700)
+
+ # If the above operation fails due to lack of permissions, then
+ # just carry on without running gpgconf and hope for the best.
+ #
+ # NOTE(opadron): Without a dir in which to create a socket for IPC,
+ # gnupg may fail if GNUPGHOME is set to a path that
+ # is too long, where "too long" in this context is
+ # actually quite short; somewhere in the
+ # neighborhood of more than 100 characters.
+ #
+ # TODO(opadron): Maybe a warning should be printed in this case?
+ except OSError as exc:
+ if exc.errno not in (errno.EPERM, errno.EACCES):
+ raise
+ user_dir = None
+
+ # return the last iteration that provides a usable user run dir
+ if user_dir is not None:
+ result = user_dir
+
+ return result
+
+ def clear(self):
+ for attr in ('gpgconf_string', 'gpg_string', 'user_run_dir'):
+ try:
+ delattr(self, attr)
+ except AttributeError:
+ pass
+
+
+GpgConstants = GpgConstants()
+
+
+def ensure_gpg(reevaluate=False):
+ if reevaluate:
+ GpgConstants.clear()
+
+ if GpgConstants.user_run_dir is not None:
+ GpgConstants.gpgconf_string
+
+ GpgConstants.gpg_string
+ return True
+
+
+def has_gpg(*args, **kwargs):
+ try:
+ return ensure_gpg(*args, **kwargs)
+ except SpackGPGError:
+ return False
+
+
+# NOTE(opadron): When adding methods to this class, consider adding convenience
+# wrapper functions further down in this file.
class Gpg(object):
- _gpg = None
-
- @staticmethod
- def gpg():
- # TODO: Support loading up a GPG environment from a built gpg.
- if Gpg._gpg is None:
- gpg = which('gpg2', 'gpg')
-
- if not gpg:
- raise SpackGPGError("Spack requires gpg version 2 or higher.")
-
- # ensure that the version is actually >= 2 if we find 'gpg'
- if gpg.name == 'gpg':
- output = gpg('--version', output=str)
- match = re.search(_gnupg_version_re, output, re.M)
-
- if not match:
- raise SpackGPGError("Couldn't determine version of gpg")
-
- v = spack.version.Version(match.group(1))
- if v < spack.version.Version('2'):
- raise SpackGPGError("Spack requires GPG version >= 2")
-
- # make the GNU PG path if we need to
- # TODO: does this need to be in the spack directory?
- # we should probably just use GPG's regular conventions
- if not os.path.exists(GNUPGHOME):
- os.makedirs(GNUPGHOME)
- os.chmod(GNUPGHOME, 0o700)
- gpg.add_default_env('GNUPGHOME', GNUPGHOME)
-
- Gpg._gpg = gpg
- return Gpg._gpg
-
- @classmethod
- def create(cls, **kwargs):
+ def __init__(self, gnupg_home=None):
+ self.gnupg_home = get_gnupg_home(gnupg_home)
+
+ @cached_property
+ def prep(self):
+ # Make sure that suitable versions of gpgconf and gpg are available
+ ensure_gpg()
+
+ # Make sure that the GNUPGHOME exists
+ if not os.path.exists(self.gnupg_home):
+ os.makedirs(self.gnupg_home)
+ os.chmod(self.gnupg_home, 0o700)
+
+ if not os.path.isdir(self.gnupg_home):
+ raise SpackGPGError(
+ 'GNUPGHOME "{0}" exists and is not a directory'.format(
+ self.gnupg_home))
+
+ if GpgConstants.user_run_dir is not None:
+ self.gpgconf_exe('--create-socketdir')
+
+ return True
+
+ @cached_property
+ def gpgconf_exe(self):
+ exe = spack.util.executable.Executable(GpgConstants.gpgconf_string)
+ exe.add_default_env('GNUPGHOME', self.gnupg_home)
+ return exe
+
+ @cached_property
+ def gpg_exe(self):
+ exe = spack.util.executable.Executable(GpgConstants.gpg_string)
+ exe.add_default_env('GNUPGHOME', self.gnupg_home)
+ return exe
+
+ def __call__(self, *args, **kwargs):
+ if self.prep:
+ return self.gpg_exe(*args, **kwargs)
+
+ def create(self, **kwargs):
r, w = os.pipe()
r = os.fdopen(r, 'r')
w = os.fdopen(w, 'w')
@@ -83,64 +323,108 @@ class Gpg(object):
%%commit
''' % kwargs)
w.close()
- cls.gpg()('--gen-key', '--batch', input=r)
+ self('--gen-key', '--batch', input=r)
r.close()
- @classmethod
- def signing_keys(cls):
- output = cls.gpg()('--list-secret-keys', '--with-colons',
- '--fingerprint', '--fingerprint', output=str)
- return parse_keys_output(output)
-
- @classmethod
- def export_keys(cls, location, *keys):
- cls.gpg()('--armor', '--export', '--output', location, *keys)
-
- @classmethod
- def trust(cls, keyfile):
- cls.gpg()('--import', keyfile)
-
- @classmethod
- def untrust(cls, signing, *keys):
- args = [
- '--yes',
- '--batch',
- ]
+ def signing_keys(self, *args):
+ output = self('--list-secret-keys', '--with-colons', '--fingerprint',
+ *args, output=str)
+ return parse_secret_keys_output(output)
+
+ def public_keys(self, *args):
+ output = self('--list-public-keys', '--with-colons', '--fingerprint',
+ *args, output=str)
+ return parse_public_keys_output(output)
+
+ def export_keys(self, location, *keys):
+ self('--batch', '--yes',
+ '--armor', '--export',
+ '--output', location, *keys)
+
+ def trust(self, keyfile):
+ self('--import', keyfile)
+
+ def untrust(self, signing, *keys):
if signing:
- signing_args = args + ['--delete-secret-keys'] + list(keys)
- cls.gpg()(*signing_args)
- args.append('--delete-keys')
- args.extend(keys)
- cls.gpg()(*args)
-
- @classmethod
- def sign(cls, key, file, output, clearsign=False):
- args = [
- '--armor',
- '--default-key', key,
- '--output', output,
- file,
- ]
- if clearsign:
- args.insert(0, '--clearsign')
- else:
- args.insert(0, '--detach-sign')
- cls.gpg()(*args)
-
- @classmethod
- def verify(cls, signature, file, suppress_warnings=False):
- if suppress_warnings:
- cls.gpg()('--verify', signature, file, error=str)
- else:
- cls.gpg()('--verify', signature, file)
-
- @classmethod
- def list(cls, trusted, signing):
+ skeys = self.signing_keys(*keys)
+ self('--batch', '--yes', '--delete-secret-keys', *skeys)
+
+ pkeys = self.public_keys(*keys)
+ self('--batch', '--yes', '--delete-keys', *pkeys)
+
+ def sign(self, key, file, output, clearsign=False):
+ self(('--clearsign' if clearsign else '--detach-sign'),
+ '--armor', '--default-key', key,
+ '--output', output, file)
+
+ def verify(self, signature, file, suppress_warnings=False):
+ self('--verify', signature, file,
+ **({'error': str} if suppress_warnings else {}))
+
+ def list(self, trusted, signing):
if trusted:
- cls.gpg()('--list-public-keys')
+ self('--list-public-keys')
+
if signing:
- cls.gpg()('--list-secret-keys')
+ self('--list-secret-keys')
class SpackGPGError(spack.error.SpackError):
"""Class raised when GPG errors are detected."""
+
+
+# Convenience wrappers for methods of the Gpg class
+
+# __call__ is a bit of a special case, since the Gpg instance is, itself, the
+# "thing" that is being called.
+@functools.wraps(Gpg.__call__)
+def gpg(*args, **kwargs):
+ return get_global_gpg_instance()(*args, **kwargs)
+
+
+gpg.name = 'gpg'
+
+
+@functools.wraps(Gpg.create)
+def create(*args, **kwargs):
+ return get_global_gpg_instance().create(*args, **kwargs)
+
+
+@functools.wraps(Gpg.signing_keys)
+def signing_keys(*args, **kwargs):
+ return get_global_gpg_instance().signing_keys(*args, **kwargs)
+
+
+@functools.wraps(Gpg.public_keys)
+def public_keys(*args, **kwargs):
+ return get_global_gpg_instance().public_keys(*args, **kwargs)
+
+
+@functools.wraps(Gpg.export_keys)
+def export_keys(*args, **kwargs):
+ return get_global_gpg_instance().export_keys(*args, **kwargs)
+
+
+@functools.wraps(Gpg.trust)
+def trust(*args, **kwargs):
+ return get_global_gpg_instance().trust(*args, **kwargs)
+
+
+@functools.wraps(Gpg.untrust)
+def untrust(*args, **kwargs):
+ return get_global_gpg_instance().untrust(*args, **kwargs)
+
+
+@functools.wraps(Gpg.sign)
+def sign(*args, **kwargs):
+ return get_global_gpg_instance().sign(*args, **kwargs)
+
+
+@functools.wraps(Gpg.verify)
+def verify(*args, **kwargs):
+ return get_global_gpg_instance().verify(*args, **kwargs)
+
+
+@functools.wraps(Gpg.list)
+def list(*args, **kwargs):
+ return get_global_gpg_instance().list(*args, **kwargs)
diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py
index 3f71dd1f71..1aad85550a 100644
--- a/lib/spack/spack/util/web.py
+++ b/lib/spack/spack/util/web.py
@@ -224,7 +224,7 @@ def url_exists(url):
try:
read_from_url(url)
return True
- except URLError:
+ except (SpackWebError, URLError):
return False
@@ -295,15 +295,27 @@ def _iter_s3_prefix(client, url, num_entries=1024):
break
-def list_url(url):
+def _iter_local_prefix(path):
+ for root, _, files in os.walk(path):
+ for f in files:
+ yield os.path.relpath(os.path.join(root, f), path)
+
+
+def list_url(url, recursive=False):
url = url_util.parse(url)
local_path = url_util.local_file_path(url)
if local_path:
- return os.listdir(local_path)
+ if recursive:
+ return list(_iter_local_prefix(local_path))
+ return [subpath for subpath in os.listdir(local_path)
+ if os.path.isfile(os.path.join(local_path, subpath))]
if url.scheme == 's3':
s3 = s3_util.create_s3_session(url)
+ if recursive:
+ return list(_iter_s3_prefix(s3, url))
+
return list(set(
key.split('/', 1)[0]
for key in _iter_s3_prefix(s3, url)))
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-29 07:05:32 +0000