diff options
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 214 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/gpg.py | 68 | ||||
| -rw-r--r-- | lib/spack/spack/test/bindist.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/test/ci.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/gpg.py | 36 | ||||
| -rw-r--r-- | lib/spack/spack/test/conftest.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/test/packaging.py | 25 | ||||
| -rw-r--r-- | lib/spack/spack/test/stage.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/test/util/util_gpg.py | 32 | ||||
| -rw-r--r-- | lib/spack/spack/test/web.py | 28 | ||||
| -rw-r--r-- | lib/spack/spack/util/gpg.py | 468 | ||||
| -rw-r--r-- | lib/spack/spack/util/web.py | 18 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 11 |
14 files changed, 746 insertions, 226 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 590f6e6710..3e70ddf792 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -26,17 +26,18 @@ import spack.cmd import spack.config as config import spack.database as spack_db import spack.fetch_strategy as fs -import spack.util.gpg import spack.relocate as relocate +import spack.util.gpg +import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.mirror import spack.util.url as url_util import spack.util.web as web_util from spack.spec import Spec from spack.stage import Stage -from spack.util.gpg import Gpg _build_cache_relative_path = 'build_cache' +_build_cache_keys_relative_path = '_pgp' BUILD_CACHE_INDEX_TEMPLATE = ''' <html> @@ -247,15 +248,9 @@ def checksum_tarball(file): return hasher.hexdigest() -def sign_tarball(key, force, specfile_path): - # Sign the packages if keys available - if spack.util.gpg.Gpg.gpg() is None: - raise NoGpgException( - "gpg2 is not available in $PATH .\n" - "Use spack install gnupg and spack load gnupg.") - +def select_signing_key(key=None): if key is None: - keys = Gpg.signing_keys() + keys = spack.util.gpg.signing_keys() if len(keys) == 1: key = keys[0] @@ -263,26 +258,30 @@ def sign_tarball(key, force, specfile_path): raise PickKeyException(str(keys)) if len(keys) == 0: - msg = "No default key available for signing.\n" - msg += "Use spack gpg init and spack gpg create" - msg += " to create a default key." - raise NoKeyException(msg) + raise NoKeyException( + "No default key available for signing.\n" + "Use spack gpg init and spack gpg create" + " to create a default key.") + return key + +def sign_tarball(key, force, specfile_path): if os.path.exists('%s.asc' % specfile_path): if force: os.remove('%s.asc' % specfile_path) else: raise NoOverwriteException('%s.asc' % specfile_path) - Gpg.sign(key, specfile_path, '%s.asc' % specfile_path) + key = select_signing_key(key) + spack.util.gpg.sign(key, specfile_path, '%s.asc' % specfile_path) def generate_package_index(cache_prefix): """Create the build cache index page. Creates (or replaces) the "index.json" page at the location given in - cache_prefix. This page contains a link for each binary package (*.yaml) - and public key (*.key) under cache_prefix. + cache_prefix. This page contains a link for each binary package (.yaml) + under cache_prefix. """ tmpdir = tempfile.mkdtemp() db_root_dir = os.path.join(tmpdir, 'db_root') @@ -325,6 +324,45 @@ def generate_package_index(cache_prefix): shutil.rmtree(tmpdir) +def generate_key_index(key_prefix, tmpdir=None): + """Create the key index page. + + Creates (or replaces) the "index.json" page at the location given in + key_prefix. This page contains an entry for each key (.pub) under + key_prefix. + """ + + tty.debug(' '.join(('Retrieving key.pub files from', + url_util.format(key_prefix), + 'to build key index'))) + + fingerprints = ( + entry[:-4] + for entry in web_util.list_url(key_prefix, recursive=False) + if entry.endswith('.pub')) + + keys_local = url_util.local_file_path(key_prefix) + if keys_local: + target = os.path.join(keys_local, 'index.json') + else: + target = os.path.join(tmpdir, 'index.json') + + index = { + 'keys': dict( + (fingerprint, {}) for fingerprint + in sorted(set(fingerprints))) + } + with open(target, 'w') as f: + sjson.dump(index, f) + + if not keys_local: + web_util.push_to_url( + target, + url_util.join(key_prefix, 'index.json'), + keep_original=False, + extra_args={'ContentType': 'application/json'}) + + def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, allow_root=False, key=None, regenerate_index=False): """ @@ -445,7 +483,9 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, # sign the tarball and spec file with gpg if not unsigned: + key = select_signing_key(key) sign_tarball(key, force, specfile_path) + # put tarball, spec and signature files in .spack archive with closing(tarfile.open(spackfile_path, 'w')) as tar: tar.add(name=tarfile_path, arcname='%s' % tarfile_name) @@ -468,7 +508,15 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, .format(spec, remote_spackfile_path)) try: - # create an index.html for the build_cache directory so specs can be + # push the key to the build cache's _pgp directory so it can be + # imported + if not unsigned: + push_keys(outdir, + keys=[key], + regenerate_index=regenerate_index, + tmpdir=tmpdir) + + # create an index.json for the build_cache directory so specs can be # found if regenerate_index: generate_package_index(url_util.join( @@ -695,7 +743,8 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, if os.path.exists('%s.asc' % specfile_path): try: suppress = config.get('config:suppress_gpg_warnings', False) - Gpg.verify('%s.asc' % specfile_path, specfile_path, suppress) + spack.util.gpg.verify( + '%s.asc' % specfile_path, specfile_path, suppress) except Exception as e: shutil.rmtree(tmpdir) raise e @@ -898,41 +947,46 @@ def get_specs(): return _cached_specs -def get_keys(install=False, trust=False, force=False): - """ - Get pgp public keys available on mirror - with suffix .key or .pub +def get_keys(install=False, trust=False, force=False, mirrors=None): + """Get pgp public keys available on mirror with suffix .pub """ - if not spack.mirror.MirrorCollection(): + mirror_collection = (mirrors or spack.mirror.MirrorCollection()) + + if not mirror_collection: tty.die("Please add a spack mirror to allow " + "download of build caches.") - keys = set() + for mirror in mirror_collection.values(): + fetch_url = mirror.fetch_url + keys_url = url_util.join(fetch_url, + _build_cache_relative_path, + _build_cache_keys_relative_path) + keys_index = url_util.join(keys_url, 'index.json') - for mirror in spack.mirror.MirrorCollection().values(): - fetch_url_build_cache = url_util.join( - mirror.fetch_url, _build_cache_relative_path) + tty.debug('Finding public keys in {0}'.format( + url_util.format(fetch_url))) - mirror_dir = url_util.local_file_path(fetch_url_build_cache) - if mirror_dir: - tty.debug('Finding public keys in {0}'.format(mirror_dir)) - files = os.listdir(str(mirror_dir)) - for file in files: - if re.search(r'\.key', file) or re.search(r'\.pub', file): - link = url_util.join(fetch_url_build_cache, file) - keys.add(link) - else: - tty.debug('Finding public keys at {0}' - .format(url_util.format(fetch_url_build_cache))) - # For s3 mirror need to request index.html directly - p, links = web_util.spider( - url_util.join(fetch_url_build_cache, 'index.html')) + try: + _, _, json_file = web_util.read_from_url(keys_index) + json_index = sjson.load(codecs.getreader('utf-8')(json_file)) + except (URLError, web_util.SpackWebError) as url_err: + if web_util.url_exists(keys_index): + err_msg = [ + 'Unable to find public keys in {0},', + ' caught exception attempting to read from {1}.', + ] - for link in links: - if re.search(r'\.key', link) or re.search(r'\.pub', link): - keys.add(link) + tty.error(''.join(err_msg).format( + url_util.format(fetch_url), + url_util.format(keys_index))) + + tty.debug(url_err) + + continue + + for fingerprint, key_attributes in json_index['keys'].items(): + link = os.path.join(keys_url, fingerprint + '.pub') - for link in keys: with Stage(link, name="build_cache", keep=True) as stage: if os.path.exists(stage.save_filename) and force: os.remove(stage.save_filename) @@ -941,16 +995,80 @@ def get_keys(install=False, trust=False, force=False): stage.fetch() except fs.FetchError: continue - tty.debug('Found key {0}'.format(link)) + + tty.debug('Found key {0}'.format(fingerprint)) if install: if trust: - Gpg.trust(stage.save_filename) + spack.util.gpg.trust(stage.save_filename) tty.debug('Added this key to trusted keys.') else: tty.debug('Will not add this key to trusted keys.' 'Use -t to install all downloaded keys') +def push_keys(*mirrors, **kwargs): + """ + Upload pgp public keys to the given mirrors + """ + keys = kwargs.get('keys') + regenerate_index = kwargs.get('regenerate_index', False) + tmpdir = kwargs.get('tmpdir') + remove_tmpdir = False + + keys = spack.util.gpg.public_keys(*(keys or [])) + + try: + for mirror in mirrors: + push_url = getattr(mirror, 'push_url', mirror) + keys_url = url_util.join(push_url, + _build_cache_relative_path, + _build_cache_keys_relative_path) + keys_local = url_util.local_file_path(keys_url) + + verb = 'Writing' if keys_local else 'Uploading' + tty.debug('{0} public keys to {1}'.format( + verb, url_util.format(push_url))) + + if keys_local: # mirror is local, don't bother with the tmpdir + prefix = keys_local + mkdirp(keys_local) + else: + # A tmp dir is created for the first mirror that is non-local. + # On the off-hand chance that all the mirrors are local, then + # we can avoid the need to create a tmp dir. + if tmpdir is None: + tmpdir = tempfile.mkdtemp() + remove_tmpdir = True + prefix = tmpdir + + for fingerprint in keys: + tty.debug(' ' + fingerprint) + filename = fingerprint + '.pub' + + export_target = os.path.join(prefix, filename) + spack.util.gpg.export_keys(export_target, fingerprint) + + # If mirror is local, the above export writes directly to the + # mirror (export_target points directly to the mirror). + # + # If not, then export_target is a tmpfile that needs to be + # uploaded to the mirror. + if not keys_local: + spack.util.web.push_to_url( + export_target, + url_util.join(keys_url, filename), + keep_original=False) + + if regenerate_index: + if keys_local: + generate_key_index(keys_url) + else: + generate_key_index(keys_url, tmpdir) + finally: + if remove_tmpdir: + shutil.rmtree(tmpdir) + + def needs_rebuild(spec, mirror_url, rebuild_on_errors=False): if not spec.concrete: raise ValueError('spec must be concrete to check against mirror') diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index 0a77812c12..59fba4a4df 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -6,9 +6,10 @@ import os import argparse +import spack.binary_distribution import spack.cmd.common.arguments as arguments import spack.paths -from spack.util.gpg import Gpg +import spack.util.gpg description = "handle GPG actions for spack" section = "packaging" @@ -81,37 +82,64 @@ def setup_parser(subparser): 'all secret keys if unspecified') export.set_defaults(func=gpg_export) + publish = subparsers.add_parser('publish', help=gpg_publish.__doc__) + + output = publish.add_mutually_exclusive_group(required=True) + output.add_argument('-d', '--directory', + metavar='directory', + type=str, + help="local directory where " + + "keys will be published.") + output.add_argument('-m', '--mirror-name', + metavar='mirror-name', + type=str, + help="name of the mirror where " + + "keys will be published.") + output.add_argument('--mirror-url', + metavar='mirror-url', + type=str, + help="URL of the mirror where " + + "keys will be published.") + publish.add_argument('--rebuild-index', action='store_true', + default=False, help=( + "Regenerate buildcache key index " + "after publishing key(s)")) + publish.add_argument('keys', nargs='*', + help='the keys to publish; ' + 'all public keys if unspecified') + publish.set_defaults(func=gpg_publish) + def gpg_create(args): """create a new key""" if args.export: - old_sec_keys = Gpg.signing_keys() - Gpg.create(name=args.name, email=args.email, - comment=args.comment, expires=args.expires) + old_sec_keys = spack.util.gpg.signing_keys() + spack.util.gpg.create(name=args.name, email=args.email, + comment=args.comment, expires=args.expires) if args.export: - new_sec_keys = set(Gpg.signing_keys()) + new_sec_keys = set(spack.util.gpg.signing_keys()) new_keys = new_sec_keys.difference(old_sec_keys) - Gpg.export_keys(args.export, *new_keys) + spack.util.gpg.export_keys(args.export, *new_keys) def gpg_export(args): """export a secret key""" keys = args.keys if not keys: - keys = Gpg.signing_keys() - Gpg.export_keys(args.location, *keys) + keys = spack.util.gpg.signing_keys() + spack.util.gpg.export_keys(args.location, *keys) def gpg_list(args): """list keys available in the keyring""" - Gpg.list(args.trusted, args.signing) + spack.util.gpg.list(args.trusted, args.signing) def gpg_sign(args): """sign a package""" key = args.key if key is None: - keys = Gpg.signing_keys() + keys = spack.util.gpg.signing_keys() if len(keys) == 1: key = keys[0] elif not keys: @@ -123,12 +151,12 @@ def gpg_sign(args): if not output: output = args.spec[0] + '.asc' # TODO: Support the package format Spack creates. - Gpg.sign(key, ' '.join(args.spec), output, args.clearsign) + spack.util.gpg.sign(key, ' '.join(args.spec), output, args.clearsign) def gpg_trust(args): """add a key to the keyring""" - Gpg.trust(args.keyfile) + spack.util.gpg.trust(args.keyfile) def gpg_init(args): @@ -141,12 +169,12 @@ def gpg_init(args): for filename in filenames: if not filename.endswith('.key'): continue - Gpg.trust(os.path.join(root, filename)) + spack.util.gpg.trust(os.path.join(root, filename)) def gpg_untrust(args): """remove a key from the keyring""" - Gpg.untrust(args.signing, *args.keys) + spack.util.gpg.untrust(args.signing, *args.keys) def gpg_verify(args): @@ -155,7 +183,17 @@ def gpg_verify(args): signature = args.signature if signature is None: signature = args.spec[0] + '.asc' - Gpg.verify(signature, ' '.join(args.spec)) + spack.util.gpg.verify(signature, ' '.join(args.spec)) + + +def gpg_publish(args): + """publish public keys to a build cache""" + + # TODO(opadron): switch to using the mirror args once #17547 is merged + mirror = args.directory + + spack.binary_distribution.push_keys( + mirror, keys=args.keys, regenerate_index=args.rebuild_index) def gpg(parser, args): diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py index a7995ca43b..9b2f1b1408 100644 --- a/lib/spack/spack/test/bindist.py +++ b/lib/spack/spack/test/bindist.py @@ -19,8 +19,10 @@ import spack.cmd.buildcache as buildcache import spack.cmd.install as install import spack.cmd.uninstall as uninstall import spack.cmd.mirror as mirror -from spack.spec import Spec +import spack.mirror +import spack.util.gpg from spack.directory_layout import YamlDirectoryLayout +from spack.spec import Spec def_install_path_scheme = '${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}' # noqa: E501 @@ -469,3 +471,40 @@ def test_relative_rpaths_install_nondefault(tmpdir, margs = mparser.parse_args( ['rm', '--scope', 'site', 'test-mirror-rel']) mirror.mirror(mparser, margs) + + +def test_push_and_fetch_keys(mock_gnupghome): + testpath = str(mock_gnupghome) + + mirror = os.path.join(testpath, 'mirror') + mirrors = {'test-mirror': mirror} + mirrors = spack.mirror.MirrorCollection(mirrors) + mirror = spack.mirror.Mirror('file://' + mirror) + + gpg_dir1 = os.path.join(testpath, 'gpg1') + gpg_dir2 = os.path.join(testpath, 'gpg2') + + # dir 1: create a new key, record its fingerprint, and push it to a new + # mirror + with spack.util.gpg.gnupg_home_override(gpg_dir1): + spack.util.gpg.create(name='test-key', + email='fake@test.key', + expires='0', + comment=None) + + keys = spack.util.gpg.public_keys() + assert len(keys) == 1 + fpr = keys[0] + + bindist.push_keys(mirror, keys=[fpr], regenerate_index=True) + + # dir 2: import the key from the mirror, and confirm that its fingerprint + # matches the one created above + with spack.util.gpg.gnupg_home_override(gpg_dir2): + assert len(spack.util.gpg.public_keys()) == 0 + + bindist.get_keys(mirrors=mirrors, install=True, trust=True, force=True) + + new_keys = spack.util.gpg.public_keys() + assert len(new_keys) == 1 + assert new_keys[0] == fpr diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index a2ad2708a1..f1986b34d0 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -53,15 +53,8 @@ def test_urlencode_string(): assert(s_enc == 'Spack+Test+Project') -def has_gpg(): - try: - gpg = spack.util.gpg.Gpg.gpg() - except spack.util.gpg.SpackGPGError: - gpg = None - return bool(gpg) - - -@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg') +@pytest.mark.skipif(not spack.util.gpg.has_gpg(), + reason='This test requires gpg') def test_import_signing_key(mock_gnupghome): signing_key_dir = spack_paths.mock_gpg_keys_path signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 82f7fd86cc..abb9f5c5ad 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -35,14 +35,6 @@ buildcache_cmd = spack.main.SpackCommand('buildcache') git = exe.which('git', required=True) -def has_gpg(): - try: - gpg = spack.util.gpg.Gpg.gpg() - except spack.util.gpg.SpackGPGError: - gpg = None - return bool(gpg) - - @pytest.fixture() def env_deactivate(): yield @@ -690,7 +682,8 @@ spack: @pytest.mark.disable_clean_stage_check -@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg') +@pytest.mark.skipif(not spack.util.gpg.has_gpg(), + reason='This test requires gpg') def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate, install_mockery, mock_packages, mock_fetch, mock_stage, mock_gnupghome): diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index 56d178f449..751790182b 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -29,39 +29,35 @@ gpg = SpackCommand('gpg') ('gpg2', 'gpg (GnuPG) 2.2.19'), # gpg2 command ]) def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch): + TEMPLATE = ('#!/bin/sh\n' + 'echo "{version}"\n') + with tmpdir.as_cwd(): - with open(cmd_name, 'w') as f: - f.write("""\ -#!/bin/sh -echo "{version}" -""".format(version=version)) - fs.set_executable(cmd_name) + for fname in (cmd_name, 'gpgconf'): + with open(fname, 'w') as f: + f.write(TEMPLATE.format(version=version)) + fs.set_executable(fname) monkeypatch.setitem(os.environ, "PATH", str(tmpdir)) if version == 'undetectable' or version.endswith('1.3.4'): with pytest.raises(spack.util.gpg.SpackGPGError): - exe = spack.util.gpg.Gpg.gpg() + spack.util.gpg.ensure_gpg(reevaluate=True) else: - exe = spack.util.gpg.Gpg.gpg() - assert isinstance(exe, spack.util.executable.Executable) + spack.util.gpg.ensure_gpg(reevaluate=True) + gpg_exe = spack.util.gpg.get_global_gpg_instance().gpg_exe + assert isinstance(gpg_exe, spack.util.executable.Executable) + gpgconf_exe = spack.util.gpg.get_global_gpg_instance().gpgconf_exe + assert isinstance(gpgconf_exe, spack.util.executable.Executable) def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch): monkeypatch.setitem(os.environ, "PATH", str(tmpdir)) with pytest.raises(spack.util.gpg.SpackGPGError): - spack.util.gpg.Gpg.gpg() - - -def has_gpg(): - try: - gpg = spack.util.gpg.Gpg.gpg() - except spack.util.gpg.SpackGPGError: - gpg = None - return bool(gpg) + spack.util.gpg.ensure_gpg(reevaluate=True) @pytest.mark.maybeslow -@pytest.mark.skipif(not has_gpg(), +@pytest.mark.skipif(not spack.util.gpg.has_gpg(), reason='These tests require gnupg2') def test_gpg(tmpdir, mock_gnupghome): # Verify a file with an empty keyring. @@ -103,7 +99,7 @@ def test_gpg(tmpdir, mock_gnupghome): '--export', str(keypath), 'Spack testing 1', 'spack@googlegroups.com') - keyfp = spack.util.gpg.Gpg.signing_keys()[0] + keyfp = spack.util.gpg.signing_keys()[0] # List the keys. # TODO: Test the output here. diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index a8cdce90ee..152c2c846e 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -781,10 +781,8 @@ def mock_gnupghome(monkeypatch): # This comes up because tmp paths on macOS are already long-ish, and # pytest makes them longer. short_name_tmpdir = tempfile.mkdtemp() - monkeypatch.setattr(spack.util.gpg, 'GNUPGHOME', short_name_tmpdir) - monkeypatch.setattr(spack.util.gpg.Gpg, '_gpg', None) - - yield + with spack.util.gpg.gnupg_home_override(short_name_tmpdir): + yield short_name_tmpdir # clean up, since we are doing this manually shutil.rmtree(short_name_tmpdir) diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index bee0d01b16..e2cef22577 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -20,6 +20,7 @@ import spack.repo import spack.store import spack.binary_distribution as bindist import spack.cmd.buildcache as buildcache +import spack.util.gpg from spack.spec import Spec from spack.paths import mock_gpg_keys_path from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite @@ -31,14 +32,6 @@ from spack.relocate import _placeholder, macho_find_paths from spack.relocate import file_is_relocatable -def has_gpg(): - try: - gpg = spack.util.gpg.Gpg.gpg() - except spack.util.gpg.SpackGPGError: - gpg = None - return bool(gpg) - - def fake_fetchify(url, pkg): """Fake the URL for a package so it downloads from a file.""" fetcher = FetchStrategyComposite() @@ -46,7 +39,8 @@ def fake_fetchify(url, pkg): pkg.fetcher = fetcher -@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg') +@pytest.mark.skipif(not spack.util.gpg.has_gpg(), + reason='This test requires gpg') @pytest.mark.usefixtures('install_mockery', 'mock_gnupghome') def test_buildcache(mock_archive, tmpdir): # tweak patchelf to only do a download @@ -101,12 +95,9 @@ echo $PATH""" create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash] # Create a private key to sign package with if gpg2 available - if spack.util.gpg.Gpg.gpg(): - spack.util.gpg.Gpg.create(name='test key 1', expires='0', - email='spack@googlegroups.com', - comment='Spack test key') - else: - create_args.insert(create_args.index('-a'), '-u') + spack.util.gpg.create(name='test key 1', expires='0', + email='spack@googlegroups.com', + comment='Spack test key') create_args.insert(create_args.index('-a'), '--rebuild-index') @@ -119,8 +110,6 @@ echo $PATH""" pkg.do_uninstall(force=True) install_args = ['install', '-a', '-f', pkghash] - if not spack.util.gpg.Gpg.gpg(): - install_args.insert(install_args.index('-a'), '-u') args = parser.parse_args(install_args) # Test install buildcache.buildcache(parser, args) @@ -144,8 +133,6 @@ echo $PATH""" # Uninstall the package pkg.do_uninstall(force=True) - if not spack.util.gpg.Gpg.gpg(): - install_args.insert(install_args.index('-a'), '-u') args = parser.parse_args(install_args) buildcache.buildcache(parser, args) diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 204a6e6a3e..9dea091730 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -839,7 +839,8 @@ class TestStage(object): assert 'spack' in path.split(os.path.sep) # Make sure cached stage path value was changed appropriately - assert spack.stage._stage_root == test_path + assert spack.stage._stage_root in ( + test_path, os.path.join(test_path, getpass.getuser())) # Make sure the directory exists assert os.path.isdir(spack.stage._stage_root) diff --git a/lib/spack/spack/test/util/util_gpg.py b/lib/spack/spack/test/util/util_gpg.py index 4a243985db..83fad508f9 100644 --- a/lib/spack/spack/test/util/util_gpg.py +++ b/lib/spack/spack/test/util/util_gpg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import spack.util.gpg as gpg +import os +import pytest + +import spack.util.gpg def test_parse_gpg_output_case_one(): @@ -17,7 +20,7 @@ fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY: uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>: ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:::::::::: """ - keys = gpg.parse_keys_output(output) + keys = spack.util.gpg.parse_secret_keys_output(output) assert len(keys) == 2 assert keys[0] == 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' @@ -34,7 +37,7 @@ ssb:-:2048:1:AAAAAAAAA::::::esa:::+:::23: fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY: grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: """ - keys = gpg.parse_keys_output(output) + keys = spack.util.gpg.parse_secret_keys_output(output) assert len(keys) == 1 assert keys[0] == 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' @@ -53,8 +56,29 @@ uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>: ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:::::::::: fpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:""" - keys = gpg.parse_keys_output(output) + keys = spack.util.gpg.parse_secret_keys_output(output) assert len(keys) == 2 assert keys[0] == 'WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW' assert keys[1] == 'YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY' + + +@pytest.mark.skipif(not spack.util.gpg.GpgConstants.user_run_dir, + reason='This test requires /var/run/user/$(id -u)') +def test_really_long_gnupg_home_dir(tmpdir): + N = 960 + + tdir = str(tmpdir) + while len(tdir) < N: + tdir = os.path.join(tdir, 'filler') + + tdir = tdir[:N].rstrip(os.sep) + tdir += '0' * (N - len(tdir)) + + with spack.util.gpg.gnupg_home_override(tdir): + spack.util.gpg.create(name='Spack testing 1', + email='test@spack.io', + comment='Spack testing key', + expires='0') + + spack.util.gpg.list(True, True) diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index dfca41c95b..472ab0dab7 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -167,3 +167,31 @@ def test_get_header(): # If there isn't even a fuzzy match, raise KeyError with pytest.raises(KeyError): spack.util.web.get_header(headers, 'ContentLength') + + +def test_list_url(tmpdir): + testpath = str(tmpdir) + + os.mkdir(os.path.join(testpath, 'dir')) + + with open(os.path.join(testpath, 'file-0.txt'), 'w'): + pass + with open(os.path.join(testpath, 'file-1.txt'), 'w'): + pass + with open(os.path.join(testpath, 'file-2.txt'), 'w'): + pass + + with open(os.path.join(testpath, 'dir', 'another-file.txt'), 'w'): + pass + + list_url = lambda recursive: list(sorted( + spack.util.web.list_url(testpath, recursive=recursive))) + + assert list_url(False) == ['file-0.txt', + 'file-1.txt', + 'file-2.txt'] + + assert list_url(True) == ['dir/another-file.txt', + 'file-0.txt', + 'file-1.txt', + 'file-2.txt'] diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index e6b7f56741..eaa199e417 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -3,20 +3,73 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import contextlib +import errno +import functools import os import re +import llnl.util.lang + import spack.error import spack.paths +import spack.util.executable import spack.version -from spack.util.executable import which -_gnupg_version_re = r"^gpg \(GnuPG\) (.*)$" -GNUPGHOME = os.getenv('SPACK_GNUPGHOME', spack.paths.gpg_path) +_gnupg_version_re = r"^gpg(conf)? \(GnuPG\) (.*)$" +_gnupg_home_override = None +_global_gpg_instance = None + + +def get_gnupg_home(gnupg_home=None): + """Returns the directory that should be used as the GNUPGHOME environment + variable when calling gpg. + + If a [gnupg_home] is passed directly (and not None), that value will be + used. + + Otherwise, if there is an override set (and it is not None), then that + value will be used. + + Otherwise, if the environment variable "SPACK_GNUPGHOME" is set, then that + value will be used. + + Otherwise, the default gpg path for Spack will be used. + + See also: gnupg_home_override() + """ + return (gnupg_home or + _gnupg_home_override or + os.getenv('SPACK_GNUPGHOME') or + spack.paths.gpg_path) + + +@contextlib.contextmanager +def gnupg_home_override(new_gnupg_home): + global _gnupg_home_override + global _global_gpg_instance + + old_gnupg_home_override = _gnupg_home_override + old_global_gpg_instance = _global_gpg_instance + + _gnupg_home_override = new_gnupg_home + _global_gpg_instance = None + + yield + + _gnupg_home_override = old_gnupg_home_override + _global_gpg_instance = old_global_gpg_instance -def parse_keys_output(output): +def get_global_gpg_instance(): + global _global_gpg_instance + if _global_gpg_instance is None: + _global_gpg_instance = Gpg() + return _global_gpg_instance + + +def parse_secret_keys_output(output): keys = [] found_sec = False for line in output.split('\n'): @@ -31,43 +84,230 @@ def parse_keys_output(output): return keys +def parse_public_keys_output(output): + keys = [] + found_pub = False + for line in output.split('\n'): + if found_pub: + if line.startswith('fpr'): + keys.append(line.split(':')[9]) + found_pub = False + elif line.startswith('ssb'): + found_pub = False + elif line.startswith('pub'): + found_pub = True + return keys + + +cached_property = getattr(functools, 'cached_property', None) + +# If older python version has no cached_property, emulate it here. +# TODO(opadron): maybe this shim should be moved to llnl.util.lang? +if not cached_property: + def cached_property(*args, **kwargs): + result = property(llnl.util.lang.memoized(*args, **kwargs)) + attr = result.fget.__name__ + + @result.deleter + def result(self): + getattr(type(self), attr).fget.cache.pop((self,), None) + + return result + + +class GpgConstants(object): + @cached_property + def target_version(self): + return spack.version.Version('2') + + @cached_property + def gpgconf_string(self): + exe_str = spack.util.executable.which_string( + 'gpgconf', 'gpg2conf', 'gpgconf2') + + no_gpgconf_msg = ( + 'Spack requires gpgconf version >= 2\n' + ' To install a suitable version using Spack, run\n' + ' spack install gnupg@2:\n' + ' and load it by running\n' + ' spack load gnupg@2:') + + if not exe_str: + raise SpackGPGError(no_gpgconf_msg) + + exe = spack.util.executable.Executable(exe_str) + output = exe('--version', output=str) + match = re.search(_gnupg_version_re, output, re.M) + + if not match: + raise SpackGPGError('Could not determine gpgconf version') + + if spack.version.Version(match.group(2)) < self.target_version: + raise SpackGPGError(no_gpgconf_msg) + + # ensure that the gpgconf we found can run "gpgconf --create-socketdir" + try: + exe('--dry-run', '--create-socketdir') + except spack.util.executable.ProcessError: + # no dice + exe_str = None + + return exe_str + + @cached_property + def gpg_string(self): + exe_str = spack.util.executable.which_string('gpg2', 'gpg') + + no_gpg_msg = ( + 'Spack requires gpg version >= 2\n' + ' To install a suitable version using Spack, run\n' + ' spack install gnupg@2:\n' + ' and load it by running\n' + ' spack load gnupg@2:') + + if not exe_str: + raise SpackGPGError(no_gpg_msg) + + exe = spack.util.executable.Executable(exe_str) + output = exe('--version', output=str) + match = re.search(_gnupg_version_re, output, re.M) + + if not match: + raise SpackGPGError('Could not determine gpg version') + + if spack.version.Version(match.group(2)) < self.target_version: + raise SpackGPGError(no_gpg_msg) + + return exe_str + + @cached_property + def user_run_dir(self): + # Try to ensure that (/var)/run/user/$(id -u) exists so that + # `gpgconf --create-socketdir` can be run later. + # + # NOTE(opadron): This action helps prevent a large class of + # "file-name-too-long" errors in gpg. + + try: + has_suitable_gpgconf = bool(GpgConstants.gpgconf_string) + except SpackGPGError: + has_suitable_gpgconf = False + + # If there is no suitable gpgconf, don't even bother trying to + # precreate a user run dir. + if not has_suitable_gpgconf: + return None + + result = None + for var_run in ('/run', '/var/run'): + if not os.path.exists(var_run): + continue + + var_run_user = os.path.join(var_run, 'user') + try: + if not os.path.exists(var_run_user): + os.mkdir(var_run_user) + os.chmod(var_run_user, 0o777) + + user_dir = os.path.join(var_run_user, str(os.getuid())) + + if not os.path.exists(user_dir): + os.mkdir(user_dir) + os.chmod(user_dir, 0o700) + + # If the above operation fails due to lack of permissions, then + # just carry on without running gpgconf and hope for the best. + # + # NOTE(opadron): Without a dir in which to create a socket for IPC, + # gnupg may fail if GNUPGHOME is set to a path that + # is too long, where "too long" in this context is + # actually quite short; somewhere in the + # neighborhood of more than 100 characters. + # + # TODO(opadron): Maybe a warning should be printed in this case? + except OSError as exc: + if exc.errno not in (errno.EPERM, errno.EACCES): + raise + user_dir = None + + # return the last iteration that provides a usable user run dir + if user_dir is not None: + result = user_dir + + return result + + def clear(self): + for attr in ('gpgconf_string', 'gpg_string', 'user_run_dir'): + try: + delattr(self, attr) + except AttributeError: + pass + + +GpgConstants = GpgConstants() + + +def ensure_gpg(reevaluate=False): + if reevaluate: + GpgConstants.clear() + + if GpgConstants.user_run_dir is not None: + GpgConstants.gpgconf_string + + GpgConstants.gpg_string + return True + + +def has_gpg(*args, **kwargs): + try: + return ensure_gpg(*args, **kwargs) + except SpackGPGError: + return False + + +# NOTE(opadron): When adding methods to this class, consider adding convenience +# wrapper functions further down in this file. class Gpg(object): - _gpg = None - - @staticmethod - def gpg(): - # TODO: Support loading up a GPG environment from a built gpg. - if Gpg._gpg is None: - gpg = which('gpg2', 'gpg') - - if not gpg: - raise SpackGPGError("Spack requires gpg version 2 or higher.") - - # ensure that the version is actually >= 2 if we find 'gpg' - if gpg.name == 'gpg': - output = gpg('--version', output=str) - match = re.search(_gnupg_version_re, output, re.M) - - if not match: - raise SpackGPGError("Couldn't determine version of gpg") - - v = spack.version.Version(match.group(1)) - if v < spack.version.Version('2'): - raise SpackGPGError("Spack requires GPG version >= 2") - - # make the GNU PG path if we need to - # TODO: does this need to be in the spack directory? - # we should probably just use GPG's regular conventions - if not os.path.exists(GNUPGHOME): - os.makedirs(GNUPGHOME) - os.chmod(GNUPGHOME, 0o700) - gpg.add_default_env('GNUPGHOME', GNUPGHOME) - - Gpg._gpg = gpg - return Gpg._gpg - - @classmethod - def create(cls, **kwargs): + def __init__(self, gnupg_home=None): + self.gnupg_home = get_gnupg_home(gnupg_home) + + @cached_property + def prep(self): + # Make sure that suitable versions of gpgconf and gpg are available + ensure_gpg() + + # Make sure that the GNUPGHOME exists + if not os.path.exists(self.gnupg_home): + os.makedirs(self.gnupg_home) + os.chmod(self.gnupg_home, 0o700) + + if not os.path.isdir(self.gnupg_home): + raise SpackGPGError( + 'GNUPGHOME "{0}" exists and is not a directory'.format( + self.gnupg_home)) + + if GpgConstants.user_run_dir is not None: + self.gpgconf_exe('--create-socketdir') + + return True + + @cached_property + def gpgconf_exe(self): + exe = spack.util.executable.Executable(GpgConstants.gpgconf_string) + exe.add_default_env('GNUPGHOME', self.gnupg_home) + return exe + + @cached_property + def gpg_exe(self): + exe = spack.util.executable.Executable(GpgConstants.gpg_string) + exe.add_default_env('GNUPGHOME', self.gnupg_home) + return exe + + def __call__(self, *args, **kwargs): + if self.prep: + return self.gpg_exe(*args, **kwargs) + + def create(self, **kwargs): r, w = os.pipe() r = os.fdopen(r, 'r') w = os.fdopen(w, 'w') @@ -83,64 +323,108 @@ class Gpg(object): %%commit ''' % kwargs) w.close() - cls.gpg()('--gen-key', '--batch', input=r) + self('--gen-key', '--batch', input=r) r.close() - @classmethod - def signing_keys(cls): - output = cls.gpg()('--list-secret-keys', '--with-colons', - '--fingerprint', '--fingerprint', output=str) - return parse_keys_output(output) - - @classmethod - def export_keys(cls, location, *keys): - cls.gpg()('--armor', '--export', '--output', location, *keys) - - @classmethod - def trust(cls, keyfile): - cls.gpg()('--import', keyfile) - - @classmethod - def untrust(cls, signing, *keys): - args = [ - '--yes', - '--batch', - ] + def signing_keys(self, *args): + output = self('--list-secret-keys', '--with-colons', '--fingerprint', + *args, output=str) + return parse_secret_keys_output(output) + + def public_keys(self, *args): + output = self('--list-public-keys', '--with-colons', '--fingerprint', + *args, output=str) + return parse_public_keys_output(output) + + def export_keys(self, location, *keys): + self('--batch', '--yes', + '--armor', '--export', + '--output', location, *keys) + + def trust(self, keyfile): + self('--import', keyfile) + + def untrust(self, signing, *keys): if signing: - signing_args = args + ['--delete-secret-keys'] + list(keys) - cls.gpg()(*signing_args) - args.append('--delete-keys') - args.extend(keys) - cls.gpg()(*args) - - @classmethod - def sign(cls, key, file, output, clearsign=False): - args = [ - '--armor', - '--default-key', key, - '--output', output, - file, - ] - if clearsign: - args.insert(0, '--clearsign') - else: - args.insert(0, '--detach-sign') - cls.gpg()(*args) - - @classmethod - def verify(cls, signature, file, suppress_warnings=False): - if suppress_warnings: - cls.gpg()('--verify', signature, file, error=str) - else: - cls.gpg()('--verify', signature, file) - - @classmethod - def list(cls, trusted, signing): + skeys = self.signing_keys(*keys) + self('--batch', '--yes', '--delete-secret-keys', *skeys) + + pkeys = self.public_keys(*keys) + self('--batch', '--yes', '--delete-keys', *pkeys) + + def sign(self, key, file, output, clearsign=False): + self(('--clearsign' if clearsign else '--detach-sign'), + '--armor', '--default-key', key, + '--output', output, file) + + def verify(self, signature, file, suppress_warnings=False): + self('--verify', signature, file, + **({'error': str} if suppress_warnings else {})) + + def list(self, trusted, signing): if trusted: - cls.gpg()('--list-public-keys') + self('--list-public-keys') + if signing: - cls.gpg()('--list-secret-keys') + self('--list-secret-keys') class SpackGPGError(spack.error.SpackError): """Class raised when GPG errors are detected.""" + + +# Convenience wrappers for methods of the Gpg class + +# __call__ is a bit of a special case, since the Gpg instance is, itself, the +# "thing" that is being called. +@functools.wraps(Gpg.__call__) +def gpg(*args, **kwargs): + return get_global_gpg_instance()(*args, **kwargs) + + +gpg.name = 'gpg' + + +@functools.wraps(Gpg.create) +def create(*args, **kwargs): + return get_global_gpg_instance().create(*args, **kwargs) + + +@functools.wraps(Gpg.signing_keys) +def signing_keys(*args, **kwargs): + return get_global_gpg_instance().signing_keys(*args, **kwargs) + + +@functools.wraps(Gpg.public_keys) +def public_keys(*args, **kwargs): + return get_global_gpg_instance().public_keys(*args, **kwargs) + + +@functools.wraps(Gpg.export_keys) +def export_keys(*args, **kwargs): + return get_global_gpg_instance().export_keys(*args, **kwargs) + + +@functools.wraps(Gpg.trust) +def trust(*args, **kwargs): + return get_global_gpg_instance().trust(*args, **kwargs) + + +@functools.wraps(Gpg.untrust) +def untrust(*args, **kwargs): + return get_global_gpg_instance().untrust(*args, **kwargs) + + +@functools.wraps(Gpg.sign) +def sign(*args, **kwargs): + return get_global_gpg_instance().sign(*args, **kwargs) + + +@functools.wraps(Gpg.verify) +def verify(*args, **kwargs): + return get_global_gpg_instance().verify(*args, **kwargs) + + +@functools.wraps(Gpg.list) +def list(*args, **kwargs): + return get_global_gpg_instance().list(*args, **kwargs) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 3f71dd1f71..1aad85550a 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -224,7 +224,7 @@ def url_exists(url): try: read_from_url(url) return True - except URLError: + except (SpackWebError, URLError): return False @@ -295,15 +295,27 @@ def _iter_s3_prefix(client, url, num_entries=1024): break -def list_url(url): +def _iter_local_prefix(path): + for root, _, files in os.walk(path): + for f in files: + yield os.path.relpath(os.path.join(root, f), path) + + +def list_url(url, recursive=False): url = url_util.parse(url) local_path = url_util.local_file_path(url) if local_path: - return os.listdir(local_path) + if recursive: + return list(_iter_local_prefix(local_path)) + return [subpath for subpath in os.listdir(local_path) + if os.path.isfile(os.path.join(local_path, subpath))] if url.scheme == 's3': s3 = s3_util.create_s3_session(url) + if recursive: + return list(_iter_s3_prefix(s3, url)) + return list(set( key.split('/', 1)[0] for key in _iter_s3_prefix(s3, url))) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 2de74b888c..102a205f20 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -906,7 +906,7 @@ _spack_gpg() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="verify trust untrust sign create list init export" + SPACK_COMPREPLY="verify trust untrust sign create list init export publish" fi } @@ -972,6 +972,15 @@ _spack_gpg_export() { fi } +_spack_gpg_publish() { + if $list_options + then + SPACK_COMPREPLY="-h --help -d --directory -m --mirror-name --mirror-url --rebuild-index" + else + _keys + fi +} + _spack_graph() { if $list_options then |