diff options
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 251 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 263 | ||||
| -rw-r--r-- | lib/spack/spack/spec.py | 32 | ||||
| -rw-r--r-- | lib/spack/spack/test/spec_yaml.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/util/web.py | 91 | ||||
| -rw-r--r-- | share/spack/templates/misc/buildcache_index.html | 11 |
6 files changed, 610 insertions, 79 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index fc05b078c3..46ac7790e4 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -12,7 +12,9 @@ import tempfile import hashlib from contextlib import closing -import ruamel.yaml as yaml +import json + +from six.moves.urllib.error import URLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp, install_tree, get_filetype @@ -21,12 +23,17 @@ import spack.cmd import spack.fetch_strategy as fs import spack.util.gpg as gpg_util import spack.relocate as relocate +import spack.util.spack_yaml as syaml +from spack.spec import Spec from spack.stage import Stage from spack.util.gpg import Gpg -from spack.util.web import spider +from spack.util.web import spider, read_from_url from spack.util.executable import ProcessError +_build_cache_relative_path = 'build_cache' + + class NoOverwriteException(Exception): """ Raised when a file exists and must be overwritten. @@ -90,11 +97,19 @@ def has_gnupg2(): return False +def build_cache_relative_path(): + return _build_cache_relative_path + + +def build_cache_directory(prefix): + return os.path.join(prefix, build_cache_relative_path()) + + def buildinfo_file_name(prefix): """ Filename of the binary package meta-data file """ - name = prefix + "/.spack/binary_distribution" + name = os.path.join(prefix, ".spack/binary_distribution") return name @@ -105,7 +120,7 @@ def read_buildinfo_file(prefix): filename = buildinfo_file_name(prefix) with open(filename, 'r') as inputfile: content = inputfile.read() - buildinfo = yaml.load(content) + buildinfo = syaml.load(content) return buildinfo @@ -162,7 +177,7 @@ def write_buildinfo_file(prefix, workdir, rel=False): buildinfo['relocate_links'] = link_to_relocate filename = buildinfo_file_name(workdir) with open(filename, 'w') as outfile: - outfile.write(yaml.dump(buildinfo, default_flow_style=True)) + outfile.write(syaml.dump(buildinfo, default_flow_style=True)) def tarball_directory_name(spec): @@ -235,35 +250,50 @@ def sign_tarball(key, force, specfile_path): Gpg.sign(key, specfile_path, '%s.asc' % specfile_path) -def generate_index(outdir, indexfile_path): - f = open(indexfile_path, 'w') +def _generate_html_index(path_list, output_path): + f = open(output_path, 'w') header = """<html>\n <head>\n</head>\n <list>\n""" footer = "</list>\n</html>\n" - paths = os.listdir(outdir + '/build_cache') f.write(header) - for path in paths: + for path in path_list: rel = os.path.basename(path) f.write('<li><a href="%s"> %s</a>\n' % (rel, rel)) f.write(footer) f.close() +def generate_package_index(build_cache_dir): + yaml_list = os.listdir(build_cache_dir) + path_list = [os.path.join(build_cache_dir, l) for l in yaml_list] + + index_html_path_tmp = os.path.join(build_cache_dir, 'index.html.tmp') + index_html_path = os.path.join(build_cache_dir, 'index.html') + + _generate_html_index(path_list, index_html_path_tmp) + shutil.move(index_html_path_tmp, index_html_path) + + def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, - allow_root=False, key=None): + allow_root=False, key=None, regenerate_index=False): """ Build a tarball from given spec and put it into the directory structure used at the mirror (following <tarball_directory_name>). """ + if not spec.concrete: + raise ValueError('spec must be concrete to build tarball') + # set up some paths + build_cache_dir = build_cache_directory(outdir) + tarfile_name = tarball_name(spec, '.tar.gz') - tarfile_dir = os.path.join(outdir, "build_cache", + tarfile_dir = os.path.join(build_cache_dir, tarball_directory_name(spec)) tarfile_path = os.path.join(tarfile_dir, tarfile_name) mkdirp(tarfile_dir) spackfile_path = os.path.join( - outdir, "build_cache", tarball_path_name(spec, '.spack')) + build_cache_dir, tarball_path_name(spec, '.spack')) if os.path.exists(spackfile_path): if force: os.remove(spackfile_path) @@ -275,8 +305,8 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, spec_file = os.path.join(spec.prefix, ".spack", "spec.yaml") specfile_name = tarball_name(spec, '.spec.yaml') specfile_path = os.path.realpath( - os.path.join(outdir, "build_cache", specfile_name)) - indexfile_path = os.path.join(outdir, "build_cache", "index.html") + os.path.join(build_cache_dir, specfile_name)) + if os.path.exists(specfile_path): if force: os.remove(specfile_path) @@ -319,7 +349,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, spec_dict = {} with open(spec_file, 'r') as inputfile: content = inputfile.read() - spec_dict = yaml.load(content) + spec_dict = syaml.load(content) bchecksum = {} bchecksum['hash_algorithm'] = 'sha256' bchecksum['hash'] = checksum @@ -330,8 +360,15 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, buildinfo['relative_prefix'] = os.path.relpath( spec.prefix, spack.store.layout.root) spec_dict['buildinfo'] = buildinfo + spec_dict['full_hash'] = spec.full_hash() + + tty.debug('The full_hash ({0}) of {1} will be written into {2}'.format( + spec_dict['full_hash'], spec.name, specfile_path)) + tty.debug(spec.tree()) + with open(specfile_path, 'w') as outfile: - outfile.write(yaml.dump(spec_dict)) + outfile.write(syaml.dump(spec_dict)) + # sign the tarball and spec file with gpg if not unsigned: sign_tarball(key, force, specfile_path) @@ -349,9 +386,9 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, os.remove('%s.asc' % specfile_path) # create an index.html for the build_cache directory so specs can be found - if os.path.exists(indexfile_path): - os.remove(indexfile_path) - generate_index(outdir, indexfile_path) + if regenerate_index: + generate_package_index(build_cache_dir) + return None @@ -365,8 +402,8 @@ def download_tarball(spec): tty.die("Please add a spack mirror to allow " + "download of pre-compiled packages.") tarball = tarball_path_name(spec, '.spack') - for key in mirrors: - url = mirrors[key] + "/build_cache/" + tarball + for mirror_name, mirror_url in mirrors.items(): + url = mirror_url + '/' + _build_cache_relative_path + '/' + tarball # stage the tarball into standard place stage = Stage(url, name="build_cache", keep=True) try: @@ -493,7 +530,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, spec_dict = {} with open(specfile_path, 'r') as inputfile: content = inputfile.read() - spec_dict = yaml.load(content) + spec_dict = syaml.load(content) bchecksum = spec_dict['binary_cache_checksum'] # if the checksums don't match don't install @@ -563,10 +600,9 @@ def get_specs(force=False): path = str(spack.architecture.sys_type()) urls = set() - for key in mirrors: - url = mirrors[key] - if url.startswith('file'): - mirror = url.replace('file://', '') + '/build_cache' + for mirror_name, mirror_url in mirrors.items(): + if mirror_url.startswith('file'): + mirror = mirror_url.replace('file://', '') + "/" + _build_cache_relative_path tty.msg("Finding buildcaches in %s" % mirror) if os.path.exists(mirror): files = os.listdir(mirror) @@ -575,8 +611,8 @@ def get_specs(force=False): link = 'file://' + mirror + '/' + file urls.add(link) else: - tty.msg("Finding buildcaches on %s" % url) - p, links = spider(url + "/build_cache") + tty.msg("Finding buildcaches on %s" % mirror_url) + p, links = spider(mirror_url + "/" + _build_cache_relative_path) for link in links: if re.search("spec.yaml", link) and re.search(path, link): urls.add(link) @@ -595,7 +631,7 @@ def get_specs(force=False): # read the spec from the build cache file. All specs # in build caches are concrete (as they are built) so # we need to mark this spec concrete on read-in. - spec = spack.spec.Spec.from_yaml(f) + spec = Spec.from_yaml(f) spec._mark_concrete() _cached_specs.append(spec) @@ -612,10 +648,10 @@ def get_keys(install=False, trust=False, force=False): "download of build caches.") keys = set() - for key in mirrors: - url = mirrors[key] - if url.startswith('file'): - mirror = url.replace('file://', '') + '/build_cache' + for mirror_name, mirror_url in mirrors.items(): + if mirror_url.startswith('file'): + mirror = os.path.join( + mirror_url.replace('file://', ''), _build_cache_relative_path) tty.msg("Finding public keys in %s" % mirror) files = os.listdir(mirror) for file in files: @@ -623,8 +659,8 @@ def get_keys(install=False, trust=False, force=False): link = 'file://' + mirror + '/' + file keys.add(link) else: - tty.msg("Finding public keys on %s" % url) - p, links = spider(url + "/build_cache", depth=1) + tty.msg("Finding public keys on %s" % mirror_url) + p, links = spider(mirror_url + "/build_cache", depth=1) for link in links: if re.search(r'\.key', link): keys.add(link) @@ -645,3 +681,148 @@ def get_keys(install=False, trust=False, force=False): else: tty.msg('Will not add this key to trusted keys.' 'Use -t to install all downloaded keys') + + +def needs_rebuild(spec, mirror_url, rebuild_on_errors=False): + if not spec.concrete: + raise ValueError('spec must be concrete to check against mirror') + + pkg_name = spec.name + pkg_version = spec.version + + pkg_hash = spec.dag_hash() + pkg_full_hash = spec.full_hash() + + tty.debug('Checking {0}-{1}, dag_hash = {2}, full_hash = {3}'.format( + pkg_name, pkg_version, pkg_hash, pkg_full_hash)) + tty.debug(spec.tree()) + + # Try to retrieve the .spec.yaml directly, based on the known + # format of the name, in order to determine if the package + # needs to be rebuilt. + build_cache_dir = build_cache_directory(mirror_url) + spec_yaml_file_name = tarball_name(spec, '.spec.yaml') + file_path = os.path.join(build_cache_dir, spec_yaml_file_name) + + result_of_error = 'Package ({0}) will {1}be rebuilt'.format( + spec.short_spec, '' if rebuild_on_errors else 'not ') + + try: + yaml_contents = read_from_url(file_path) + except URLError as url_err: + err_msg = [ + 'Unable to determine whether {0} needs rebuilding,', + ' caught URLError attempting to read from {1}.', + ] + tty.error(''.join(err_msg).format(spec.short_spec, file_path)) + tty.debug(url_err) + tty.warn(result_of_error) + return rebuild_on_errors + + if not yaml_contents: + tty.error('Reading {0} returned nothing'.format(file_path)) + tty.warn(result_of_error) + return rebuild_on_errors + + spec_yaml = syaml.load(yaml_contents) + + # If either the full_hash didn't exist in the .spec.yaml file, or it + # did, but didn't match the one we computed locally, then we should + # just rebuild. This can be simplified once the dag_hash and the + # full_hash become the same thing. + if ('full_hash' not in spec_yaml or + spec_yaml['full_hash'] != pkg_full_hash): + if 'full_hash' in spec_yaml: + reason = 'hash mismatch, remote = {0}, local = {1}'.format( + spec_yaml['full_hash'], pkg_full_hash) + else: + reason = 'full_hash was missing from remote spec.yaml' + tty.msg('Rebuilding {0}, reason: {1}'.format( + spec.short_spec, reason)) + tty.msg(spec.tree()) + return True + + return False + + +def check_specs_against_mirrors(mirrors, specs, output_file=None, + rebuild_on_errors=False): + """Check all the given specs against buildcaches on the given mirrors and + determine if any of the specs need to be rebuilt. Reasons for needing to + rebuild include binary cache for spec isn't present on a mirror, or it is + present but the full_hash has changed since last time spec was built. + + Arguments: + mirrors (dict): Mirrors to check against + specs (iterable): Specs to check against mirrors + output_file (string): Path to output file to be written. If provided, + mirrors with missing or out-of-date specs will be formatted as a + JSON object and written to this file. + rebuild_on_errors (boolean): Treat any errors encountered while + checking specs as a signal to rebuild package. + + Returns: 1 if any spec was out-of-date on any mirror, 0 otherwise. + + """ + rebuilds = {} + for mirror_name, mirror_url in mirrors.items(): + tty.msg('Checking for built specs at %s' % mirror_url) + + rebuild_list = [] + + for spec in specs: + if needs_rebuild(spec, mirror_url, rebuild_on_errors): + rebuild_list.append({ + 'short_spec': spec.short_spec, + 'hash': spec.dag_hash() + }) + + if rebuild_list: + rebuilds[mirror_url] = { + 'mirrorName': mirror_name, + 'mirrorUrl': mirror_url, + 'rebuildSpecs': rebuild_list + } + + if output_file: + with open(output_file, 'w') as outf: + outf.write(json.dumps(rebuilds)) + + return 1 if rebuilds else 0 + + +def _download_buildcache_entry(mirror_root, descriptions): + for description in descriptions: + url = os.path.join(mirror_root, description['url']) + path = description['path'] + fail_if_missing = not description['required'] + + mkdirp(path) + + stage = Stage(url, name="build_cache", path=path, keep=True) + + try: + stage.fetch() + except fs.FetchError: + if fail_if_missing: + tty.error('Failed to download required url {0}'.format(url)) + return False + + return True + + +def download_buildcache_entry(file_descriptions): + mirrors = spack.config.get('mirrors') + if len(mirrors) == 0: + tty.die("Please add a spack mirror to allow " + + "download of buildcache entries.") + + for mirror_name, mirror_url in mirrors.items(): + mirror_root = os.path.join(mirror_url, _build_cache_relative_path) + + if _download_buildcache_entry(mirror_root, file_descriptions): + return True + else: + continue + + return False diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 5e8c211b70..fe91312c42 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -4,14 +4,21 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse +import os +import sys import llnl.util.tty as tty import spack.cmd import spack.environment as ev +from spack.error import SpecError +import spack.config import spack.repo import spack.store -import spack.spec +from spack.paths import etc_path +from spack.spec import Spec, save_dependency_spec_yamls +from spack.spec_set import CombinatorialSpecSet + import spack.binary_distribution as bindist import spack.cmd.common.arguments as arguments from spack.cmd import display_specs @@ -43,6 +50,11 @@ def setup_parser(subparser): create.add_argument('-d', '--directory', metavar='directory', type=str, default='.', help="directory in which to save the tarballs.") + create.add_argument('--no-rebuild-index', action='store_true', + default=False, help="skip rebuilding index after " + + "building package(s)") + create.add_argument('-y', '--spec-yaml', default=None, + help='Create buildcache entry for spec from yaml file') create.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to create buildcache for") @@ -88,6 +100,81 @@ def setup_parser(subparser): help="force new download of keys") dlkeys.set_defaults(func=getkeys) + # Check if binaries need to be rebuilt on remote mirror + check = subparsers.add_parser('check', help=check_binaries.__doc__) + check.add_argument( + '-m', '--mirror-url', default=None, + help='Override any configured mirrors with this mirror url') + + check.add_argument( + '-o', '--output-file', default=None, + help='File where rebuild info should be written') + + # used to construct scope arguments below + scopes = spack.config.scopes() + scopes_metavar = spack.config.scopes_metavar + + check.add_argument( + '--scope', choices=scopes, metavar=scopes_metavar, + default=spack.config.default_modify_scope(), + help="configuration scope containing mirrors to check") + + check.add_argument( + '-s', '--spec', default=None, + help='Check single spec instead of release specs file') + + check.add_argument( + '-y', '--spec-yaml', default=None, + help='Check single spec from yaml file instead of release specs file') + + check.add_argument( + '--rebuild-on-error', default=False, action='store_true', + help="Default to rebuilding packages if errors are encountered " + + "during the process of checking whether rebuilding is needed") + + check.set_defaults(func=check_binaries) + + # Download tarball and spec.yaml + dltarball = subparsers.add_parser('download', help=get_tarball.__doc__) + dltarball.add_argument( + '-s', '--spec', default=None, + help="Download built tarball for spec from mirror") + dltarball.add_argument( + '-y', '--spec-yaml', default=None, + help="Download built tarball for spec (from yaml file) from mirror") + dltarball.add_argument( + '-p', '--path', default=None, + help="Path to directory where tarball should be downloaded") + dltarball.add_argument( + '-c', '--require-cdashid', action='store_true', default=False, + help="Require .cdashid file to be downloaded with buildcache entry") + dltarball.set_defaults(func=get_tarball) + + # Get buildcache name + getbuildcachename = subparsers.add_parser('get-buildcache-name', + help=get_buildcache_name.__doc__) + getbuildcachename.add_argument( + '-s', '--spec', default=None, + help='Spec string for which buildcache name is desired') + getbuildcachename.add_argument( + '-y', '--spec-yaml', default=None, + help='Path to spec yaml file for which buildcache name is desired') + getbuildcachename.set_defaults(func=get_buildcache_name) + + # Given the root spec, save the yaml of the dependent spec to a file + saveyaml = subparsers.add_parser('save-yaml', + help=save_spec_yamls.__doc__) + saveyaml.add_argument( + '-r', '--root-spec', default=None, + help='Root spec of dependent spec') + saveyaml.add_argument( + '-s', '--specs', default=None, + help='List of dependent specs for which saved yaml is desired') + saveyaml.add_argument( + '-y', '--yaml-dir', default=None, + help='Path to directory where spec yamls should be saved') + saveyaml.set_defaults(func=save_spec_yamls) + def find_matching_specs( pkgs, allow_multiple_matches=False, force=False, env=None): @@ -106,6 +193,7 @@ def find_matching_specs( # List of specs that match expressions given via command line specs_from_cli = [] has_errors = False + tty.debug('find_matching_specs: about to parse specs for {0}'.format(pkgs)) specs = spack.cmd.parse_specs(pkgs) for spec in specs: matching = spack.store.db.query(spec, hashes=hashes) @@ -178,10 +266,22 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False): def createtarball(args): """create a binary package from an existing install""" - if not args.packages: + if args.spec_yaml: + packages = set() + tty.msg('createtarball, reading spec from {0}'.format(args.spec_yaml)) + with open(args.spec_yaml, 'r') as fd: + yaml_text = fd.read() + tty.debug('createtarball read spec yaml:') + tty.debug(yaml_text) + s = Spec.from_yaml(yaml_text) + packages.add('/{0}'.format(s.dag_hash())) + elif args.packages: + packages = args.packages + else: tty.die("build cache file creation requires at least one" + - " installed package argument") - pkgs = set(args.packages) + " installed package argument or else path to a" + + " yaml file containing a spec to install") + pkgs = set(packages) specs = set() outdir = '.' if args.directory: @@ -194,7 +294,12 @@ def createtarball(args): env = ev.get_env(args, 'buildcache create') matches = find_matching_specs(pkgs, False, False, env=env) + + if matches: + tty.msg('Found at least one matching spec') + for match in matches: + tty.msg('examining match {0}'.format(match.format())) if match.external or match.virtual: tty.msg('skipping external or virtual spec %s' % match.format()) @@ -217,7 +322,8 @@ def createtarball(args): for spec in specs: tty.msg('creating binary cache file for package %s ' % spec.format()) bindist.build_tarball(spec, outdir, args.force, args.rel, - args.unsigned, args.allow_root, signkey) + args.unsigned, args.allow_root, signkey, + not args.no_rebuild_index) def installtarball(args): @@ -233,7 +339,7 @@ def installtarball(args): def install_tarball(spec, args): - s = spack.spec.Spec(spec) + s = Spec(spec) if s.external or s.virtual: tty.warn("Skipping external or virtual package %s" % spec.format()) return @@ -272,6 +378,151 @@ def getkeys(args): bindist.get_keys(args.install, args.trust, args.force) +def check_binaries(args): + """Check specs (either a single spec from --spec, or else the full set + of release specs) against remote binary mirror(s) to see if any need + to be rebuilt. This command uses the process exit code to indicate + its result, specifically, if the exit code is non-zero, then at least + one of the indicated specs needs to be rebuilt. + """ + if args.spec or args.spec_yaml: + specs = [get_concrete_spec(args)] + else: + release_specs_path = os.path.join( + etc_path, 'spack', 'defaults', 'release.yaml') + spec_set = CombinatorialSpecSet.from_file(release_specs_path) + specs = [spec for spec in spec_set] + + if not specs: + tty.msg('No specs provided, exiting.') + sys.exit(0) + + for spec in specs: + spec.concretize() + + # Next see if there are any configured binary mirrors + configured_mirrors = spack.config.get('mirrors', scope=args.scope) + + if args.mirror_url: + configured_mirrors = {'additionalMirrorUrl': args.mirror_url} + + if not configured_mirrors: + tty.msg('No mirrors provided, exiting.') + sys.exit(0) + + sys.exit(bindist.check_specs_against_mirrors( + configured_mirrors, specs, args.output_file, args.rebuild_on_error)) + + +def get_tarball(args): + """Download buildcache entry from a remote mirror to local folder. This + command uses the process exit code to indicate its result, specifically, + a non-zero exit code indicates that the command failed to download at + least one of the required buildcache components. Normally, just the + tarball and .spec.yaml files are required, but if the --require-cdashid + argument was provided, then a .cdashid file is also required.""" + if not args.spec and not args.spec_yaml: + tty.msg('No specs provided, exiting.') + sys.exit(0) + + if not args.path: + tty.msg('No download path provided, exiting') + sys.exit(0) + + spec = get_concrete_spec(args) + + tarfile_name = bindist.tarball_name(spec, '.spack') + tarball_dir_name = bindist.tarball_directory_name(spec) + tarball_path_name = os.path.join(tarball_dir_name, tarfile_name) + local_tarball_path = os.path.join(args.path, tarball_dir_name) + + files_to_fetch = [ + { + 'url': tarball_path_name, + 'path': local_tarball_path, + 'required': True, + }, { + 'url': bindist.tarball_name(spec, '.spec.yaml'), + 'path': args.path, + 'required': True, + }, { + 'url': bindist.tarball_name(spec, '.cdashid'), + 'path': args.path, + 'required': args.require_cdashid, + }, + ] + + result = bindist.download_buildcache_entry(files_to_fetch) + + if result: + sys.exit(0) + + sys.exit(1) + + +def get_concrete_spec(args): + spec_str = args.spec + spec_yaml_path = args.spec_yaml + + if not spec_str and not spec_yaml_path: + tty.msg('Must provide either spec string or path to ' + + 'yaml to concretize spec') + sys.exit(1) + + if spec_str: + try: + spec = Spec(spec_str) + spec.concretize() + except SpecError as spec_error: + tty.error('Unable to concrectize spec {0}'.format(args.spec)) + tty.debug(spec_error) + sys.exit(1) + + return spec + + with open(spec_yaml_path, 'r') as fd: + return Spec.from_yaml(fd.read()) + + +def get_buildcache_name(args): + """Get name (prefix) of buildcache entries for this spec""" + spec = get_concrete_spec(args) + buildcache_name = bindist.tarball_name(spec, '') + + print('{0}'.format(buildcache_name)) + + sys.exit(0) + + +def save_spec_yamls(args): + """Get full spec for dependencies, relative to root spec, and write them + to files in the specified output directory. Uses exit code to signal + success or failure. An exit code of zero means the command was likely + successful. If any errors or exceptions are encountered, or if expected + command-line arguments are not provided, then the exit code will be + non-zero.""" + if not args.root_spec: + tty.msg('No root spec provided, exiting.') + sys.exit(1) + + if not args.specs: + tty.msg('No dependent specs provided, exiting.') + sys.exit(1) + + if not args.yaml_dir: + tty.msg('No yaml directory provided, exiting.') + sys.exit(1) + + root_spec = Spec(args.root_spec) + root_spec.concretize() + root_spec_as_yaml = root_spec.to_yaml(all_deps=True) + + save_dependency_spec_yamls( + root_spec_as_yaml, args.yaml_dir, args.specs.split()) + + sys.exit(0) + + def buildcache(parser, args): if args.func: args.func(args) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7fcfe82c35..d37a6e402f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -3695,6 +3695,33 @@ def parse_anonymous_spec(spec_like, pkg_name): return anon_spec +def save_dependency_spec_yamls( + root_spec_as_yaml, output_directory, dependencies=None): + """Given a root spec (represented as a yaml object), index it with a subset + of its dependencies, and write each dependency to a separate yaml file + in the output directory. By default, all dependencies will be written + out. To choose a smaller subset of dependencies to be written, pass a + list of package names in the dependencies parameter. In case of any + kind of error, SaveSpecDependenciesError is raised with a specific + message about what went wrong.""" + root_spec = Spec.from_yaml(root_spec_as_yaml) + + dep_list = dependencies + if not dep_list: + dep_list = [dep.name for dep in root_spec.traverse()] + + for dep_name in dep_list: + if dep_name not in root_spec: + msg = 'Dependency {0} does not exist in root spec {1}'.format( + dep_name, root_spec.name) + raise SpecDependencyNotFoundError(msg) + dep_spec = root_spec[dep_name] + yaml_path = os.path.join(output_directory, '{0}.yaml'.format(dep_name)) + + with open(yaml_path, 'w') as fd: + fd.write(dep_spec.to_yaml(all_deps=True)) + + def base32_prefix_bits(hash_string, bits): """Return the first <bits> bits of a base32 string as an integer.""" if bits > len(hash_string) * 5: @@ -3880,3 +3907,8 @@ class ConflictsInSpecError(SpecError, RuntimeError): long_message += match_fmt_custom.format(idx + 1, c, w, msg) super(ConflictsInSpecError, self).__init__(message, long_message) + + +class SpecDependencyNotFoundError(SpecError): + """Raised when a failure is encountered writing the dependencies of + a spec.""" diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index ea04259cf3..1a0ef94f57 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -8,12 +8,16 @@ YAML format preserves DAG information in the spec. """ +import os + from collections import Iterable, Mapping import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -from spack.spec import Spec +from spack import repo +from spack.spec import Spec, save_dependency_spec_yamls from spack.util.spack_yaml import syaml_dict +from spack.test.conftest import MockPackage, MockPackageMultiRepo def check_yaml_round_trip(spec): @@ -198,3 +202,38 @@ def reverse_all_dicts(data): return type(data)(reverse_all_dicts(elt) for elt in data) else: return data + + +def check_specs_equal(original_spec, spec_yaml_path): + with open(spec_yaml_path, 'r') as fd: + spec_yaml = fd.read() + spec_from_yaml = Spec.from_yaml(spec_yaml) + return original_spec.eq_dag(spec_from_yaml) + + +def test_save_dependency_spec_yamls_subset(tmpdir, config): + output_path = str(tmpdir.mkdir('spec_yamls')) + + default = ('build', 'link') + + g = MockPackage('g', [], []) + f = MockPackage('f', [], []) + e = MockPackage('e', [], []) + d = MockPackage('d', [f, g], [default, default]) + c = MockPackage('c', [], []) + b = MockPackage('b', [d, e], [default, default]) + a = MockPackage('a', [b, c], [default, default]) + + mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) + + with repo.swap(mock_repo): + spec_a = Spec('a') + spec_a.concretize() + b_spec = spec_a['b'] + c_spec = spec_a['c'] + spec_a_yaml = spec_a.to_yaml(all_deps=True) + + save_dependency_spec_yamls(spec_a_yaml, output_path, ['b', 'c']) + + assert check_specs_equal(b_spec, os.path.join(output_path, 'b.yaml')) + assert check_specs_equal(c_spec, os.path.join(output_path, 'c.yaml')) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index db1d96b2d4..042ed9c60e 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -86,6 +86,58 @@ else: super(NonDaemonPool, self).__init__(*args, **kwargs) +def _read_from_url(url, accept_content_type=None): + context = None + verify_ssl = spack.config.get('config:verify_ssl') + pyver = sys.version_info + if (pyver < (2, 7, 9) or (3,) < pyver < (3, 4, 3)): + if verify_ssl: + tty.warn("Spack will not check SSL certificates. You need to " + "update your Python to enable certificate " + "verification.") + elif verify_ssl: + # without a defined context, urlopen will not verify the ssl cert for + # python 3.x + context = ssl.create_default_context() + else: + context = ssl._create_unverified_context() + + req = Request(url) + + if accept_content_type: + # Make a HEAD request first to check the content type. This lets + # us ignore tarballs and gigantic files. + # It would be nice to do this with the HTTP Accept header to avoid + # one round-trip. However, most servers seem to ignore the header + # if you ask for a tarball with Accept: text/html. + req.get_method = lambda: "HEAD" + resp = _urlopen(req, timeout=_timeout, context=context) + + if "Content-type" not in resp.headers: + tty.debug("ignoring page " + url) + return None, None + + if not resp.headers["Content-type"].startswith(accept_content_type): + tty.debug("ignoring page " + url + " with content type " + + resp.headers["Content-type"]) + return None, None + + # Do the real GET request when we know it's just HTML. + req.get_method = lambda: "GET" + response = _urlopen(req, timeout=_timeout, context=context) + response_url = response.geturl() + + # Read the page and and stick it in the map we'll return + page = response.read().decode('utf-8') + + return response_url, page + + +def read_from_url(url, accept_content_type=None): + resp_url, contents = _read_from_url(url, accept_content_type) + return contents + + def _spider(url, visited, root, depth, max_depth, raise_on_error): """Fetches URL and any pages it links to up to max_depth. @@ -107,46 +159,11 @@ def _spider(url, visited, root, depth, max_depth, raise_on_error): root = re.sub('/index.html$', '', root) try: - context = None - verify_ssl = spack.config.get('config:verify_ssl') - pyver = sys.version_info - if (pyver < (2, 7, 9) or (3,) < pyver < (3, 4, 3)): - if verify_ssl: - tty.warn("Spack will not check SSL certificates. You need to " - "update your Python to enable certificate " - "verification.") - elif verify_ssl: - # We explicitly create default context to avoid error described in - # https://blog.sucuri.net/2016/03/beware-unverified-tls-certificates-php-python.html - context = ssl.create_default_context() - else: - context = ssl._create_unverified_context() + response_url, page = _read_from_url(url, 'text/html') - # Make a HEAD request first to check the content type. This lets - # us ignore tarballs and gigantic files. - # It would be nice to do this with the HTTP Accept header to avoid - # one round-trip. However, most servers seem to ignore the header - # if you ask for a tarball with Accept: text/html. - req = Request(url) - req.get_method = lambda: "HEAD" - resp = _urlopen(req, timeout=_timeout, context=context) - - if "Content-type" not in resp.headers: - tty.debug("ignoring page " + url) - return pages, links - - if not resp.headers["Content-type"].startswith('text/html'): - tty.debug("ignoring page " + url + " with content type " + - resp.headers["Content-type"]) + if not response_url or not page: return pages, links - # Do the real GET request when we know it's just HTML. - req.get_method = lambda: "GET" - response = _urlopen(req, timeout=_timeout, context=context) - response_url = response.geturl() - - # Read the page and and stick it in the map we'll return - page = response.read().decode('utf-8') pages[response_url] = page # Parse out the links in the page diff --git a/share/spack/templates/misc/buildcache_index.html b/share/spack/templates/misc/buildcache_index.html new file mode 100644 index 0000000000..26a14c4ec8 --- /dev/null +++ b/share/spack/templates/misc/buildcache_index.html @@ -0,0 +1,11 @@ +<html> + <head> + </head> + <body> + <ul> +{% for bucket_key in top_level_keys %} + <li><a href="{{ bucket_key }}">{{ bucket_key }}</a></li> +{% endfor %} + </ul> + </body> +</html> |