diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 114 | ||||
| -rw-r--r-- | lib/spack/spack/ci.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/upload_s3.py | 214 | ||||
| -rw-r--r-- | lib/spack/spack/database.py | 72 | ||||
| -rw-r--r-- | lib/spack/spack/schema/buildcache_spec.py | 42 | ||||
| -rw-r--r-- | lib/spack/spack/schema/database_index.py | 58 | ||||
| -rw-r--r-- | lib/spack/spack/schema/spec.py | 159 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/buildcache.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 24 | ||||
| -rw-r--r-- | lib/spack/spack/test/database.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/test/packaging.py | 4 |
12 files changed, 440 insertions, 305 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 926f4679a1..05a7904815 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -25,6 +25,7 @@ from llnl.util.filesystem import mkdirp import spack.cmd import spack.config as config +import spack.database as spack_db import spack.fetch_strategy as fs import spack.util.gpg import spack.relocate as relocate @@ -32,7 +33,6 @@ import spack.util.spack_yaml as syaml import spack.mirror import spack.util.url as url_util import spack.util.web as web_util - from spack.spec import Spec from spack.stage import Stage from spack.util.gpg import Gpg @@ -282,31 +282,47 @@ def sign_tarball(key, force, specfile_path): def generate_package_index(cache_prefix): """Create the build cache index page. - Creates (or replaces) the "index.html" page at the location given in + Creates (or replaces) the "index.json" page at the location given in cache_prefix. This page contains a link for each binary package (*.yaml) and public key (*.key) under cache_prefix. """ tmpdir = tempfile.mkdtemp() + db_root_dir = os.path.join(tmpdir, 'db_root') + db = spack_db.Database(None, db_dir=db_root_dir, + enable_transaction_locking=False, + record_fields=['spec', 'ref_count']) + + file_list = ( + entry + for entry in web_util.list_url(cache_prefix) + if entry.endswith('.yaml')) + + tty.debug('Retrieving spec.yaml files from {0} to build index'.format( + cache_prefix)) + for file_path in file_list: + try: + yaml_url = url_util.join(cache_prefix, file_path) + tty.debug('fetching {0}'.format(yaml_url)) + _, _, yaml_file = web_util.read_from_url(yaml_url) + yaml_contents = codecs.getreader('utf-8')(yaml_file).read() + # yaml_obj = syaml.load(yaml_contents) + # s = Spec.from_yaml(yaml_obj) + s = Spec.from_yaml(yaml_contents) + db.add(s, None) + except (URLError, web_util.SpackWebError) as url_err: + tty.error('Error reading spec.yaml: {0}'.format(file_path)) + tty.error(url_err) + try: - index_html_path = os.path.join(tmpdir, 'index.html') - file_list = ( - entry - for entry in web_util.list_url(cache_prefix) - if (entry.endswith('.yaml') - or entry.endswith('.key'))) - - with open(index_html_path, 'w') as f: - f.write(BUILD_CACHE_INDEX_TEMPLATE.format( - title='Spack Package Index', - path_list='\n'.join( - BUILD_CACHE_INDEX_ENTRY_TEMPLATE.format(path=path) - for path in file_list))) + index_json_path = os.path.join(db_root_dir, 'index.json') + with open(index_json_path, 'w') as f: + db._write_to_file(f) web_util.push_to_url( - index_html_path, - url_util.join(cache_prefix, 'index.html'), + index_json_path, + url_util.join(cache_prefix, 'index.json'), keep_original=False, - extra_args={'ContentType': 'text/html'}) + extra_args={'ContentType': 'application/json'}) finally: shutil.rmtree(tmpdir) @@ -825,49 +841,55 @@ def get_spec(spec=None, force=False): return try_download_specs(urls=urls, force=force) -def get_specs(force=False, allarch=False): +def get_specs(allarch=False): """ Get spec.yaml's for build caches available on mirror """ + global _cached_specs arch = architecture.Arch(architecture.platform(), 'default_os', 'default_target') - arch_pattern = ('([^-]*-[^-]*-[^-]*)') - if not allarch: - arch_pattern = '(%s-%s-[^-]*)' % (arch.platform, arch.os) - - regex_pattern = '%s(.*)(spec.yaml$)' % (arch_pattern) - arch_re = re.compile(regex_pattern) if not spack.mirror.MirrorCollection(): tty.debug("No Spack mirrors are currently configured") return {} - urls = set() for mirror in spack.mirror.MirrorCollection().values(): fetch_url_build_cache = url_util.join( mirror.fetch_url, _build_cache_relative_path) - mirror_dir = url_util.local_file_path(fetch_url_build_cache) - if mirror_dir: - tty.msg("Finding buildcaches in %s" % mirror_dir) - if os.path.exists(mirror_dir): - files = os.listdir(mirror_dir) - for file in files: - m = arch_re.search(file) - if m: - link = url_util.join(fetch_url_build_cache, file) - urls.add(link) - else: - tty.msg("Finding buildcaches at %s" % - url_util.format(fetch_url_build_cache)) - p, links = web_util.spider( - url_util.join(fetch_url_build_cache, 'index.html')) - for link in links: - m = arch_re.search(link) - if m: - urls.add(link) + tty.msg("Finding buildcaches at %s" % + url_util.format(fetch_url_build_cache)) - return try_download_specs(urls=urls, force=force) + index_url = url_util.join(fetch_url_build_cache, 'index.json') + + try: + _, _, file_stream = web_util.read_from_url( + index_url, 'application/json') + index_object = codecs.getreader('utf-8')(file_stream).read() + except (URLError, web_util.SpackWebError) as url_err: + tty.error('Failed to read index {0}'.format(index_url)) + tty.debug(url_err) + # Just return whatever specs we may already have cached + return _cached_specs + + tmpdir = tempfile.mkdtemp() + index_file_path = os.path.join(tmpdir, 'index.json') + with open(index_file_path, 'w') as fd: + fd.write(index_object) + + db_root_dir = os.path.join(tmpdir, 'db_root') + db = spack_db.Database(None, db_dir=db_root_dir, + enable_transaction_locking=False) + + db._read_from_file(index_file_path) + spec_list = db.query_local(installed=False) + + for indexed_spec in spec_list: + spec_arch = architecture.arch_for_spec(indexed_spec.architecture) + if (allarch is True or spec_arch == arch): + _cached_specs.add(indexed_spec) + + return _cached_specs def get_keys(install=False, trust=False, force=False): diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 580dbf29d2..abd5ca4b60 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -1025,9 +1025,9 @@ def read_cdashid_from_mirror(spec, mirror_url): def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id): if mirror_url: tty.debug('Creating buildcache') - buildcache._createtarball(env, yaml_path, None, True, False, - mirror_url, None, True, False, False, True, - False) + buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False, + output_location=mirror_url, force=True, + allow_root=True) if build_id: tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( build_id, mirror_url)) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 29048d5787..c28397b0aa 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -68,9 +68,9 @@ def setup_parser(subparser): type=str, help="URL of the mirror where " + "buildcaches will be written.") - create.add_argument('--no-rebuild-index', action='store_true', - default=False, help="skip rebuilding index after " + - "building package(s)") + create.add_argument('--rebuild-index', action='store_true', + default=False, help="Regenerate buildcache index " + + "after building package(s)") create.add_argument('-y', '--spec-yaml', default=None, help='Create buildcache entry for spec from yaml file') create.add_argument('--only', default='package,dependencies', @@ -108,8 +108,6 @@ def setup_parser(subparser): action='store_true', dest='variants', help='show variants in output (can be long)') - listcache.add_argument('-f', '--force', action='store_true', - help="force new download of specs") listcache.add_argument('-a', '--allarch', action='store_true', help="list specs for all available architectures" + " instead of default platform and OS") @@ -291,7 +289,7 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False, specs_from_cli = [] has_errors = False allarch = other_arch - specs = bindist.get_specs(force, allarch) + specs = bindist.get_specs(allarch) for pkg in pkgs: matches = [] tty.msg("buildcache spec(s) matching %s \n" % pkg) @@ -323,9 +321,10 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False, return specs_from_cli -def _createtarball(env, spec_yaml, packages, add_spec, add_deps, - output_location, key, force, rel, unsigned, allow_root, - no_rebuild_index): +def _createtarball(env, spec_yaml=None, packages=None, add_spec=True, + add_deps=True, output_location=os.getcwd(), + signing_key=None, force=False, make_relative=False, + unsigned=False, allow_root=False, rebuild_index=False): if spec_yaml: packages = set() with open(spec_yaml, 'r') as fd: @@ -355,10 +354,6 @@ def _createtarball(env, spec_yaml, packages, add_spec, add_deps, msg = 'Buildcache files will be output to %s/build_cache' % outdir tty.msg(msg) - signkey = None - if key: - signkey = key - matches = find_matching_specs(pkgs, env=env) if matches: @@ -398,9 +393,9 @@ def _createtarball(env, spec_yaml, packages, add_spec, add_deps, for spec in specs: tty.debug('creating binary cache file for package %s ' % spec.format()) - bindist.build_tarball(spec, outdir, force, rel, - unsigned, allow_root, signkey, - not no_rebuild_index) + bindist.build_tarball(spec, outdir, force, make_relative, + unsigned, allow_root, signing_key, + rebuild_index) def createtarball(args): @@ -447,9 +442,12 @@ def createtarball(args): add_spec = ('package' in args.things_to_install) add_deps = ('dependencies' in args.things_to_install) - _createtarball(env, args.spec_yaml, args.specs, add_spec, add_deps, - output_location, args.key, args.force, args.rel, - args.unsigned, args.allow_root, args.no_rebuild_index) + _createtarball(env, spec_yaml=args.spec_yaml, packages=args.specs, + add_spec=add_spec, add_deps=add_deps, + output_location=output_location, signing_key=args.key, + force=args.force, make_relative=args.rel, + unsigned=args.unsigned, allow_root=args.allow_root, + rebuild_index=args.rebuild_index) def installtarball(args): @@ -458,8 +456,7 @@ def installtarball(args): tty.die("build cache file installation requires" + " at least one package spec argument") pkgs = set(args.specs) - matches = match_downloaded_specs(pkgs, args.multiple, args.force, - args.otherarch) + matches = match_downloaded_specs(pkgs, args.multiple, args.otherarch) for match in matches: install_tarball(match, args) @@ -491,7 +488,7 @@ def install_tarball(spec, args): def listspecs(args): """list binary packages available from mirrors""" - specs = bindist.get_specs(args.force, args.allarch) + specs = bindist.get_specs(args.allarch) if args.specs: constraints = set(args.specs) specs = [s for s in specs if any(s.satisfies(c) for c in constraints)] diff --git a/lib/spack/spack/cmd/upload_s3.py b/lib/spack/spack/cmd/upload_s3.py deleted file mode 100644 index 56d03758cf..0000000000 --- a/lib/spack/spack/cmd/upload_s3.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# TODO: This will be merged into the buildcache command once -# everything is working. - -import os -import re -import sys - -try: - import boto3 - import botocore - have_boto3_support = True -except ImportError: - have_boto3_support = False - -import llnl.util.tty as tty - -from spack.error import SpackError -import spack.tengine as template_engine -from spack.spec import Spec - - -import spack.binary_distribution as bindist - - -description = "temporary command to upload buildcaches to 's3.spack.io'" -section = "packaging" -level = "long" - - -def setup_parser(subparser): - setup_parser.parser = subparser - subparsers = subparser.add_subparsers(help='upload-s3 sub-commands') - - # sub-command to upload a built spec to s3 - spec = subparsers.add_parser('spec', help=upload_spec.__doc__) - - spec.add_argument('-s', '--spec', default=None, - help='Spec to upload') - - spec.add_argument('-y', '--spec-yaml', default=None, - help='Path to spec yaml file containing spec to upload') - - spec.add_argument('-b', '--base-dir', default=None, - help='Path to root of buildcaches') - - spec.add_argument('-e', '--endpoint-url', - default='https://s3.spack.io', help='URL of mirror') - - spec.set_defaults(func=upload_spec) - - # sub-command to update the index of a buildcache on s3 - index = subparsers.add_parser('index', help=update_index.__doc__) - - index.add_argument('-e', '--endpoint-url', - default='https://s3.spack.io', help='URL of mirror') - - index.set_defaults(func=update_index) - - -def get_s3_session(endpoint_url): - if not have_boto3_support: - raise SpackError('boto3 module not available') - - session = boto3.Session() - s3 = session.resource('s3', endpoint_url=endpoint_url) - - bucket_names = [] - for bucket in s3.buckets.all(): - bucket_names.append(bucket.name) - - if len(bucket_names) > 1: - raise SpackError('More than one bucket associated with credentials') - - bucket_name = bucket_names[0] - - return s3, bucket_name - - -def update_index(args): - """Update the index of an s3 buildcache""" - s3, bucket_name = get_s3_session(args.endpoint_url) - - bucket = s3.Bucket(bucket_name) - exists = True - - try: - s3.meta.client.head_bucket(Bucket=bucket_name) - except botocore.exceptions.ClientError as e: - # If a client error is thrown, then check that it was a 404 error. - # If it was a 404 error, then the bucket does not exist. - error_code = e.response['Error']['Code'] - if error_code == '404': - exists = False - - if not exists: - tty.error('S3 bucket "{0}" does not exist'.format(bucket_name)) - sys.exit(1) - - build_cache_dir = os.path.join( - 'mirror', bindist.build_cache_relative_path()) - - spec_yaml_regex = re.compile('{0}/(.+\\.spec\\.yaml)$'.format( - build_cache_dir)) - spack_regex = re.compile('{0}/([^/]+)/.+\\.spack$'.format( - build_cache_dir)) - - top_level_keys = set() - - for key in bucket.objects.all(): - m = spec_yaml_regex.search(key.key) - if m: - top_level_keys.add(m.group(1)) - print(m.group(1)) - continue - - m = spack_regex.search(key.key) - if m: - top_level_keys.add(m.group(1)) - print(m.group(1)) - continue - - index_data = { - 'top_level_keys': top_level_keys, - } - - env = template_engine.make_environment() - template_dir = 'misc' - index_template = os.path.join(template_dir, 'buildcache_index.html') - t = env.get_template(index_template) - contents = t.render(index_data) - - index_key = os.path.join(build_cache_dir, 'index.html') - - tty.debug('Generated index:') - tty.debug(contents) - tty.debug('Pushing it to {0} -> {1}'.format(bucket_name, index_key)) - - s3_obj = s3.Object(bucket_name, index_key) - s3_obj.put(Body=contents, ACL='public-read') - - -def upload_spec(args): - """Upload a spec to s3 bucket""" - if not args.spec and not args.spec_yaml: - tty.error('Cannot upload spec without spec arg or path to spec yaml') - sys.exit(1) - - if not args.base_dir: - tty.error('No base directory for buildcache specified') - sys.exit(1) - - if args.spec: - try: - spec = Spec(args.spec) - spec.concretize() - except Exception as e: - tty.debug(e) - tty.error('Unable to concrectize spec from string {0}'.format( - args.spec)) - sys.exit(1) - else: - try: - with open(args.spec_yaml, 'r') as fd: - spec = Spec.from_yaml(fd.read()) - except Exception as e: - tty.debug(e) - tty.error('Unable to concrectize spec from yaml {0}'.format( - args.spec_yaml)) - sys.exit(1) - - s3, bucket_name = get_s3_session(args.endpoint_url) - - build_cache_dir = bindist.build_cache_relative_path() - - tarball_key = os.path.join( - build_cache_dir, bindist.tarball_path_name(spec, '.spack')) - tarball_path = os.path.join(args.base_dir, tarball_key) - - specfile_key = os.path.join( - build_cache_dir, bindist.tarball_name(spec, '.spec.yaml')) - specfile_path = os.path.join(args.base_dir, specfile_key) - - cdashidfile_key = os.path.join( - build_cache_dir, bindist.tarball_name(spec, '.cdashid')) - cdashidfile_path = os.path.join(args.base_dir, cdashidfile_key) - - tty.msg('Uploading {0}'.format(tarball_key)) - s3.meta.client.upload_file( - tarball_path, bucket_name, - os.path.join('mirror', tarball_key), - ExtraArgs={'ACL': 'public-read'}) - - tty.msg('Uploading {0}'.format(specfile_key)) - s3.meta.client.upload_file( - specfile_path, bucket_name, - os.path.join('mirror', specfile_key), - ExtraArgs={'ACL': 'public-read'}) - - if os.path.exists(cdashidfile_path): - tty.msg('Uploading {0}'.format(cdashidfile_key)) - s3.meta.client.upload_file( - cdashidfile_path, bucket_name, - os.path.join('mirror', cdashidfile_key), - ExtraArgs={'ACL': 'public-read'}) - - -def upload_s3(parser, args): - if args.func: - args.func(args) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 2acc8c2db6..01fb4fc7a5 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -48,6 +48,12 @@ from spack.filesystem_view import YamlFilesystemView from spack.util.crypto import bit_length from spack.version import Version + +@contextlib.contextmanager +def nullcontext(*args, **kwargs): + yield + + # TODO: Provide an API automatically retyring a build after detecting and # TODO: clearing a failure. @@ -87,6 +93,17 @@ _pkg_lock_timeout = None # Types of dependencies tracked by the database _tracked_deps = ('link', 'run') +# Default list of fields written for each install record +default_install_record_fields = [ + 'spec', + 'ref_count', + 'path', + 'installed', + 'explicit', + 'installation_time', + 'deprecated_for', +] + def _now(): """Returns the time since the epoch""" @@ -187,17 +204,17 @@ class InstallRecord(object): else: return InstallStatuses.MISSING in installed - def to_dict(self): - rec_dict = { - 'spec': self.spec.to_node_dict(), - 'path': self.path, - 'installed': self.installed, - 'ref_count': self.ref_count, - 'explicit': self.explicit, - 'installation_time': self.installation_time, - } - if self.deprecated_for: - rec_dict.update({'deprecated_for': self.deprecated_for}) + def to_dict(self, include_fields=default_install_record_fields): + rec_dict = {} + + for field_name in include_fields: + if field_name == 'spec': + rec_dict.update({'spec': self.spec.to_node_dict()}) + elif field_name == 'deprecated_for' and self.deprecated_for: + rec_dict.update({'deprecated_for': self.deprecated_for}) + else: + rec_dict.update({field_name: getattr(self, field_name)}) + return rec_dict @classmethod @@ -206,9 +223,12 @@ class InstallRecord(object): d.pop('spec', None) # Old databases may have "None" for path for externals - if d['path'] == 'None': + if 'path' not in d or d['path'] == 'None': d['path'] = None + if 'installed' not in d: + d['installed'] = False + return InstallRecord(spec, **d) @@ -275,7 +295,8 @@ class Database(object): _prefix_failures = {} def __init__(self, root, db_dir=None, upstream_dbs=None, - is_upstream=False): + is_upstream=False, enable_transaction_locking=True, + record_fields=default_install_record_fields): """Create a Database for Spack installations under ``root``. A Database is a cache of Specs data from ``$prefix/spec.yaml`` @@ -293,6 +314,12 @@ class Database(object): Caller may optionally provide a custom ``db_dir`` parameter where data will be stored. This is intended to be used for testing the Database class. + + This class supports writing buildcache index files, in which case + certain fields are not needed in each install record, and no + transaction locking is required. To use this feature, provide + ``enable_transaction_locking=False``, and specify a list of needed + fields in ``record_fields``. """ self.root = root @@ -356,14 +383,23 @@ class Database(object): # message) self._fail_when_missing_deps = False + if enable_transaction_locking: + self._write_transaction_impl = lk.WriteTransaction + self._read_transaction_impl = lk.ReadTransaction + else: + self._write_transaction_impl = nullcontext + self._read_transaction_impl = nullcontext + + self._record_fields = record_fields + def write_transaction(self): """Get a write lock context manager for use in a `with` block.""" - return lk.WriteTransaction( + return self._write_transaction_impl( self.lock, acquire=self._read, release=self._write) def read_transaction(self): """Get a read lock context manager for use in a `with` block.""" - return lk.ReadTransaction(self.lock, acquire=self._read) + return self._read_transaction_impl(self.lock, acquire=self._read) def _failed_spec_path(self, spec): """Return the path to the spec's failure file, which may not exist.""" @@ -573,7 +609,8 @@ class Database(object): This function does not do any locking or transactions. """ # map from per-spec hash code to installation record. - installs = dict((k, v.to_dict()) for k, v in self._data.items()) + installs = dict((k, v.to_dict(include_fields=self._record_fields)) + for k, v in self._data.items()) # database includes installation list and version. @@ -707,7 +744,8 @@ class Database(object): self.reindex(spack.store.layout) installs = dict( - (k, v.to_dict()) for k, v in self._data.items() + (k, v.to_dict(include_fields=self._record_fields)) + for k, v in self._data.items() ) def invalid_record(hash_key, error): diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py new file mode 100644 index 0000000000..69eae2fafc --- /dev/null +++ b/lib/spack/spack/schema/buildcache_spec.py @@ -0,0 +1,42 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for a buildcache spec.yaml file + +.. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py + :lines: 14- +""" +import spack.schema.spec + + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack buildcache spec.yaml schema', + 'type': 'object', + # 'additionalProperties': True, + 'properties': { + 'buildinfo': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['relative_prefix'], + 'properties': { + 'relative_prefix': {'type': 'string'}, + 'relative_rpaths': {'type': 'boolean'}, + }, + }, + 'full_hash': {'type': 'string'}, + 'spec': { + 'type': 'array', + 'items': spack.schema.spec.properties, + }, + 'binary_cache_checksum': { + 'type': 'object', + 'properties': { + 'hash_algorithm': {'type': 'string'}, + 'hash': {'type': 'string'}, + }, + }, + }, +} diff --git a/lib/spack/spack/schema/database_index.py b/lib/spack/spack/schema/database_index.py new file mode 100644 index 0000000000..eec3d4ce02 --- /dev/null +++ b/lib/spack/spack/schema/database_index.py @@ -0,0 +1,58 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for database index.json file + +.. literalinclude:: _spack_root/lib/spack/spack/schema/database_index.py + :lines: 36- +""" +import spack.schema.spec + +# spack.schema.spec.properties + +#: Full schema with metadata +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack spec schema', + 'type': 'object', + 'required': ['database'], + 'additionalProperties': False, + 'properties': { + 'database': { + 'type': 'object', + 'required': ['installs', 'version'], + 'additionalProperties': False, + 'properties': { + 'installs': { + 'type': 'object', + 'patternProperties': { + r'^[\w\d]{32}$': { + 'type': 'object', + 'properties': { + 'spec': spack.schema.spec.properties, + 'path': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'null'}, + ], + }, + 'installed': {'type': 'boolean'}, + 'ref_count': { + 'type': 'integer', + 'minimum': 0, + }, + 'explicit': {'type': 'boolean'}, + 'installation_time': { + 'type': 'number', + } + }, + }, + }, + }, + 'version': {'type': 'string'}, + } + }, + }, +} diff --git a/lib/spack/spack/schema/spec.py b/lib/spack/spack/schema/spec.py new file mode 100644 index 0000000000..8ec17e7e76 --- /dev/null +++ b/lib/spack/spack/schema/spec.py @@ -0,0 +1,159 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for a spec found in spec.yaml or database index.json files + +.. literalinclude:: _spack_root/lib/spack/spack/schema/spec.py + :lines: 13- +""" + + +target = { + 'oneOf': [ + { + 'type': 'string', + }, { + 'type': 'object', + 'additionalProperties': False, + 'required': [ + 'name', + 'vendor', + 'features', + 'generation', + 'parents', + ], + 'properties': { + 'name': {'type': 'string'}, + 'vendor': {'type': 'string'}, + 'features': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'generation': {'type': 'integer'}, + 'parents': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + ], +} + +arch = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'platform': {}, + 'platform_os': {}, + 'target': target, + }, +} + +dependencies = { + 'type': 'object', + 'patternProperties': { + r'\w[\w-]*': { # package name + 'type': 'object', + 'properties': { + 'hash': {'type': 'string'}, + 'type': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + }, +} + +#: Properties for inclusion in other schemas +properties = { + r'\w[\w-]*': { # package name + 'type': 'object', + 'additionalProperties': False, + 'required': [ + 'version', + 'arch', + 'compiler', + 'namespace', + 'parameters', + ], + 'properties': { + 'hash': {'type': 'string'}, + 'version': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'number'}, + ], + }, + 'arch': arch, + 'compiler': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'name': {'type': 'string'}, + 'version': {'type': 'string'}, + }, + }, + 'namespace': {'type': 'string'}, + 'parameters': { + 'type': 'object', + 'required': [ + 'cflags', + 'cppflags', + 'cxxflags', + 'fflags', + 'ldflags', + 'ldlibs', + ], + 'additionalProperties': True, + 'properties': { + 'patches': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'cflags': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'cppflags': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'cxxflags': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'fflags': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'ldflags': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'ldlib': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + }, + }, + 'patches': { + 'type': 'array', + 'items': {}, + }, + 'dependencies': dependencies, + }, + }, +} + + +#: Full schema with metadata +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack spec schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': properties, +} diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index af02d2d097..eb718a91da 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -24,7 +24,7 @@ add = spack.main.SpackCommand('add') def mock_get_specs(database, monkeypatch): specs = database.query_local() monkeypatch.setattr( - spack.binary_distribution, 'get_specs', lambda x, y: specs + spack.binary_distribution, 'get_specs', lambda x: specs ) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index ab73324411..2afe43cce5 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -4,8 +4,10 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp +import json import os import pytest +from jsonschema import validate import spack import spack.ci as ci @@ -15,6 +17,8 @@ import spack.hash_types as ht from spack.main import SpackCommand import spack.paths as spack_paths import spack.repo as repo +from spack.schema.buildcache_spec import schema as spec_yaml_schema +from spack.schema.database_index import schema as db_idx_schema from spack.spec import Spec from spack.util.mock_package import MockPackageMultiRepo import spack.util.executable as exe @@ -717,10 +721,28 @@ spack: ci.push_mirror_contents( env, concrete_spec, yaml_path, mirror_url, '42') - buildcache_list_output = buildcache_cmd('list', output=str) + buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache') + + # Test generating buildcache index while we have bin mirror + buildcache_cmd('update-index', '--mirror-url', mirror_url) + index_path = os.path.join(buildcache_path, 'index.json') + with open(index_path) as idx_fd: + index_object = json.load(idx_fd) + validate(index_object, db_idx_schema) + # Now that index is regenerated, validate "buildcache list" output + buildcache_list_output = buildcache_cmd('list', output=str) assert('patchelf' in buildcache_list_output) + # Also test buildcache_spec schema + bc_files_list = os.listdir(buildcache_path) + for file_name in bc_files_list: + if file_name.endswith('.spec.yaml'): + spec_yaml_path = os.path.join(buildcache_path, file_name) + with open(spec_yaml_path) as yaml_fd: + yaml_object = syaml.load(yaml_fd) + validate(yaml_object, spec_yaml_schema) + logs_dir = working_dir.join('logs_dir') if not os.path.exists(logs_dir.strpath): os.makedirs(logs_dir.strpath) diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 70f9bffe59..63276c0e7b 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -20,6 +20,8 @@ except ImportError: _use_uuid = False pass +from jsonschema import validate + import llnl.util.lock as lk from llnl.util.tty.colify import colify @@ -30,6 +32,7 @@ import spack.package import spack.spec from spack.util.mock_package import MockPackageMultiRepo from spack.util.executable import Executable +from spack.schema.database_index import schema pytestmark = pytest.mark.db @@ -424,6 +427,10 @@ def test_005_db_exists(database): assert os.path.exists(str(index_file)) assert os.path.exists(str(lock_file)) + with open(index_file) as fd: + index_object = json.load(fd) + validate(index_object, schema) + def test_010_all_install_sanity(database): """Ensure that the install layout reflects what we think it does.""" @@ -716,6 +723,8 @@ def test_old_external_entries_prefix(mutable_database): with open(spack.store.db._index_path, 'r') as f: db_obj = json.loads(f.read()) + validate(db_obj, schema) + s = spack.spec.Spec('externaltool') s.concretize() diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 7d4adfe975..75ead64955 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -108,6 +108,8 @@ echo $PATH""" else: create_args.insert(create_args.index('-a'), '-u') + create_args.insert(create_args.index('-a'), '--rebuild-index') + args = parser.parse_args(create_args) buildcache.buildcache(parser, args) # trigger overwrite warning @@ -165,7 +167,7 @@ echo $PATH""" args = parser.parse_args(['list']) buildcache.buildcache(parser, args) - args = parser.parse_args(['list', '-f']) + args = parser.parse_args(['list']) buildcache.buildcache(parser, args) args = parser.parse_args(['list', 'trivial']) |