Use json for buildcache index (#15002)

* Start moving toward a json buildcache index

* Add spec and database index schemas

* Add a schema for buildcache spec.yaml files

* Provide a mode for database class to generate buildcache index

* Update db and ci tests to validate object w/ new schema

* Remove unused temporary upload-s3 command

* Use database class to generate buildcache index

* Do not generate index with each buildcache creation

* Make buildcache index mode into a couple of constructor args to Database class

* Use keyword args for  _createtarball 

* Parse new json index when we get specs from buildcache

Now that only one index file per mirror needs to be fetched in
order to have all the concrete specs for binaries available on the
mirror, we can just fetch and refresh the cached specs every time
instead of needing to use the '-f' flag to force re-reading.

12 files changed, 440 insertions, 305 deletions

diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index 926f4679a1..05a7904815 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -25,6 +25,7 @@ from llnl.util.filesystem import mkdirp
 
 import spack.cmd
 import spack.config as config
+import spack.database as spack_db
 import spack.fetch_strategy as fs
 import spack.util.gpg
 import spack.relocate as relocate
@@ -32,7 +33,6 @@ import spack.util.spack_yaml as syaml
 import spack.mirror
 import spack.util.url as url_util
 import spack.util.web as web_util
-
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.util.gpg import Gpg
@@ -282,31 +282,47 @@ def sign_tarball(key, force, specfile_path):
 def generate_package_index(cache_prefix):
     """Create the build cache index page.
 
-    Creates (or replaces) the "index.html" page at the location given in
+    Creates (or replaces) the "index.json" page at the location given in
     cache_prefix.  This page contains a link for each binary package (*.yaml)
     and public key (*.key) under cache_prefix.
     """
     tmpdir = tempfile.mkdtemp()
+    db_root_dir = os.path.join(tmpdir, 'db_root')
+    db = spack_db.Database(None, db_dir=db_root_dir,
+                           enable_transaction_locking=False,
+                           record_fields=['spec', 'ref_count'])
+
+    file_list = (
+        entry
+        for entry in web_util.list_url(cache_prefix)
+        if entry.endswith('.yaml'))
+
+    tty.debug('Retrieving spec.yaml files from {0} to build index'.format(
+        cache_prefix))
+    for file_path in file_list:
+        try:
+            yaml_url = url_util.join(cache_prefix, file_path)
+            tty.debug('fetching {0}'.format(yaml_url))
+            _, _, yaml_file = web_util.read_from_url(yaml_url)
+            yaml_contents = codecs.getreader('utf-8')(yaml_file).read()
+            # yaml_obj = syaml.load(yaml_contents)
+            # s = Spec.from_yaml(yaml_obj)
+            s = Spec.from_yaml(yaml_contents)
+            db.add(s, None)
+        except (URLError, web_util.SpackWebError) as url_err:
+            tty.error('Error reading spec.yaml: {0}'.format(file_path))
+            tty.error(url_err)
+
     try:
-        index_html_path = os.path.join(tmpdir, 'index.html')
-        file_list = (
-            entry
-            for entry in web_util.list_url(cache_prefix)
-            if (entry.endswith('.yaml')
-                or entry.endswith('.key')))
-
-        with open(index_html_path, 'w') as f:
-            f.write(BUILD_CACHE_INDEX_TEMPLATE.format(
-                title='Spack Package Index',
-                path_list='\n'.join(
-                    BUILD_CACHE_INDEX_ENTRY_TEMPLATE.format(path=path)
-                    for path in file_list)))
+        index_json_path = os.path.join(db_root_dir, 'index.json')
+        with open(index_json_path, 'w') as f:
+            db._write_to_file(f)
 
         web_util.push_to_url(
-            index_html_path,
-            url_util.join(cache_prefix, 'index.html'),
+            index_json_path,
+            url_util.join(cache_prefix, 'index.json'),
             keep_original=False,
-            extra_args={'ContentType': 'text/html'})
+            extra_args={'ContentType': 'application/json'})
     finally:
         shutil.rmtree(tmpdir)
 
@@ -825,49 +841,55 @@ def get_spec(spec=None, force=False):
     return try_download_specs(urls=urls, force=force)
 
 
-def get_specs(force=False, allarch=False):
+def get_specs(allarch=False):
     """
     Get spec.yaml's for build caches available on mirror
     """
+    global _cached_specs
     arch = architecture.Arch(architecture.platform(),
                              'default_os', 'default_target')
-    arch_pattern = ('([^-]*-[^-]*-[^-]*)')
-    if not allarch:
-        arch_pattern = '(%s-%s-[^-]*)' % (arch.platform, arch.os)
-
-    regex_pattern = '%s(.*)(spec.yaml$)' % (arch_pattern)
-    arch_re = re.compile(regex_pattern)
 
     if not spack.mirror.MirrorCollection():
         tty.debug("No Spack mirrors are currently configured")
         return {}
 
-    urls = set()
     for mirror in spack.mirror.MirrorCollection().values():
         fetch_url_build_cache = url_util.join(
             mirror.fetch_url, _build_cache_relative_path)
 
-        mirror_dir = url_util.local_file_path(fetch_url_build_cache)
-        if mirror_dir:
-            tty.msg("Finding buildcaches in %s" % mirror_dir)
-            if os.path.exists(mirror_dir):
-                files = os.listdir(mirror_dir)
-                for file in files:
-                    m = arch_re.search(file)
-                    if m:
-                        link = url_util.join(fetch_url_build_cache, file)
-                        urls.add(link)
-        else:
-            tty.msg("Finding buildcaches at %s" %
-                    url_util.format(fetch_url_build_cache))
-            p, links = web_util.spider(
-                url_util.join(fetch_url_build_cache, 'index.html'))
-            for link in links:
-                m = arch_re.search(link)
-                if m:
-                    urls.add(link)
+        tty.msg("Finding buildcaches at %s" %
+                url_util.format(fetch_url_build_cache))
 
-    return try_download_specs(urls=urls, force=force)
+        index_url = url_util.join(fetch_url_build_cache, 'index.json')
+
+        try:
+            _, _, file_stream = web_util.read_from_url(
+                index_url, 'application/json')
+            index_object = codecs.getreader('utf-8')(file_stream).read()
+        except (URLError, web_util.SpackWebError) as url_err:
+            tty.error('Failed to read index {0}'.format(index_url))
+            tty.debug(url_err)
+            # Just return whatever specs we may already have cached
+            return _cached_specs
+
+        tmpdir = tempfile.mkdtemp()
+        index_file_path = os.path.join(tmpdir, 'index.json')
+        with open(index_file_path, 'w') as fd:
+            fd.write(index_object)
+
+        db_root_dir = os.path.join(tmpdir, 'db_root')
+        db = spack_db.Database(None, db_dir=db_root_dir,
+                               enable_transaction_locking=False)
+
+        db._read_from_file(index_file_path)
+        spec_list = db.query_local(installed=False)
+
+        for indexed_spec in spec_list:
+            spec_arch = architecture.arch_for_spec(indexed_spec.architecture)
+            if (allarch is True or spec_arch == arch):
+                _cached_specs.add(indexed_spec)
+
+    return _cached_specs
 
 
 def get_keys(install=False, trust=False, force=False):
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index 580dbf29d2..abd5ca4b60 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -1025,9 +1025,9 @@ def read_cdashid_from_mirror(spec, mirror_url):
 def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id):
     if mirror_url:
         tty.debug('Creating buildcache')
-        buildcache._createtarball(env, yaml_path, None, True, False,
-                                  mirror_url, None, True, False, False, True,
-                                  False)
+        buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False,
+                                  output_location=mirror_url, force=True,
+                                  allow_root=True)
         if build_id:
             tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
                 build_id, mirror_url))
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py
index 29048d5787..c28397b0aa 100644
--- a/lib/spack/spack/cmd/buildcache.py
+++ b/lib/spack/spack/cmd/buildcache.py
@@ -68,9 +68,9 @@ def setup_parser(subparser):
                         type=str,
                         help="URL of the mirror where " +
                              "buildcaches will be written.")
-    create.add_argument('--no-rebuild-index', action='store_true',
-                        default=False, help="skip rebuilding index after " +
-                                            "building package(s)")
+    create.add_argument('--rebuild-index', action='store_true',
+                        default=False, help="Regenerate buildcache index " +
+                                            "after building package(s)")
     create.add_argument('-y', '--spec-yaml', default=None,
                         help='Create buildcache entry for spec from yaml file')
     create.add_argument('--only', default='package,dependencies',
@@ -108,8 +108,6 @@ def setup_parser(subparser):
                            action='store_true',
                            dest='variants',
                            help='show variants in output (can be long)')
-    listcache.add_argument('-f', '--force', action='store_true',
-                           help="force new download of specs")
     listcache.add_argument('-a', '--allarch', action='store_true',
                            help="list specs for all available architectures" +
                                  " instead of default platform and OS")
@@ -291,7 +289,7 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False,
     specs_from_cli = []
     has_errors = False
     allarch = other_arch
-    specs = bindist.get_specs(force, allarch)
+    specs = bindist.get_specs(allarch)
     for pkg in pkgs:
         matches = []
         tty.msg("buildcache spec(s) matching %s \n" % pkg)
@@ -323,9 +321,10 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False,
     return specs_from_cli
 
 
-def _createtarball(env, spec_yaml, packages, add_spec, add_deps,
-                   output_location, key, force, rel, unsigned, allow_root,
-                   no_rebuild_index):
+def _createtarball(env, spec_yaml=None, packages=None, add_spec=True,
+                   add_deps=True, output_location=os.getcwd(),
+                   signing_key=None, force=False, make_relative=False,
+                   unsigned=False, allow_root=False, rebuild_index=False):
     if spec_yaml:
         packages = set()
         with open(spec_yaml, 'r') as fd:
@@ -355,10 +354,6 @@ def _createtarball(env, spec_yaml, packages, add_spec, add_deps,
     msg = 'Buildcache files will be output to %s/build_cache' % outdir
     tty.msg(msg)
 
-    signkey = None
-    if key:
-        signkey = key
-
     matches = find_matching_specs(pkgs, env=env)
 
     if matches:
@@ -398,9 +393,9 @@ def _createtarball(env, spec_yaml, packages, add_spec, add_deps,
 
     for spec in specs:
         tty.debug('creating binary cache file for package %s ' % spec.format())
-        bindist.build_tarball(spec, outdir, force, rel,
-                              unsigned, allow_root, signkey,
-                              not no_rebuild_index)
+        bindist.build_tarball(spec, outdir, force, make_relative,
+                              unsigned, allow_root, signing_key,
+                              rebuild_index)
 
 
 def createtarball(args):
@@ -447,9 +442,12 @@ def createtarball(args):
     add_spec = ('package' in args.things_to_install)
     add_deps = ('dependencies' in args.things_to_install)
 
-    _createtarball(env, args.spec_yaml, args.specs, add_spec, add_deps,
-                   output_location, args.key, args.force, args.rel,
-                   args.unsigned, args.allow_root, args.no_rebuild_index)
+    _createtarball(env, spec_yaml=args.spec_yaml, packages=args.specs,
+                   add_spec=add_spec, add_deps=add_deps,
+                   output_location=output_location, signing_key=args.key,
+                   force=args.force, make_relative=args.rel,
+                   unsigned=args.unsigned, allow_root=args.allow_root,
+                   rebuild_index=args.rebuild_index)
 
 
 def installtarball(args):
@@ -458,8 +456,7 @@ def installtarball(args):
         tty.die("build cache file installation requires" +
                 " at least one package spec argument")
     pkgs = set(args.specs)
-    matches = match_downloaded_specs(pkgs, args.multiple, args.force,
-                                     args.otherarch)
+    matches = match_downloaded_specs(pkgs, args.multiple, args.otherarch)
 
     for match in matches:
         install_tarball(match, args)
@@ -491,7 +488,7 @@ def install_tarball(spec, args):
 
 def listspecs(args):
     """list binary packages available from mirrors"""
-    specs = bindist.get_specs(args.force, args.allarch)
+    specs = bindist.get_specs(args.allarch)
     if args.specs:
         constraints = set(args.specs)
         specs = [s for s in specs if any(s.satisfies(c) for c in constraints)]
diff --git a/lib/spack/spack/cmd/upload_s3.py b/lib/spack/spack/cmd/upload_s3.py
deleted file mode 100644
index 56d03758cf..0000000000
--- a/lib/spack/spack/cmd/upload_s3.py
+++ /dev/null
@@ -1,214 +0,0 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-# TODO: This will be merged into the buildcache command once
-# everything is working.
-
-import os
-import re
-import sys
-
-try:
-    import boto3
-    import botocore
-    have_boto3_support = True
-except ImportError:
-    have_boto3_support = False
-
-import llnl.util.tty as tty
-
-from spack.error import SpackError
-import spack.tengine as template_engine
-from spack.spec import Spec
-
-
-import spack.binary_distribution as bindist
-
-
-description = "temporary command to upload buildcaches to 's3.spack.io'"
-section = "packaging"
-level = "long"
-
-
-def setup_parser(subparser):
-    setup_parser.parser = subparser
-    subparsers = subparser.add_subparsers(help='upload-s3 sub-commands')
-
-    # sub-command to upload a built spec to s3
-    spec = subparsers.add_parser('spec', help=upload_spec.__doc__)
-
-    spec.add_argument('-s', '--spec', default=None,
-                      help='Spec to upload')
-
-    spec.add_argument('-y', '--spec-yaml', default=None,
-                      help='Path to spec yaml file containing spec to upload')
-
-    spec.add_argument('-b', '--base-dir', default=None,
-                      help='Path to root of buildcaches')
-
-    spec.add_argument('-e', '--endpoint-url',
-                      default='https://s3.spack.io', help='URL of mirror')
-
-    spec.set_defaults(func=upload_spec)
-
-    # sub-command to update the index of a buildcache on s3
-    index = subparsers.add_parser('index', help=update_index.__doc__)
-
-    index.add_argument('-e', '--endpoint-url',
-                       default='https://s3.spack.io', help='URL of mirror')
-
-    index.set_defaults(func=update_index)
-
-
-def get_s3_session(endpoint_url):
-    if not have_boto3_support:
-        raise SpackError('boto3 module not available')
-
-    session = boto3.Session()
-    s3 = session.resource('s3', endpoint_url=endpoint_url)
-
-    bucket_names = []
-    for bucket in s3.buckets.all():
-        bucket_names.append(bucket.name)
-
-    if len(bucket_names) > 1:
-        raise SpackError('More than one bucket associated with credentials')
-
-    bucket_name = bucket_names[0]
-
-    return s3, bucket_name
-
-
-def update_index(args):
-    """Update the index of an s3 buildcache"""
-    s3, bucket_name = get_s3_session(args.endpoint_url)
-
-    bucket = s3.Bucket(bucket_name)
-    exists = True
-
-    try:
-        s3.meta.client.head_bucket(Bucket=bucket_name)
-    except botocore.exceptions.ClientError as e:
-        # If a client error is thrown, then check that it was a 404 error.
-        # If it was a 404 error, then the bucket does not exist.
-        error_code = e.response['Error']['Code']
-        if error_code == '404':
-            exists = False
-
-    if not exists:
-        tty.error('S3 bucket "{0}" does not exist'.format(bucket_name))
-        sys.exit(1)
-
-    build_cache_dir = os.path.join(
-        'mirror', bindist.build_cache_relative_path())
-
-    spec_yaml_regex = re.compile('{0}/(.+\\.spec\\.yaml)$'.format(
-        build_cache_dir))
-    spack_regex = re.compile('{0}/([^/]+)/.+\\.spack$'.format(
-        build_cache_dir))
-
-    top_level_keys = set()
-
-    for key in bucket.objects.all():
-        m = spec_yaml_regex.search(key.key)
-        if m:
-            top_level_keys.add(m.group(1))
-            print(m.group(1))
-            continue
-
-        m = spack_regex.search(key.key)
-        if m:
-            top_level_keys.add(m.group(1))
-            print(m.group(1))
-            continue
-
-    index_data = {
-        'top_level_keys': top_level_keys,
-    }
-
-    env = template_engine.make_environment()
-    template_dir = 'misc'
-    index_template = os.path.join(template_dir, 'buildcache_index.html')
-    t = env.get_template(index_template)
-    contents = t.render(index_data)
-
-    index_key = os.path.join(build_cache_dir, 'index.html')
-
-    tty.debug('Generated index:')
-    tty.debug(contents)
-    tty.debug('Pushing it to {0} -> {1}'.format(bucket_name, index_key))
-
-    s3_obj = s3.Object(bucket_name, index_key)
-    s3_obj.put(Body=contents, ACL='public-read')
-
-
-def upload_spec(args):
-    """Upload a spec to s3 bucket"""
-    if not args.spec and not args.spec_yaml:
-        tty.error('Cannot upload spec without spec arg or path to spec yaml')
-        sys.exit(1)
-
-    if not args.base_dir:
-        tty.error('No base directory for buildcache specified')
-        sys.exit(1)
-
-    if args.spec:
-        try:
-            spec = Spec(args.spec)
-            spec.concretize()
-        except Exception as e:
-            tty.debug(e)
-            tty.error('Unable to concrectize spec from string {0}'.format(
-                args.spec))
-            sys.exit(1)
-    else:
-        try:
-            with open(args.spec_yaml, 'r') as fd:
-                spec = Spec.from_yaml(fd.read())
-        except Exception as e:
-            tty.debug(e)
-            tty.error('Unable to concrectize spec from yaml {0}'.format(
-                args.spec_yaml))
-            sys.exit(1)
-
-    s3, bucket_name = get_s3_session(args.endpoint_url)
-
-    build_cache_dir = bindist.build_cache_relative_path()
-
-    tarball_key = os.path.join(
-        build_cache_dir, bindist.tarball_path_name(spec, '.spack'))
-    tarball_path = os.path.join(args.base_dir, tarball_key)
-
-    specfile_key = os.path.join(
-        build_cache_dir, bindist.tarball_name(spec, '.spec.yaml'))
-    specfile_path = os.path.join(args.base_dir, specfile_key)
-
-    cdashidfile_key = os.path.join(
-        build_cache_dir, bindist.tarball_name(spec, '.cdashid'))
-    cdashidfile_path = os.path.join(args.base_dir, cdashidfile_key)
-
-    tty.msg('Uploading {0}'.format(tarball_key))
-    s3.meta.client.upload_file(
-        tarball_path, bucket_name,
-        os.path.join('mirror', tarball_key),
-        ExtraArgs={'ACL': 'public-read'})
-
-    tty.msg('Uploading {0}'.format(specfile_key))
-    s3.meta.client.upload_file(
-        specfile_path, bucket_name,
-        os.path.join('mirror', specfile_key),
-        ExtraArgs={'ACL': 'public-read'})
-
-    if os.path.exists(cdashidfile_path):
-        tty.msg('Uploading {0}'.format(cdashidfile_key))
-        s3.meta.client.upload_file(
-            cdashidfile_path, bucket_name,
-            os.path.join('mirror', cdashidfile_key),
-            ExtraArgs={'ACL': 'public-read'})
-
-
-def upload_s3(parser, args):
-    if args.func:
-        args.func(args)
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py
index 2acc8c2db6..01fb4fc7a5 100644
--- a/lib/spack/spack/database.py
+++ b/lib/spack/spack/database.py
@@ -48,6 +48,12 @@ from spack.filesystem_view import YamlFilesystemView
 from spack.util.crypto import bit_length
 from spack.version import Version
 
+
+@contextlib.contextmanager
+def nullcontext(*args, **kwargs):
+    yield
+
+
 # TODO: Provide an API automatically retyring a build after detecting and
 # TODO: clearing a failure.
 
@@ -87,6 +93,17 @@ _pkg_lock_timeout = None
 # Types of dependencies tracked by the database
 _tracked_deps = ('link', 'run')
 
+# Default list of fields written for each install record
+default_install_record_fields = [
+    'spec',
+    'ref_count',
+    'path',
+    'installed',
+    'explicit',
+    'installation_time',
+    'deprecated_for',
+]
+
 
 def _now():
     """Returns the time since the epoch"""
@@ -187,17 +204,17 @@ class InstallRecord(object):
         else:
             return InstallStatuses.MISSING in installed
 
-    def to_dict(self):
-        rec_dict = {
-            'spec': self.spec.to_node_dict(),
-            'path': self.path,
-            'installed': self.installed,
-            'ref_count': self.ref_count,
-            'explicit': self.explicit,
-            'installation_time': self.installation_time,
-        }
-        if self.deprecated_for:
-            rec_dict.update({'deprecated_for': self.deprecated_for})
+    def to_dict(self, include_fields=default_install_record_fields):
+        rec_dict = {}
+
+        for field_name in include_fields:
+            if field_name == 'spec':
+                rec_dict.update({'spec': self.spec.to_node_dict()})
+            elif field_name == 'deprecated_for' and self.deprecated_for:
+                rec_dict.update({'deprecated_for': self.deprecated_for})
+            else:
+                rec_dict.update({field_name: getattr(self, field_name)})
+
         return rec_dict
 
     @classmethod
@@ -206,9 +223,12 @@ class InstallRecord(object):
         d.pop('spec', None)
 
         # Old databases may have "None" for path for externals
-        if d['path'] == 'None':
+        if 'path' not in d or d['path'] == 'None':
             d['path'] = None
 
+        if 'installed' not in d:
+            d['installed'] = False
+
         return InstallRecord(spec, **d)
 
 
@@ -275,7 +295,8 @@ class Database(object):
     _prefix_failures = {}
 
     def __init__(self, root, db_dir=None, upstream_dbs=None,
-                 is_upstream=False):
+                 is_upstream=False, enable_transaction_locking=True,
+                 record_fields=default_install_record_fields):
         """Create a Database for Spack installations under ``root``.
 
         A Database is a cache of Specs data from ``$prefix/spec.yaml``
@@ -293,6 +314,12 @@ class Database(object):
         Caller may optionally provide a custom ``db_dir`` parameter
         where data will be stored. This is intended to be used for
         testing the Database class.
+
+        This class supports writing buildcache index files, in which case
+        certain fields are not needed in each install record, and no
+        transaction locking is required.  To use this feature, provide
+        ``enable_transaction_locking=False``, and specify a list of needed
+        fields in ``record_fields``.
         """
         self.root = root
 
@@ -356,14 +383,23 @@ class Database(object):
         # message)
         self._fail_when_missing_deps = False
 
+        if enable_transaction_locking:
+            self._write_transaction_impl = lk.WriteTransaction
+            self._read_transaction_impl = lk.ReadTransaction
+        else:
+            self._write_transaction_impl = nullcontext
+            self._read_transaction_impl = nullcontext
+
+        self._record_fields = record_fields
+
     def write_transaction(self):
         """Get a write lock context manager for use in a `with` block."""
-        return lk.WriteTransaction(
+        return self._write_transaction_impl(
             self.lock, acquire=self._read, release=self._write)
 
     def read_transaction(self):
         """Get a read lock context manager for use in a `with` block."""
-        return lk.ReadTransaction(self.lock, acquire=self._read)
+        return self._read_transaction_impl(self.lock, acquire=self._read)
 
     def _failed_spec_path(self, spec):
         """Return the path to the spec's failure file, which may not exist."""
@@ -573,7 +609,8 @@ class Database(object):
         This function does not do any locking or transactions.
         """
         # map from per-spec hash code to installation record.
-        installs = dict((k, v.to_dict()) for k, v in self._data.items())
+        installs = dict((k, v.to_dict(include_fields=self._record_fields))
+                        for k, v in self._data.items())
 
         # database includes installation list and version.
 
@@ -707,7 +744,8 @@ class Database(object):
 
                 self.reindex(spack.store.layout)
                 installs = dict(
-                    (k, v.to_dict()) for k, v in self._data.items()
+                    (k, v.to_dict(include_fields=self._record_fields))
+                    for k, v in self._data.items()
                 )
 
         def invalid_record(hash_key, error):
diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py
new file mode 100644
index 0000000000..69eae2fafc
--- /dev/null
+++ b/lib/spack/spack/schema/buildcache_spec.py
@@ -0,0 +1,42 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for a buildcache spec.yaml file
+
+.. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py
+   :lines: 14-
+"""
+import spack.schema.spec
+
+
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack buildcache spec.yaml schema',
+    'type': 'object',
+    # 'additionalProperties': True,
+    'properties': {
+        'buildinfo': {
+            'type': 'object',
+            'additionalProperties': False,
+            'required': ['relative_prefix'],
+            'properties': {
+                'relative_prefix': {'type': 'string'},
+                'relative_rpaths': {'type': 'boolean'},
+            },
+        },
+        'full_hash': {'type': 'string'},
+        'spec': {
+            'type': 'array',
+            'items': spack.schema.spec.properties,
+        },
+        'binary_cache_checksum': {
+            'type': 'object',
+            'properties': {
+                'hash_algorithm': {'type': 'string'},
+                'hash': {'type': 'string'},
+            },
+        },
+    },
+}
diff --git a/lib/spack/spack/schema/database_index.py b/lib/spack/spack/schema/database_index.py
new file mode 100644
index 0000000000..eec3d4ce02
--- /dev/null
+++ b/lib/spack/spack/schema/database_index.py
@@ -0,0 +1,58 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for database index.json file
+
+.. literalinclude:: _spack_root/lib/spack/spack/schema/database_index.py
+   :lines: 36-
+"""
+import spack.schema.spec
+
+# spack.schema.spec.properties
+
+#: Full schema with metadata
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack spec schema',
+    'type': 'object',
+    'required': ['database'],
+    'additionalProperties': False,
+    'properties': {
+        'database': {
+            'type': 'object',
+            'required': ['installs', 'version'],
+            'additionalProperties': False,
+            'properties': {
+                'installs': {
+                    'type': 'object',
+                    'patternProperties': {
+                        r'^[\w\d]{32}$': {
+                            'type': 'object',
+                            'properties': {
+                                'spec': spack.schema.spec.properties,
+                                'path': {
+                                    'oneOf': [
+                                        {'type': 'string'},
+                                        {'type': 'null'},
+                                    ],
+                                },
+                                'installed': {'type': 'boolean'},
+                                'ref_count': {
+                                    'type': 'integer',
+                                    'minimum': 0,
+                                },
+                                'explicit': {'type': 'boolean'},
+                                'installation_time': {
+                                    'type': 'number',
+                                }
+                            },
+                        },
+                    },
+                },
+                'version': {'type': 'string'},
+            }
+        },
+    },
+}
diff --git a/lib/spack/spack/schema/spec.py b/lib/spack/spack/schema/spec.py
new file mode 100644
index 0000000000..8ec17e7e76
--- /dev/null
+++ b/lib/spack/spack/schema/spec.py
@@ -0,0 +1,159 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for a spec found in spec.yaml or database index.json files
+
+.. literalinclude:: _spack_root/lib/spack/spack/schema/spec.py
+   :lines: 13-
+"""
+
+
+target = {
+    'oneOf': [
+        {
+            'type': 'string',
+        }, {
+            'type': 'object',
+            'additionalProperties': False,
+            'required': [
+                'name',
+                'vendor',
+                'features',
+                'generation',
+                'parents',
+            ],
+            'properties': {
+                'name': {'type': 'string'},
+                'vendor': {'type': 'string'},
+                'features': {
+                    'type': 'array',
+                    'items': {'type': 'string'},
+                },
+                'generation': {'type': 'integer'},
+                'parents': {
+                    'type': 'array',
+                    'items': {'type': 'string'},
+                },
+            },
+        },
+    ],
+}
+
+arch = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'platform': {},
+        'platform_os': {},
+        'target': target,
+    },
+}
+
+dependencies = {
+    'type': 'object',
+    'patternProperties': {
+        r'\w[\w-]*': {  # package name
+            'type': 'object',
+            'properties': {
+                'hash': {'type': 'string'},
+                'type': {
+                    'type': 'array',
+                    'items': {'type': 'string'},
+                },
+            },
+        },
+    },
+}
+
+#: Properties for inclusion in other schemas
+properties = {
+    r'\w[\w-]*': {  # package name
+        'type': 'object',
+        'additionalProperties': False,
+        'required': [
+            'version',
+            'arch',
+            'compiler',
+            'namespace',
+            'parameters',
+        ],
+        'properties': {
+            'hash': {'type': 'string'},
+            'version': {
+                'oneOf': [
+                    {'type': 'string'},
+                    {'type': 'number'},
+                ],
+            },
+            'arch': arch,
+            'compiler': {
+                'type': 'object',
+                'additionalProperties': False,
+                'properties': {
+                    'name': {'type': 'string'},
+                    'version': {'type': 'string'},
+                },
+            },
+            'namespace': {'type': 'string'},
+            'parameters': {
+                'type': 'object',
+                'required': [
+                    'cflags',
+                    'cppflags',
+                    'cxxflags',
+                    'fflags',
+                    'ldflags',
+                    'ldlibs',
+                ],
+                'additionalProperties': True,
+                'properties': {
+                    'patches': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                    'cflags': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                    'cppflags': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                    'cxxflags': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                    'fflags': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                    'ldflags': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                    'ldlib': {
+                        'type': 'array',
+                        'items': {'type': 'string'},
+                    },
+                },
+            },
+            'patches': {
+                'type': 'array',
+                'items': {},
+            },
+            'dependencies': dependencies,
+        },
+    },
+}
+
+
+#: Full schema with metadata
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack spec schema',
+    'type': 'object',
+    'additionalProperties': False,
+    'patternProperties': properties,
+}
diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py
index af02d2d097..eb718a91da 100644
--- a/lib/spack/spack/test/cmd/buildcache.py
+++ b/lib/spack/spack/test/cmd/buildcache.py
@@ -24,7 +24,7 @@ add = spack.main.SpackCommand('add')
 def mock_get_specs(database, monkeypatch):
     specs = database.query_local()
     monkeypatch.setattr(
-        spack.binary_distribution, 'get_specs', lambda x, y: specs
+        spack.binary_distribution, 'get_specs', lambda x: specs
     )
 
 
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index ab73324411..2afe43cce5 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -4,8 +4,10 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import filecmp
+import json
 import os
 import pytest
+from jsonschema import validate
 
 import spack
 import spack.ci as ci
@@ -15,6 +17,8 @@ import spack.hash_types as ht
 from spack.main import SpackCommand
 import spack.paths as spack_paths
 import spack.repo as repo
+from spack.schema.buildcache_spec import schema as spec_yaml_schema
+from spack.schema.database_index import schema as db_idx_schema
 from spack.spec import Spec
 from spack.util.mock_package import MockPackageMultiRepo
 import spack.util.executable as exe
@@ -717,10 +721,28 @@ spack:
             ci.push_mirror_contents(
                 env, concrete_spec, yaml_path, mirror_url, '42')
 
-            buildcache_list_output = buildcache_cmd('list', output=str)
+            buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache')
+
+            # Test generating buildcache index while we have bin mirror
+            buildcache_cmd('update-index', '--mirror-url', mirror_url)
+            index_path = os.path.join(buildcache_path, 'index.json')
+            with open(index_path) as idx_fd:
+                index_object = json.load(idx_fd)
+                validate(index_object, db_idx_schema)
 
+            # Now that index is regenerated, validate "buildcache list" output
+            buildcache_list_output = buildcache_cmd('list', output=str)
             assert('patchelf' in buildcache_list_output)
 
+            # Also test buildcache_spec schema
+            bc_files_list = os.listdir(buildcache_path)
+            for file_name in bc_files_list:
+                if file_name.endswith('.spec.yaml'):
+                    spec_yaml_path = os.path.join(buildcache_path, file_name)
+                    with open(spec_yaml_path) as yaml_fd:
+                        yaml_object = syaml.load(yaml_fd)
+                        validate(yaml_object, spec_yaml_schema)
+
             logs_dir = working_dir.join('logs_dir')
             if not os.path.exists(logs_dir.strpath):
                 os.makedirs(logs_dir.strpath)
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
index 70f9bffe59..63276c0e7b 100644
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -20,6 +20,8 @@ except ImportError:
     _use_uuid = False
     pass
 
+from jsonschema import validate
+
 import llnl.util.lock as lk
 from llnl.util.tty.colify import colify
 
@@ -30,6 +32,7 @@ import spack.package
 import spack.spec
 from spack.util.mock_package import MockPackageMultiRepo
 from spack.util.executable import Executable
+from spack.schema.database_index import schema
 
 
 pytestmark = pytest.mark.db
@@ -424,6 +427,10 @@ def test_005_db_exists(database):
     assert os.path.exists(str(index_file))
     assert os.path.exists(str(lock_file))
 
+    with open(index_file) as fd:
+        index_object = json.load(fd)
+        validate(index_object, schema)
+
 
 def test_010_all_install_sanity(database):
     """Ensure that the install layout reflects what we think it does."""
@@ -716,6 +723,8 @@ def test_old_external_entries_prefix(mutable_database):
     with open(spack.store.db._index_path, 'r') as f:
         db_obj = json.loads(f.read())
 
+    validate(db_obj, schema)
+
     s = spack.spec.Spec('externaltool')
     s.concretize()
 
diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py
index 7d4adfe975..75ead64955 100644
--- a/lib/spack/spack/test/packaging.py
+++ b/lib/spack/spack/test/packaging.py
@@ -108,6 +108,8 @@ echo $PATH"""
     else:
         create_args.insert(create_args.index('-a'), '-u')
 
+    create_args.insert(create_args.index('-a'), '--rebuild-index')
+
     args = parser.parse_args(create_args)
     buildcache.buildcache(parser, args)
     # trigger overwrite warning
@@ -165,7 +167,7 @@ echo $PATH"""
     args = parser.parse_args(['list'])
     buildcache.buildcache(parser, args)
 
-    args = parser.parse_args(['list', '-f'])
+    args = parser.parse_args(['list'])
     buildcache.buildcache(parser, args)
 
     args = parser.parse_args(['list', 'trivial'])