From 77bd2dd706fee40e16ba5875cc195ed4b59d82cd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Apr 2018 19:06:53 -0700 Subject: init: replace global spack.do_checksum with config option --- lib/spack/spack/__init__.py | 5 ---- lib/spack/spack/cmd/bootstrap.py | 4 +-- lib/spack/spack/cmd/common/arguments.py | 10 +++++--- lib/spack/spack/cmd/diy.py | 7 +++-- lib/spack/spack/cmd/fetch.py | 7 +++-- lib/spack/spack/cmd/install.py | 6 ++--- lib/spack/spack/cmd/mirror.py | 5 ++-- lib/spack/spack/cmd/patch.py | 11 ++++---- lib/spack/spack/cmd/setup.py | 12 +++++---- lib/spack/spack/cmd/stage.py | 8 +++--- lib/spack/spack/package.py | 5 ++-- lib/spack/spack/test/conftest.py | 13 +++++----- lib/spack/spack/test/mirror.py | 45 ++++++++++++++++----------------- 13 files changed, 67 insertions(+), 71 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index d593b3ce2f..4734b4b990 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -50,11 +50,6 @@ template_dirs = spack.config.get('config:template_dirs') template_dirs = [canonicalize_path(x) for x in template_dirs] -#: Whether spack should allow installation of unsafe versions of software. -#: "Unsafe" versions are ones it doesn't have a checksum for. -do_checksum = spack.config.get('config:checksum', True) - - # If this is True, spack will not clean the environment to remove # potentially harmful variables before builds. dirty = spack.config.get('config:dirty', False) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 8aa0fe6171..6361d02132 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -42,9 +42,7 @@ def setup_parser(subparser): subparser.add_argument( '--keep-stage', action='store_true', dest='keep_stage', help="don't remove the build stage if installation succeeds") - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="do not check packages against checksum") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="display verbose build output while installing") diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index 9f22d706ab..9e58828560 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -123,16 +123,14 @@ _arguments['clean'] = Args( dest='dirty', help='sanitize the environment from variables that can affect how ' + ' packages find libraries or headers', - nargs=0 -) + nargs=0) _arguments['dirty'] = Args( '--dirty', action=CleanOrDirtyAction, dest='dirty', help='maintain the current environment without trying to sanitize it', - nargs=0 -) + nargs=0) _arguments['long'] = Args( '-l', '--long', action='store_true', @@ -149,3 +147,7 @@ _arguments['jobs'] = Args( _arguments['tags'] = Args( '-t', '--tags', action='append', help='filter a package query by tags') + +_arguments['no_checksum'] = Args( + '-n', '--no-checksum', action='store_true', default=False, + help="do not use checksums to verify downloadeded files (unsafe)") diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index ad9ca31071..1341715d78 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -29,6 +29,7 @@ import argparse import llnl.util.tty as tty import spack +import spack.config import spack.cmd import spack.cmd.common.arguments as arguments from spack.stage import DIYStage @@ -46,6 +47,7 @@ def setup_parser(subparser): subparser.add_argument( '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', help="don't try to install dependencies of requested packages") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '--keep-prefix', action='store_true', help="do not remove the install prefix if installation fails") @@ -101,8 +103,9 @@ def diy(self, args): # Forces the build to run out of the current directory. package.stage = DIYStage(source_path) - # TODO: make this an argument, not a global. - spack.do_checksum = False + # disable checksumming if requested + if args.no_checksum: + spack.config.set('config:checksum', False, scope='command_line') package.do_install( make_jobs=args.jobs, diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 7ec71f1c4a..ca3b49fa10 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -28,6 +28,7 @@ import llnl.util.tty as tty import spack import spack.cmd +import spack.cmd.common.arguments as arguments description = "fetch archives for packages" section = "build" @@ -35,9 +36,7 @@ level = "long" def setup_parser(subparser): - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="do not check packages against checksum") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '-m', '--missing', action='store_true', help="fetch only missing (not yet installed) dependencies") @@ -54,7 +53,7 @@ def fetch(parser, args): tty.die("fetch requires at least one package argument") if args.no_checksum: - spack.do_checksum = False + spack.config.set('config:checksum', False, scope='command_line') specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 05998b0815..aed5fddbda 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -77,9 +77,7 @@ the dependencies""" subparser.add_argument( '--source', action='store_true', dest='install_source', help="install source files in prefix") - subparser.add_argument( - '-n', '--no-checksum', action='store_true', - help="do not check packages against checksum") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '-v', '--verbose', action='store_true', help="display verbose build output while installing") @@ -176,7 +174,7 @@ def install(parser, args, **kwargs): tty.die("The -j option must be a positive integer!") if args.no_checksum: - spack.do_checksum = False # TODO: remove this global. + spack.config.set('config:checksum', False, scope='command_line') # Parse cli arguments and construct a dictionary # that will be passed to Package.do_install API diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index ef0997b534..6777c76030 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -33,6 +33,7 @@ import spack import spack.cmd import spack.config import spack.mirror +import spack.cmd.common.arguments as arguments from spack.spec import Spec from spack.error import SpackError from spack.util.spack_yaml import syaml_dict @@ -43,9 +44,7 @@ level = "long" def setup_parser(subparser): - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="do not check fetched packages against checksum") + arguments.add_common_arguments(subparser, ['no_checksum']) sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='mirror_command') diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 84ae0e5dcb..edadb75657 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -25,8 +25,11 @@ import argparse import llnl.util.tty as tty -import spack.cmd + import spack +import spack.cmd +import spack.cmd.common.arguments as arguments + description = "patch expanded archive sources in preparation for install" @@ -35,9 +38,7 @@ level = "long" def setup_parser(subparser): - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="do not check downloaded packages against checksum") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to stage") @@ -48,7 +49,7 @@ def patch(parser, args): tty.die("patch requires at least one package argument") if args.no_checksum: - spack.do_checksum = False + spack.config.set('config:checksum', False, scope='command_line') specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index bbf4740b12..65b71e0cb7 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -47,6 +47,7 @@ def setup_parser(subparser): subparser.add_argument( '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', help="do not try to install dependencies of requested packages") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="display verbose build output while installing") @@ -147,17 +148,16 @@ def setup(self, args): if not isinstance(package, spack.CMakePackage): tty.die( 'Support for {0} derived packages not yet implemented'.format( - package.build_system_class - ) - ) + package.build_system_class)) # It's OK if the package is already installed. # Forces the build to run out of the current directory. package.stage = DIYStage(os.getcwd()) - # TODO: make this an argument, not a global. - spack.do_checksum = False + # disable checksumming if requested + if args.no_checksum: + spack.config.set('config:checksum', False, scope='command_line') # Install dependencies if requested to do so if not args.ignore_deps: @@ -169,12 +169,14 @@ def setup(self, args): namespace=inst_args ) install.install(parser, inst_args) + # Generate spconfig.py tty.msg( 'Generating spconfig.py [{0}]'.format(package.spec.cshort_spec) ) dirty = args.dirty write_spconfig(package, dirty) + # Install this package to register it in the DB and permit # module file regeneration inst_args = copy.deepcopy(args) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 5dfa04b779..bceb9b8a6b 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -25,8 +25,10 @@ import argparse import llnl.util.tty as tty + import spack import spack.cmd +import spack.cmd.common.arguments as arguments description = "expand downloaded archive in preparation for install" section = "build" @@ -34,9 +36,7 @@ level = "long" def setup_parser(subparser): - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="do not check downloaded packages against checksum") + arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '-p', '--path', dest='path', help="path to stage package, does not add to spack tree") @@ -50,7 +50,7 @@ def stage(parser, args): tty.die("stage requires at least one package argument") if args.no_checksum: - spack.do_checksum = False + spack.config.set('config:checksum', False, scope='command_line') specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0fdb544b8f..83acb16ce8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1014,7 +1014,8 @@ class PackageBase(with_metaclass(PackageMeta, object)): raise ValueError("Can only fetch concrete packages.") start_time = time.time() - if spack.do_checksum and self.version not in self.versions: + checksum = spack.config.get('config:checksum') + if checksum and self.version not in self.versions: tty.warn("There is no checksum on file to fetch %s safely." % self.spec.cformat('$_$@')) @@ -1036,7 +1037,7 @@ class PackageBase(with_metaclass(PackageMeta, object)): self.stage.fetch(mirror_only) self._fetch_time = time.time() - start_time - if spack.do_checksum and self.version in self.versions: + if checksum and self.version in self.versions: self.stage.check() self.stage.cache_local() diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 6b60c31a4f..2e48d1d930 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -28,8 +28,6 @@ import os import shutil import re -import spack.util.ordereddict - import py import pytest @@ -37,6 +35,7 @@ from llnl.util.filesystem import remove_linked_tree import spack import spack.architecture +import spack.config import spack.caches import spack.database import spack.directory_layout @@ -44,6 +43,7 @@ import spack.paths import spack.platforms.test import spack.repository import spack.stage +import spack.util.ordereddict import spack.util.executable import spack.util.pattern from spack.dependency import Dependency @@ -380,11 +380,10 @@ def install_mockery(tmpdir, config, builtin_mock): new_opt, spack.store.layout) spack.store.db = spack.database.Database(new_opt) - # We use a fake package, so skip the checksum. - spack.do_checksum = False - yield - # Turn checksumming back on - spack.do_checksum = True + # We use a fake package, so temporarily disable checksumming + with spack.config.override('config:checksum', False): + yield + # Restore Spack's layout. spack.store.layout = layout spack.store.extensions = extensions diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index feda91e713..43ac2da786 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -88,29 +88,28 @@ def check_mirror(): spec = Spec(name).concretized() pkg = spec.package - saved_checksum_setting = spack.do_checksum - with pkg.stage: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path( - mock_repo.path, 'checked_out') - - svn = which('svn', required=True) - svn('checkout', mock_repo.url, original_path) - - dcmp = filecmp.dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded - # tarball - assert not dcmp.right_only - # and that all original files are present. - assert all(l in exclude for l in dcmp.left_only) - spack.do_checksum = saved_checksum_setting + with spack.config.override('config:checksum', False): + with pkg.stage: + pkg.do_stage(mirror_only=True) + + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path( + mock_repo.path, 'checked_out') + + svn = which('svn', required=True) + svn('checkout', mock_repo.url, original_path) + + dcmp = filecmp.dircmp( + original_path, pkg.stage.source_path) + + # make sure there are no new files in the expanded + # tarball + assert not dcmp.right_only + # and that all original files are present. + assert all(l in exclude for l in dcmp.left_only) @pytest.mark.usefixtures('config', 'refresh_builtin_mock') -- cgit v1.2.3-70-g09d2