From 73ab0e5dd77082ab7feee6e44de8f7cfa86798f9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 14 Apr 2018 23:09:57 -0700 Subject: init: replace spack.insecure global with spack.config.get --- lib/spack/spack/__init__.py | 5 ---- lib/spack/spack/config.py | 49 +++++++++++++++++++++++++++++++++------ lib/spack/spack/fetch_strategy.py | 6 ++--- lib/spack/spack/main.py | 3 ++- lib/spack/spack/test/config.py | 39 +++++++++++++++++++++++++++++++ lib/spack/spack/test/git_fetch.py | 10 ++++---- lib/spack/spack/test/hg_fetch.py | 10 ++++---- lib/spack/spack/test/svn_fetch.py | 10 ++++---- lib/spack/spack/test/url_fetch.py | 6 ++--- lib/spack/spack/util/web.py | 21 ++++++++++------- 10 files changed, 116 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index af306461bf..d593b3ce2f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -50,11 +50,6 @@ template_dirs = spack.config.get('config:template_dirs') template_dirs = [canonicalize_path(x) for x in template_dirs] -#: If this is enabled, tools that use SSL should not verify -#: certifiates. e.g., curl should use the -k option. -insecure = not spack.config.get('config:verify_ssl', True) - - #: Whether spack should allow installation of unsafe versions of software. #: "Unsafe" versions are ones it doesn't have a checksum for. do_checksum = spack.config.get('config:checksum', True) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index b481c48692..bbc344ebd8 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -53,6 +53,8 @@ import copy import os import re import sys +import multiprocessing +from contextlib import contextmanager from six import string_types from six import iteritems @@ -103,6 +105,17 @@ configuration_paths = ( ('user', spack.paths.user_config_path) ) +#: Hard-coded default values for some key configuration options. +#: This ensures that Spack will still work even if config.yaml in +#: the defaults scope is removed. +config_defaults = { + 'config': { + 'verify_ssl': True, + 'checksum': True, + 'dirty': False, + 'build_jobs': multiprocessing.cpu_count(), + } +} #: metavar to use for commands that accept scopes #: this is shorter and more readable than listing all choices @@ -206,9 +219,14 @@ class InternalConfigScope(ConfigScope): config file settings are accessed the same way, and Spack can easily override settings from files. """ - def __init__(self, name): + def __init__(self, name, data=None): self.name = name self.sections = syaml.syaml_dict() + if data: + for section in data: + dsec = data[section] + _validate_section({section: dsec}, section_schemas[section]) + self.sections[section] = syaml.syaml_dict({section: dsec}) def get_section_filename(self, section): raise NotImplementedError( @@ -363,7 +381,6 @@ class Configuration(object): continue if section not in data: - tty.warn("Skipping bad configuration file: '%s'" % scope.path) continue merged_section = _merge_yaml(merged_section, data) @@ -438,6 +455,21 @@ class Configuration(object): raise ConfigError("Error reading configuration: %s" % section) +@contextmanager +def override(path, value): + """Simple way to override config settings within a context.""" + overrides = InternalConfigScope('overrides') + + cfg = config() + cfg.push_scope(overrides) + cfg.set(path, value, scope='overrides') + + yield cfg + + scope = cfg.pop_scope() + assert scope is overrides + + def config(): """Singleton Configuration instance. @@ -451,16 +483,19 @@ def config(): """ global _configuration if not _configuration: - # Each scope can have per-platfom overrides in subdirectories of the - # configuration directory. + _configuration = Configuration() + + # first do the builtin, hardcoded defaults + defaults = InternalConfigScope('_builtin', config_defaults) + _configuration.push_scope(defaults) + + # Each scope can have per-platfom overrides in subdirectories platform = spack.architecture.platform().name - _configuration = Configuration() + # add each scope and its platform-specific directory for name, path in configuration_paths: - # add the regular scope _configuration.push_scope(ConfigScope(name, path)) - # add platform-specific scope plat_name = '%s/%s' % (name, platform) plat_path = os.path.join(path, platform) _configuration.push_scope(ConfigScope(plat_name, plat_path)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 779a4315fe..e1cc30c65c 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -244,7 +244,7 @@ class URLFetchStrategy(FetchStrategy): self.url, ] - if spack.insecure: + if not spack.config.get('config:verify_ssl'): curl_args.append('-k') if sys.stdout.isatty(): @@ -610,7 +610,7 @@ class GitFetchStrategy(VCSFetchStrategy): # If the user asked for insecure fetching, make that work # with git as well. - if spack.insecure: + if not spack.config.get('config:verify_ssl'): self._git.add_default_env('GIT_SSL_NO_VERIFY', 'true') return self._git @@ -899,7 +899,7 @@ class HgFetchStrategy(VCSFetchStrategy): args = ['clone'] - if spack.insecure: + if not spack.config.get('config:verify_ssl'): args.append('--insecure') args.append(self.url) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index c8ccf2447e..62ecec5592 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -41,6 +41,7 @@ import llnl.util.tty as tty from llnl.util.tty.log import log_output import spack +import spack.config import spack.paths from spack.error import SpackError @@ -357,7 +358,7 @@ def setup_main_options(args): # If the user asked for it, don't check ssl certs. if args.insecure: tty.warn("You asked for --insecure. Will NOT check SSL certificates.") - spack.insecure = True + spack.config.set('config:verify_ssl', False, scope='command_line') # when to use color (takes always, auto, or never) tty.color.set_color_when(args.color) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 1021380c88..5893e13a9c 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -438,6 +438,45 @@ def test_internal_config_filename(config, write_config_file): config.get_config_filename('command_line', 'config') +def test_internal_config_from_data(): + config = spack.config.Configuration() + + # add an internal config initialized from an inline dict + config.push_scope(spack.config.InternalConfigScope('_builtin', { + 'config': { + 'verify_ssl': False, + 'build_jobs': 6, + } + })) + + assert config.get('config:verify_ssl', scope='_builtin') is False + assert config.get('config:build_jobs', scope='_builtin') == 6 + + assert config.get('config:verify_ssl') is False + assert config.get('config:build_jobs') == 6 + + # push one on top and see what happens. + config.push_scope(spack.config.InternalConfigScope('higher', { + 'config': { + 'checksum': True, + 'verify_ssl': True, + } + })) + + assert config.get('config:verify_ssl', scope='_builtin') is False + assert config.get('config:build_jobs', scope='_builtin') == 6 + + assert config.get('config:verify_ssl', scope='higher') is True + assert config.get('config:build_jobs', scope='higher') is None + + assert config.get('config:verify_ssl') is True + assert config.get('config:build_jobs') == 6 + assert config.get('config:checksum') is True + + assert config.get('config:checksum', scope='_builtin') is None + assert config.get('config:checksum', scope='higher') is True + + def test_keys_are_ordered(): expected_order = ( diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index f9fd6381d1..2557c9ac8a 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -25,8 +25,11 @@ import os import pytest -import spack + from llnl.util.filesystem import working_dir, join_path, touch + +import spack +import spack.config from spack.spec import Spec from spack.version import ver from spack.fetch_strategy import GitFetchStrategy @@ -94,11 +97,8 @@ def test_fetch(type_of_test, # Enter the stage directory and check some properties with pkg.stage: - try: - spack.insecure = secure + with spack.config.override('config:verify_ssl', secure): pkg.do_stage() - finally: - spack.insecure = False with working_dir(pkg.stage.source_path): assert h('HEAD') == h(t.revision) diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index c894b7775e..03cbd3d641 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -25,8 +25,11 @@ import os import pytest -import spack + from llnl.util.filesystem import working_dir, join_path, touch + +import spack +import spack.config from spack.spec import Spec from spack.version import ver from spack.util.executable import which @@ -66,11 +69,8 @@ def test_fetch( # Enter the stage directory and check some properties with pkg.stage: - try: - spack.insecure = secure + with spack.config.override('config:verify_ssl', secure): pkg.do_stage() - finally: - spack.insecure = False with working_dir(pkg.stage.source_path): assert h() == t.revision diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 939c6c0ad3..85159615e0 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -25,8 +25,11 @@ import os import pytest -import spack + from llnl.util.filesystem import join_path, touch, working_dir + +import spack +import spack.config from spack.spec import Spec from spack.version import ver from spack.util.executable import which @@ -66,11 +69,8 @@ def test_fetch( # Enter the stage directory and check some properties with pkg.stage: - try: - spack.insecure = secure + with spack.config.override('config:verify_ssl', secure): pkg.do_stage() - finally: - spack.insecure = False with working_dir(pkg.stage.source_path): assert h() == t.revision diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py index 4085f89561..28fb07760a 100644 --- a/lib/spack/spack/test/url_fetch.py +++ b/lib/spack/spack/test/url_fetch.py @@ -28,6 +28,7 @@ import pytest from llnl.util.filesystem import working_dir, is_exe import spack +import spack.config from spack.fetch_strategy import from_list_url, URLFetchStrategy from spack.spec import Spec from spack.version import ver @@ -67,11 +68,8 @@ def test_fetch( # Enter the stage directory and check some properties with pkg.stage: - try: - spack.insecure = secure + with spack.config.override('config:verify_ssl', secure): pkg.do_stage() - finally: - spack.insecure = False with working_dir(pkg.stage.source_path): assert os.path.exists('configure') diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 796f64e71b..f694b49e7a 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -49,8 +49,12 @@ except ImportError: import llnl.util.tty as tty -import spack +import spack.config +import spack.cmd +import spack.url +import spack.stage import spack.error +import spack.util.crypto from spack.util.compression import ALLOWED_ARCHIVE_TYPES @@ -111,18 +115,19 @@ def _spider(url, visited, root, depth, max_depth, raise_on_error): try: context = None - if sys.version_info < (2, 7, 9) or \ - ((3,) < sys.version_info < (3, 4, 3)): - if not spack.insecure: + verify_ssl = spack.config.get('config:verify_ssl') + pyver = sys.version_info + if (pyver < (2, 7, 9) or (3,) < pyver < (3, 4, 3)): + if verify_ssl: tty.warn("Spack will not check SSL certificates. You need to " "update your Python to enable certificate " "verification.") - else: + elif verify_ssl: # We explicitly create default context to avoid error described in # https://blog.sucuri.net/2016/03/beware-unverified-tls-certificates-php-python.html - context = ssl._create_unverified_context() \ - if spack.insecure \ - else ssl.create_default_context() + context = ssl.create_default_context() + else: + context = ssl._create_unverified_context() # Make a HEAD request first to check the content type. This lets # us ignore tarballs and gigantic files. -- cgit v1.2.3-60-g2f50