From 74aee60f7d036907712b1196c6b73af5b4e43808 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 7 Apr 2018 23:27:51 -0700 Subject: init: simplify import ordering in __init__.py --- lib/spack/spack/__init__.py | 164 ++++++++++++++++--------------- lib/spack/spack/abi.py | 5 +- lib/spack/spack/concretize.py | 11 ++- lib/spack/spack/config.py | 5 +- lib/spack/spack/error.py | 3 +- lib/spack/spack/operating_systems/cnl.py | 8 +- lib/spack/spack/util/path.py | 2 + lib/spack/spack/version.py | 1 + 8 files changed, 112 insertions(+), 87 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ab6281e018..bc41c9ff23 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,13 +23,11 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import multiprocessing import os import sys -import tempfile -import getpass -from llnl.util.filesystem import * -import llnl.util.tty as tty +import multiprocessing + +from llnl.util.filesystem import ancestor #----------------------------------------------------------------------------- # Variables describing how Spack is laid out in its prefix. @@ -38,102 +36,73 @@ import llnl.util.tty as tty spack_root = ancestor(__file__, 4) # The spack script itself -spack_file = join_path(spack_root, "bin", "spack") +spack_file = os.path.join(spack_root, "bin", "spack") # spack directory hierarchy -lib_path = join_path(spack_root, "lib", "spack") -external_path = join_path(lib_path, "external") -build_env_path = join_path(lib_path, "env") -module_path = join_path(lib_path, "spack") -platform_path = join_path(module_path, 'platforms') -compilers_path = join_path(module_path, "compilers") -build_systems_path = join_path(module_path, 'build_systems') -operating_system_path = join_path(module_path, 'operating_systems') -test_path = join_path(module_path, "test") -hooks_path = join_path(module_path, "hooks") -var_path = join_path(spack_root, "var", "spack") -stage_path = join_path(var_path, "stage") -repos_path = join_path(var_path, "repos") -share_path = join_path(spack_root, "share", "spack") +lib_path = os.path.join(spack_root, "lib", "spack") +external_path = os.path.join(lib_path, "external") +build_env_path = os.path.join(lib_path, "env") +module_path = os.path.join(lib_path, "spack") +platform_path = os.path.join(module_path, 'platforms') +compilers_path = os.path.join(module_path, "compilers") +build_systems_path = os.path.join(module_path, 'build_systems') +operating_system_path = os.path.join(module_path, 'operating_systems') +test_path = os.path.join(module_path, "test") +hooks_path = os.path.join(module_path, "hooks") +var_path = os.path.join(spack_root, "var", "spack") +stage_path = os.path.join(var_path, "stage") +repos_path = os.path.join(var_path, "repos") +share_path = os.path.join(spack_root, "share", "spack") # Paths to built-in Spack repositories. -packages_path = join_path(repos_path, "builtin") -mock_packages_path = join_path(repos_path, "builtin.mock") +packages_path = os.path.join(repos_path, "builtin") +mock_packages_path = os.path.join(repos_path, "builtin.mock") # User configuration location user_config_path = os.path.expanduser('~/.spack') prefix = spack_root -opt_path = join_path(prefix, "opt") -etc_path = join_path(prefix, "etc") +opt_path = os.path.join(prefix, "opt") +etc_path = os.path.join(prefix, "etc") system_etc_path = '/etc' # GPG paths. -gpg_keys_path = join_path(var_path, "gpg") -mock_gpg_data_path = join_path(var_path, "gpg.mock", "data") -mock_gpg_keys_path = join_path(var_path, "gpg.mock", "keys") -gpg_path = join_path(opt_path, "spack", "gpg") +gpg_keys_path = os.path.join(var_path, "gpg") +mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") +mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") +gpg_path = os.path.join(opt_path, "spack", "gpg") #----------------------------------------------------------------------------- -# Initial imports (only for use in this file -- see __all__ below.) +# Below code imports spack packages. #----------------------------------------------------------------------------- -# These imports depend on the paths above, or on each other -# Group them here so it's easy to understand the order. -# TODO: refactor this stuff to be more init order agnostic. -import spack.repository -import spack.error -import spack.config -import spack.fetch_strategy -from spack.file_cache import FileCache -from spack.abi import ABI -from spack.concretize import DefaultConcretizer -from spack.version import Version -from spack.util.path import canonicalize_path -from spack.package_prefs import PackageTesting +# The imports depend on paths above, or on each other, so ordering is tricky. +# TODO: refactor everything below to be more init order agnostic. + #----------------------------------------------------------------------------- -# Initialize various data structures & objects at the core of Spack. +# Import spack.config first, as other modules may rely on its options. +# TODO: Below code should not import modules other than spack.config #----------------------------------------------------------------------------- -# Version information -spack_version = Version("0.11.2") - - -# Set up the default packages database. -try: - repo = spack.repository.RepoPath() - sys.meta_path.append(repo) -except spack.error.SpackError as e: - tty.die('while initializing Spack RepoPath:', e.message) - - -# Tests ABI compatibility between packages -abi = ABI() - +import spack.config +from spack.util.path import canonicalize_path -# This controls how things are concretized in spack. -# Replace it with a subclass if you want different -# policies. -concretizer = DefaultConcretizer() -#----------------------------------------------------------------------------- -# config.yaml options -#----------------------------------------------------------------------------- +# handle basic configuration first _config = spack.config.get_config('config') # Path where downloaded source code is cached cache_path = canonicalize_path( - _config.get('source_cache', join_path(var_path, "cache"))) -fetch_cache = spack.fetch_strategy.FsCache(cache_path) + _config.get('source_cache', os.path.join(var_path, "cache"))) # cache for miscellaneous stuff. misc_cache_path = canonicalize_path( - _config.get('misc_cache', join_path(user_config_path, 'cache'))) -misc_cache = FileCache(misc_cache_path) + _config.get('misc_cache', os.path.join(user_config_path, 'cache'))) +# TODO: get this out of __init__.py binary_cache_retrieved_specs = set() @@ -141,13 +110,14 @@ binary_cache_retrieved_specs = set() template_dirs = spack.config.get_config('config')['template_dirs'] template_dirs = [canonicalize_path(x) for x in template_dirs] -# If this is enabled, tools that use SSL should not verify -# certifiates. e.g., curl should use the -k option. + +#: If this is enabled, tools that use SSL should not verify +#: certifiates. e.g., curl should use the -k option. insecure = not _config.get('verify_ssl', True) -# Whether spack should allow installation of unsafe versions of software. -# "Unsafe" versions are ones it doesn't have a checksum for. +#: Whether spack should allow installation of unsafe versions of software. +#: "Unsafe" versions are ones it doesn't have a checksum for. do_checksum = _config.get('checksum', True) @@ -156,17 +126,57 @@ do_checksum = _config.get('checksum', True) dirty = _config.get('dirty', False) -# The number of jobs to use when building in parallel. -# By default, use all cores on the machine. +#: The number of jobs to use when building in parallel. +#: By default, use all cores on the machine. build_jobs = _config.get('build_jobs', multiprocessing.cpu_count()) -# Needed for test dependencies +#----------------------------------------------------------------------------- +# Initialize various data structures & objects at the core of Spack. +# +# TODO: move all of these imports out of __init__ to avoid importing the whole +# TODO: world on Spack startup. There are some design changes that need to be +# TODO: made to enable this (decoupling Spec, repo, DB, and store state). +# +# TODO: Spack probably needs some kind of object to manage this state so that +# TODO: this stuff doesn't have to be at module scope. +# ----------------------------------------------------------------------------- +# Version information +from spack.version import Version +spack_version = Version("0.11.2") + + +# set up the caches after getting all config options +import spack.fetch_strategy +from spack.file_cache import FileCache +misc_cache = FileCache(misc_cache_path) +fetch_cache = spack.fetch_strategy.FsCache(cache_path) + + +# Set up the default packages database. +import spack.error +try: + import spack.repository + repo = spack.repository.RepoPath() + sys.meta_path.append(repo) +except spack.error.SpackError as e: + import llnl.util.tty as tty + tty.die('while initializing Spack RepoPath:', e.message) + + +#: Concretizer class implements policy decisions for concretization +from spack.concretize import Concretizer +concretizer = Concretizer() + + +#: Needed for test dependencies +from spack.package_prefs import PackageTesting package_testing = PackageTesting() #----------------------------------------------------------------------------- -# When packages call 'from spack import *', this extra stuff is brought in. +# When packages call 'from spack import *', we import a set of things that +# should be useful for builds. # # Spack internal code should call 'import spack' and accesses other # variables (spack.repo, paths, etc.) directly. diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 1d3a940831..6663adf29f 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -22,15 +22,16 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - import os + +from llnl.util.lang import memoized + import spack import spack.spec from spack.build_environment import dso_suffix from spack.spec import CompilerSpec from spack.util.executable import Executable, ProcessError from spack.compilers.clang import Clang -from llnl.util.lang import memoized class ABI(object): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 151aee13a9..6efc365b5d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -39,6 +39,7 @@ from functools_backport import reverse_order from six import iteritems import spack +import spack.abi import spack.spec import spack.compilers import spack.architecture @@ -47,7 +48,11 @@ from spack.version import ver, Version, VersionList, VersionRange from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable -class DefaultConcretizer(object): +#: impements rudimentary logic for ABI compatibility +abi = spack.abi.ABI() + + +class Concretizer(object): """You can subclass this class to override some of the default concretization strategies, or you can override all of them. """ @@ -132,8 +137,8 @@ class DefaultConcretizer(object): return sorted(candidates, reverse=True, key=lambda spec: ( - spack.abi.compatible(spec, abi_exemplar, loose=True), - spack.abi.compatible(spec, abi_exemplar))) + abi.compatible(spec, abi_exemplar, loose=True), + abi.compatible(spec, abi_exemplar))) def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 96225c0691..a83eaf4fb6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -60,15 +60,16 @@ import yaml import jsonschema from yaml.error import MarkedYAMLError from jsonschema import Draft4Validator, validators -from spack.util.ordereddict import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp import spack import spack.architecture -from spack.error import SpackError import spack.schema +from spack.error import SpackError +from spack.util.ordereddict import OrderedDict + # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index 08112bf910..3beb5a187a 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -25,10 +25,11 @@ from __future__ import print_function import sys +import inspect import llnl.util.tty as tty + import spack -import inspect class SpackError(Exception): diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 153cacca13..d5e8f46077 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -26,8 +26,6 @@ import re import llnl.util.tty as tty -import spack.spec -import spack.compilers from spack.architecture import OperatingSystem from spack.util.multiproc import parmap from spack.util.module_cmd import get_module_cmd @@ -58,6 +56,9 @@ class Cnl(OperatingSystem): return latest_version def find_compilers(self, *paths): + # function-local so that cnl doesn't depend on spack.config + import spack.compilers + types = spack.compilers.all_compiler_types() compiler_lists = parmap( lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) @@ -68,6 +69,9 @@ class Cnl(OperatingSystem): return clist def find_compiler(self, cmp_cls, *paths): + # function-local so that cnl doesn't depend on spack.config + import spack.spec + compilers = [] if cmp_cls.PrgEnv: if not cmp_cls.PrgEnv_compiler: diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 83d00b67f0..a030c1160b 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -23,6 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """Utilities for managing paths in Spack. + +TODO: this is really part of spack.config. Consolidate it. """ import os import re diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index c863f52525..09dce77c88 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -51,6 +51,7 @@ from six import string_types from spack.util.spack_yaml import syaml_dict + __all__ = ['Version', 'VersionRange', 'VersionList', 'ver'] # Valid version characters -- cgit v1.2.3-70-g09d2