diff options
author | Todd Gamblin <tgamblin@llnl.gov> | 2022-07-30 15:19:18 -0700 |
---|---|---|
committer | Todd Gamblin <tgamblin@llnl.gov> | 2022-07-31 13:29:20 -0700 |
commit | f52f6e99dbf1131886a80112b8c79dfc414afb7c (patch) | |
tree | 05cb7d64b2395922f2f24683da49f472075be12c /lib/spack/spack/build_environment.py | |
parent | 549ba1ed32372c67fc57271cde3797d58b7dec6e (diff) | |
download | spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.tar.gz spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.tar.bz2 spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.tar.xz spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.zip |
black: reformat entire repository with black
Diffstat (limited to 'lib/spack/spack/build_environment.py')
-rw-r--r-- | lib/spack/spack/build_environment.py | 575 |
1 files changed, 293 insertions, 282 deletions
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index ae961a695b..2d0fd2b5e6 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -84,31 +84,31 @@ from spack.util.string import plural # # This can be set by the user to globally disable parallel builds. # -SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE' +SPACK_NO_PARALLEL_MAKE = "SPACK_NO_PARALLEL_MAKE" # # These environment variables are set by # set_wrapper_variables and used to pass parameters to # Spack's compiler wrappers. # -SPACK_ENV_PATH = 'SPACK_ENV_PATH' -SPACK_INCLUDE_DIRS = 'SPACK_INCLUDE_DIRS' -SPACK_LINK_DIRS = 'SPACK_LINK_DIRS' -SPACK_RPATH_DIRS = 'SPACK_RPATH_DIRS' -SPACK_RPATH_DEPS = 'SPACK_RPATH_DEPS' -SPACK_LINK_DEPS = 'SPACK_LINK_DEPS' -SPACK_PREFIX = 'SPACK_PREFIX' -SPACK_INSTALL = 'SPACK_INSTALL' -SPACK_DEBUG = 'SPACK_DEBUG' -SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' -SPACK_DEBUG_LOG_ID = 'SPACK_DEBUG_LOG_ID' -SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' -SPACK_CCACHE_BINARY = 'SPACK_CCACHE_BINARY' -SPACK_SYSTEM_DIRS = 'SPACK_SYSTEM_DIRS' +SPACK_ENV_PATH = "SPACK_ENV_PATH" +SPACK_INCLUDE_DIRS = "SPACK_INCLUDE_DIRS" +SPACK_LINK_DIRS = "SPACK_LINK_DIRS" +SPACK_RPATH_DIRS = "SPACK_RPATH_DIRS" +SPACK_RPATH_DEPS = "SPACK_RPATH_DEPS" +SPACK_LINK_DEPS = "SPACK_LINK_DEPS" +SPACK_PREFIX = "SPACK_PREFIX" +SPACK_INSTALL = "SPACK_INSTALL" +SPACK_DEBUG = "SPACK_DEBUG" +SPACK_SHORT_SPEC = "SPACK_SHORT_SPEC" +SPACK_DEBUG_LOG_ID = "SPACK_DEBUG_LOG_ID" +SPACK_DEBUG_LOG_DIR = "SPACK_DEBUG_LOG_DIR" +SPACK_CCACHE_BINARY = "SPACK_CCACHE_BINARY" +SPACK_SYSTEM_DIRS = "SPACK_SYSTEM_DIRS" # Platform-specific library suffix. -dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' +dso_suffix = "dylib" if sys.platform == "darwin" else "so" def should_set_parallel_jobs(jobserver_support=False): @@ -117,9 +117,9 @@ def should_set_parallel_jobs(jobserver_support=False): - jobserver_support is enabled, and a jobserver was found. """ if ( - jobserver_support and - 'MAKEFLAGS' in os.environ and - '--jobserver' in os.environ['MAKEFLAGS'] + jobserver_support + and "MAKEFLAGS" in os.environ + and "--jobserver" in os.environ["MAKEFLAGS"] ): return False return not env_flag(SPACK_NO_PARALLEL_MAKE) @@ -127,13 +127,13 @@ def should_set_parallel_jobs(jobserver_support=False): class MakeExecutable(Executable): """Special callable executable object for make so the user can specify - parallelism options on a per-invocation basis. Specifying - 'parallel' to the call will override whatever the package's - global setting is, so you can either default to true or false and - override particular calls. Specifying 'jobs_env' to a particular - call will name an environment variable which will be set to the - parallelism level (without affecting the normal invocation with - -j). + parallelism options on a per-invocation basis. Specifying + 'parallel' to the call will override whatever the package's + global setting is, so you can either default to true or false and + override particular calls. Specifying 'jobs_env' to a particular + call will name an environment variable which will be set to the + parallelism level (without affecting the normal invocation with + -j). """ def __init__(self, name, jobs): @@ -144,25 +144,26 @@ class MakeExecutable(Executable): """parallel, and jobs_env from kwargs are swallowed and used here; remaining arguments are passed through to the superclass. """ - parallel = should_set_parallel_jobs(jobserver_support=True) and \ - kwargs.pop('parallel', self.jobs > 1) + parallel = should_set_parallel_jobs(jobserver_support=True) and kwargs.pop( + "parallel", self.jobs > 1 + ) if parallel: - args = ('-j{0}'.format(self.jobs),) + args - jobs_env = kwargs.pop('jobs_env', None) + args = ("-j{0}".format(self.jobs),) + args + jobs_env = kwargs.pop("jobs_env", None) if jobs_env: # Caller wants us to set an environment variable to # control the parallelism. - kwargs['extra_env'] = {jobs_env: str(self.jobs)} + kwargs["extra_env"] = {jobs_env: str(self.jobs)} return super(MakeExecutable, self).__call__(*args, **kwargs) def _on_cray(): host_platform = spack.platforms.host() - host_os = host_platform.operating_system('default_os') - on_cray = str(host_platform) == 'cray' - using_cnl = re.match(r'cnl\d+', str(host_os)) + host_os = host_platform.operating_system("default_os") + on_cray = str(host_platform) == "cray" + using_cnl = re.match(r"cnl\d+", str(host_os)) return on_cray, using_cnl @@ -175,33 +176,33 @@ def clean_environment(): # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. - env.unset('LD_LIBRARY_PATH') - env.unset('LD_RUN_PATH') - env.unset('DYLD_LIBRARY_PATH') - env.unset('DYLD_FALLBACK_LIBRARY_PATH') + env.unset("LD_LIBRARY_PATH") + env.unset("LD_RUN_PATH") + env.unset("DYLD_LIBRARY_PATH") + env.unset("DYLD_FALLBACK_LIBRARY_PATH") # These vars affect how the compiler finds libraries and include dirs. - env.unset('LIBRARY_PATH') - env.unset('CPATH') - env.unset('C_INCLUDE_PATH') - env.unset('CPLUS_INCLUDE_PATH') - env.unset('OBJC_INCLUDE_PATH') + env.unset("LIBRARY_PATH") + env.unset("CPATH") + env.unset("C_INCLUDE_PATH") + env.unset("CPLUS_INCLUDE_PATH") + env.unset("OBJC_INCLUDE_PATH") - env.unset('CMAKE_PREFIX_PATH') - env.unset('PYTHONPATH') + env.unset("CMAKE_PREFIX_PATH") + env.unset("PYTHONPATH") # Affects GNU make, can e.g. indirectly inhibit enabling parallel build # env.unset('MAKEFLAGS') # Avoid that libraries of build dependencies get hijacked. - env.unset('LD_PRELOAD') - env.unset('DYLD_INSERT_LIBRARIES') + env.unset("LD_PRELOAD") + env.unset("DYLD_INSERT_LIBRARIES") # Avoid <packagename>_ROOT user variables overriding spack dependencies # https://cmake.org/cmake/help/latest/variable/PackageName_ROOT.html # Spack needs SPACK_ROOT though, so we need to exclude that for varname in os.environ.keys(): - if varname.endswith('_ROOT') and varname != 'SPACK_ROOT': + if varname.endswith("_ROOT") and varname != "SPACK_ROOT": env.unset(varname) # On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid @@ -210,46 +211,55 @@ def clean_environment(): # depending on the CNL version). on_cray, using_cnl = _on_cray() if on_cray and not using_cnl: - env.unset('CRAY_LD_LIBRARY_PATH') + env.unset("CRAY_LD_LIBRARY_PATH") for varname in os.environ.keys(): - if 'PKGCONF' in varname: + if "PKGCONF" in varname: env.unset(varname) # Unset the following variables because they can affect installation of # Autotools and CMake packages. build_system_vars = [ - 'CC', 'CFLAGS', 'CPP', 'CPPFLAGS', # C variables - 'CXX', 'CCC', 'CXXFLAGS', 'CXXCPP', # C++ variables - 'F77', 'FFLAGS', 'FLIBS', # Fortran77 variables - 'FC', 'FCFLAGS', 'FCLIBS', # Fortran variables - 'LDFLAGS', 'LIBS' # linker variables + "CC", + "CFLAGS", + "CPP", + "CPPFLAGS", # C variables + "CXX", + "CCC", + "CXXFLAGS", + "CXXCPP", # C++ variables + "F77", + "FFLAGS", + "FLIBS", # Fortran77 variables + "FC", + "FCFLAGS", + "FCLIBS", # Fortran variables + "LDFLAGS", + "LIBS", # linker variables ] for v in build_system_vars: env.unset(v) # Unset mpi environment vars. These flags should only be set by # mpi providers for packages with mpi dependencies - mpi_vars = [ - 'MPICC', 'MPICXX', 'MPIFC', 'MPIF77', 'MPIF90' - ] + mpi_vars = ["MPICC", "MPICXX", "MPIFC", "MPIF77", "MPIF90"] for v in mpi_vars: env.unset(v) - build_lang = spack.config.get('config:build_language') + build_lang = spack.config.get("config:build_language") if build_lang: # Override language-related variables. This can be used to force # English compiler messages etc., which allows parse_log_events to # show useful matches. - env.set('LC_ALL', build_lang) + env.set("LC_ALL", build_lang) # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" - path = get_path('PATH') + path = get_path("PATH") for p in path: - if '/macports/' in p: - env.remove_path('PATH', p) + if "/macports/" in p: + env.remove_path("PATH", p) return env @@ -263,8 +273,7 @@ def set_compiler_environment_variables(pkg, env): compiler.verify_executables() # Set compiler variables used by CMake and autotools - assert all(key in compiler.link_paths for key in ( - 'cc', 'cxx', 'f77', 'fc')) + assert all(key in compiler.link_paths for key in ("cc", "cxx", "f77", "fc")) # Populate an object with the list of environment modifications # and return it @@ -274,36 +283,36 @@ def set_compiler_environment_variables(pkg, env): # Set SPACK compiler variables so that our wrapper knows what to call if compiler.cc: - env.set('SPACK_CC', compiler.cc) - env.set('CC', os.path.join(link_dir, compiler.link_paths['cc'])) + env.set("SPACK_CC", compiler.cc) + env.set("CC", os.path.join(link_dir, compiler.link_paths["cc"])) if compiler.cxx: - env.set('SPACK_CXX', compiler.cxx) - env.set('CXX', os.path.join(link_dir, compiler.link_paths['cxx'])) + env.set("SPACK_CXX", compiler.cxx) + env.set("CXX", os.path.join(link_dir, compiler.link_paths["cxx"])) if compiler.f77: - env.set('SPACK_F77', compiler.f77) - env.set('F77', os.path.join(link_dir, compiler.link_paths['f77'])) + env.set("SPACK_F77", compiler.f77) + env.set("F77", os.path.join(link_dir, compiler.link_paths["f77"])) if compiler.fc: - env.set('SPACK_FC', compiler.fc) - env.set('FC', os.path.join(link_dir, compiler.link_paths['fc'])) + env.set("SPACK_FC", compiler.fc) + env.set("FC", os.path.join(link_dir, compiler.link_paths["fc"])) # Set SPACK compiler rpath flags so that our wrapper knows what to use - env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg) - env.set('SPACK_CXX_RPATH_ARG', compiler.cxx_rpath_arg) - env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg) - env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg) - env.set('SPACK_LINKER_ARG', compiler.linker_arg) + env.set("SPACK_CC_RPATH_ARG", compiler.cc_rpath_arg) + env.set("SPACK_CXX_RPATH_ARG", compiler.cxx_rpath_arg) + env.set("SPACK_F77_RPATH_ARG", compiler.f77_rpath_arg) + env.set("SPACK_FC_RPATH_ARG", compiler.fc_rpath_arg) + env.set("SPACK_LINKER_ARG", compiler.linker_arg) # Check whether we want to force RPATH or RUNPATH - if spack.config.get('config:shared_linking') == 'rpath': - env.set('SPACK_DTAGS_TO_STRIP', compiler.enable_new_dtags) - env.set('SPACK_DTAGS_TO_ADD', compiler.disable_new_dtags) + if spack.config.get("config:shared_linking") == "rpath": + env.set("SPACK_DTAGS_TO_STRIP", compiler.enable_new_dtags) + env.set("SPACK_DTAGS_TO_ADD", compiler.disable_new_dtags) else: - env.set('SPACK_DTAGS_TO_STRIP', compiler.disable_new_dtags) - env.set('SPACK_DTAGS_TO_ADD', compiler.enable_new_dtags) + env.set("SPACK_DTAGS_TO_STRIP", compiler.disable_new_dtags) + env.set("SPACK_DTAGS_TO_ADD", compiler.enable_new_dtags) # Set the target parameters that the compiler will add isa_arg = spec.architecture.target.optimization_flags(compiler) - env.set('SPACK_TARGET_ARGS', isa_arg) + env.set("SPACK_TARGET_ARGS", isa_arg) # Trap spack-tracked compiler flags as appropriate. # env_flags are easy to accidentally override. @@ -333,16 +342,16 @@ def set_compiler_environment_variables(pkg, env): # Concreteness guarantees key safety here if inject_flags[flag]: # variables SPACK_<FLAG> inject flags through wrapper - var_name = 'SPACK_{0}'.format(flag.upper()) - env.set(var_name, ' '.join(f for f in inject_flags[flag])) + var_name = "SPACK_{0}".format(flag.upper()) + env.set(var_name, " ".join(f for f in inject_flags[flag])) if env_flags[flag]: # implicit variables - env.set(flag.upper(), ' '.join(f for f in env_flags[flag])) + env.set(flag.upper(), " ".join(f for f in env_flags[flag])) pkg.flags_to_build_system_args(build_system_flags) - env.set('SPACK_COMPILER_SPEC', str(spec.compiler)) + env.set("SPACK_COMPILER_SPEC", str(spec.compiler)) - env.set('SPACK_SYSTEM_DIRS', ':'.join(system_dirs)) + env.set("SPACK_SYSTEM_DIRS", ":".join(system_dirs)) compiler.setup_custom_environment(pkg, env) @@ -351,17 +360,17 @@ def set_compiler_environment_variables(pkg, env): def set_wrapper_variables(pkg, env): """Set environment variables used by the Spack compiler wrapper - (which have the prefix `SPACK_`) and also add the compiler wrappers - to PATH. - - This determines the injected -L/-I/-rpath options; each - of these specifies a search order and this function computes these - options in a manner that is intended to match the DAG traversal order - in `modifications_from_dependencies`: that method uses a post-order - traversal so that `PrependPath` actions from dependencies take lower - precedence; we use a post-order traversal here to match the visitation - order of `modifications_from_dependencies` (so we are visiting the - lowest priority packages first). + (which have the prefix `SPACK_`) and also add the compiler wrappers + to PATH. + + This determines the injected -L/-I/-rpath options; each + of these specifies a search order and this function computes these + options in a manner that is intended to match the DAG traversal order + in `modifications_from_dependencies`: that method uses a post-order + traversal so that `PrependPath` actions from dependencies take lower + precedence; we use a post-order traversal here to match the visitation + order of `modifications_from_dependencies` (so we are visiting the + lowest priority packages first). """ # Set environment variables if specified for # the given compiler @@ -369,8 +378,8 @@ def set_wrapper_variables(pkg, env): env.extend(spack.schema.environment.parse(compiler.environment)) if compiler.extra_rpaths: - extra_rpaths = ':'.join(compiler.extra_rpaths) - env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths) + extra_rpaths = ":".join(compiler.extra_rpaths) + env.set("SPACK_COMPILER_EXTRA_RPATHS", extra_rpaths) # Add spack build environment path with compiler wrappers first in # the path. We add the compiler wrapper path, which includes default @@ -384,35 +393,35 @@ def set_wrapper_variables(pkg, env): # directory. Add that to the path too. env_paths = [] compiler_specific = os.path.join( - spack.paths.build_env_path, - os.path.dirname(pkg.compiler.link_paths['cc'])) + spack.paths.build_env_path, os.path.dirname(pkg.compiler.link_paths["cc"]) + ) for item in [spack.paths.build_env_path, compiler_specific]: env_paths.append(item) - ci = os.path.join(item, 'case-insensitive') + ci = os.path.join(item, "case-insensitive") if os.path.isdir(ci): env_paths.append(ci) tty.debug("Adding compiler bin/ paths: " + " ".join(env_paths)) for item in env_paths: - env.prepend_path('PATH', item) + env.prepend_path("PATH", item) env.set_path(SPACK_ENV_PATH, env_paths) # Working directory for the spack command itself, for debug logs. - if spack.config.get('config:debug'): - env.set(SPACK_DEBUG, 'TRUE') + if spack.config.get("config:debug"): + env.set(SPACK_DEBUG, "TRUE") env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) - env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format('{name}-{hash:7}')) + env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}")) env.set(SPACK_DEBUG_LOG_DIR, spack.main.spack_working_dir) # Find ccache binary and hand it to build environment - if spack.config.get('config:ccache'): - ccache = Executable('ccache') + if spack.config.get("config:ccache"): + ccache = Executable("ccache") if not ccache: raise RuntimeError("No ccache binary found in PATH") env.set(SPACK_CCACHE_BINARY, ccache) # Gather information about various types of dependencies - link_deps = set(pkg.spec.traverse(root=False, deptype=('link'))) + link_deps = set(pkg.spec.traverse(root=False, deptype=("link"))) rpath_deps = get_rpath_deps(pkg) link_dirs = [] @@ -434,9 +443,8 @@ def set_wrapper_variables(pkg, env): except NoLibrariesError: tty.debug("No libraries found for {0}".format(dep.name)) - for default_lib_dir in ['lib', 'lib64']: - default_lib_prefix = os.path.join( - dep.prefix, default_lib_dir) + for default_lib_dir in ["lib", "lib64"]: + default_lib_prefix = os.path.join(dep.prefix, default_lib_dir) if os.path.isdir(default_lib_prefix): dep_link_dirs.append(default_lib_prefix) @@ -449,21 +457,21 @@ def set_wrapper_variables(pkg, env): except NoHeadersError: tty.debug("No headers found for {0}".format(dep.name)) - for dspec in pkg.spec.traverse(root=False, order='post'): + for dspec in pkg.spec.traverse(root=False, order="post"): if dspec.external: update_compiler_args_for_dep(dspec) # Just above, we prepended entries for -L/-rpath for externals. We # now do this for non-external packages so that Spack-built packages # are searched first for libraries etc. - for dspec in pkg.spec.traverse(root=False, order='post'): + for dspec in pkg.spec.traverse(root=False, order="post"): if not dspec.external: update_compiler_args_for_dep(dspec) # The top-level package is always RPATHed. It hasn't been installed yet # so the RPATHs are added unconditionally (e.g. even though lib64/ may # not be created for the install). - for libdir in ['lib64', 'lib']: + for libdir in ["lib64", "lib"]: lib_path = os.path.join(pkg.prefix, libdir) rpath_dirs.insert(0, lib_path) @@ -471,13 +479,14 @@ def set_wrapper_variables(pkg, env): include_dirs = list(dedupe(filter_system_paths(include_dirs))) rpath_dirs = list(dedupe(filter_system_paths(rpath_dirs))) - env.set(SPACK_LINK_DIRS, ':'.join(link_dirs)) - env.set(SPACK_INCLUDE_DIRS, ':'.join(include_dirs)) - env.set(SPACK_RPATH_DIRS, ':'.join(rpath_dirs)) + env.set(SPACK_LINK_DIRS, ":".join(link_dirs)) + env.set(SPACK_INCLUDE_DIRS, ":".join(include_dirs)) + env.set(SPACK_RPATH_DIRS, ":".join(rpath_dirs)) def determine_number_of_jobs( - parallel=False, command_line=None, config_default=None, max_cpus=None): + parallel=False, command_line=None, config_default=None, max_cpus=None +): """ Packages that require sequential builds need 1 job. Otherwise we use the number of jobs set on the command line. If not set, then we use the config @@ -494,8 +503,8 @@ def determine_number_of_jobs( if not parallel: return 1 - if command_line is None and 'command_line' in spack.config.scopes(): - command_line = spack.config.get('config:build_jobs', scope='command_line') + if command_line is None and "command_line" in spack.config.scopes(): + command_line = spack.config.get("config:build_jobs", scope="command_line") if command_line is not None: return command_line @@ -503,7 +512,7 @@ def determine_number_of_jobs( max_cpus = max_cpus or cpus_available() # in some rare cases _builtin config may not be set, so default to max 16 - config_default = config_default or spack.config.get('config:build_jobs', 16) + config_default = config_default or spack.config.get("config:build_jobs", 16) return min(max_cpus, config_default) @@ -512,7 +521,7 @@ def _set_variables_for_single_module(pkg, module): """Helper function to set module variables for single module.""" # Put a marker on this module so that it won't execute the body of this # function again, since it is not needed - marker = '_set_run_already_called' + marker = "_set_run_already_called" if getattr(module, marker, False): return @@ -522,24 +531,24 @@ def _set_variables_for_single_module(pkg, module): m.make_jobs = jobs # TODO: make these build deps that can be installed if not found. - m.make = MakeExecutable('make', jobs) - m.gmake = MakeExecutable('gmake', jobs) - m.scons = MakeExecutable('scons', jobs) - m.ninja = MakeExecutable('ninja', jobs) + m.make = MakeExecutable("make", jobs) + m.gmake = MakeExecutable("gmake", jobs) + m.scons = MakeExecutable("scons", jobs) + m.ninja = MakeExecutable("ninja", jobs) # easy shortcut to os.environ m.env = os.environ # Find the configure script in the archive path # Don't use which for this; we want to find it in the current dir. - m.configure = Executable('./configure') + m.configure = Executable("./configure") - m.meson = Executable('meson') - m.cmake = Executable('cmake') - m.ctest = MakeExecutable('ctest', jobs) + m.meson = Executable("meson") + m.cmake = Executable("cmake") + m.ctest = MakeExecutable("ctest", jobs) - if sys.platform == 'win32': - m.nmake = Executable('nmake') + if sys.platform == "win32": + m.nmake = Executable("nmake") # Standard CMake arguments m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg) m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg) @@ -547,10 +556,10 @@ def _set_variables_for_single_module(pkg, module): # Put spack compiler paths in module scope. link_dir = spack.paths.build_env_path - m.spack_cc = os.path.join(link_dir, pkg.compiler.link_paths['cc']) - m.spack_cxx = os.path.join(link_dir, pkg.compiler.link_paths['cxx']) - m.spack_f77 = os.path.join(link_dir, pkg.compiler.link_paths['f77']) - m.spack_fc = os.path.join(link_dir, pkg.compiler.link_paths['fc']) + m.spack_cc = os.path.join(link_dir, pkg.compiler.link_paths["cc"]) + m.spack_cxx = os.path.join(link_dir, pkg.compiler.link_paths["cxx"]) + m.spack_f77 = os.path.join(link_dir, pkg.compiler.link_paths["f77"]) + m.spack_fc = os.path.join(link_dir, pkg.compiler.link_paths["fc"]) # Emulate some shell commands for convenience m.pwd = os.getcwd @@ -575,11 +584,12 @@ def _set_variables_for_single_module(pkg, module): m.dso_suffix = dso_suffix def static_to_shared_library(static_lib, shared_lib=None, **kwargs): - compiler_path = kwargs.get('compiler', m.spack_cc) + compiler_path = kwargs.get("compiler", m.spack_cc) compiler = Executable(compiler_path) - return _static_to_shared_library(pkg.spec.architecture, compiler, - static_lib, shared_lib, **kwargs) + return _static_to_shared_library( + pkg.spec.architecture, compiler, static_lib, shared_lib, **kwargs + ) m.static_to_shared_library = static_to_shared_library @@ -590,7 +600,7 @@ def _set_variables_for_single_module(pkg, module): def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. - This makes things easier for package writers. + This makes things easier for package writers. """ # If a user makes their own package repo, e.g. # spack.pkg.mystuff.libelf.Libelf, and they inherit from an existing class @@ -602,8 +612,7 @@ def set_module_variables_for_package(pkg): _set_variables_for_single_module(pkg, mod) -def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, - **kwargs): +def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, **kwargs): """ Converts a static library to a shared library. The static library has to be built with PIC for the conversion to work. @@ -621,50 +630,49 @@ def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, compat_version (str): Library compatibility version. Default is version. """ - compiler_output = kwargs.get('compiler_output', None) - arguments = kwargs.get('arguments', []) - version = kwargs.get('version', None) - compat_version = kwargs.get('compat_version', version) + compiler_output = kwargs.get("compiler_output", None) + arguments = kwargs.get("arguments", []) + version = kwargs.get("version", None) + compat_version = kwargs.get("compat_version", version) if not shared_lib: - shared_lib = '{0}.{1}'.format(os.path.splitext(static_lib)[0], - dso_suffix) + shared_lib = "{0}.{1}".format(os.path.splitext(static_lib)[0], dso_suffix) compiler_args = [] # TODO: Compiler arguments should not be hardcoded but provided by # the different compiler classes. - if 'linux' in arch or 'cray' in arch: + if "linux" in arch or "cray" in arch: soname = os.path.basename(shared_lib) if compat_version: - soname += '.{0}'.format(compat_version) + soname += ".{0}".format(compat_version) compiler_args = [ - '-shared', - '-Wl,-soname,{0}'.format(soname), - '-Wl,--whole-archive', + "-shared", + "-Wl,-soname,{0}".format(soname), + "-Wl,--whole-archive", static_lib, - '-Wl,--no-whole-archive' + "-Wl,--no-whole-archive", ] - elif 'darwin' in arch: + elif "darwin" in arch: install_name = shared_lib if compat_version: - install_name += '.{0}'.format(compat_version) + install_name += ".{0}".format(compat_version) compiler_args = [ - '-dynamiclib', - '-install_name', '{0}'.format(install_name), - '-Wl,-force_load,{0}'.format(static_lib) + "-dynamiclib", + "-install_name", + "{0}".format(install_name), + "-Wl,-force_load,{0}".format(static_lib), ] if compat_version: - compiler_args.extend(['-compatibility_version', '{0}'.format( - compat_version)]) + compiler_args.extend(["-compatibility_version", "{0}".format(compat_version)]) if version: - compiler_args.extend(['-current_version', '{0}'.format(version)]) + compiler_args.extend(["-current_version", "{0}".format(version)]) if len(arguments) > 0: compiler_args.extend(arguments) @@ -672,11 +680,11 @@ def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, shared_lib_base = shared_lib if version: - shared_lib += '.{0}'.format(version) + shared_lib += ".{0}".format(version) elif compat_version: - shared_lib += '.{0}'.format(compat_version) + shared_lib += ".{0}".format(compat_version) - compiler_args.extend(['-o', shared_lib]) + compiler_args.extend(["-o", shared_lib]) # Create symlinks for version and compat_version shared_lib_link = os.path.basename(shared_lib) @@ -685,8 +693,7 @@ def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, symlink(shared_lib_link, shared_lib_base) if compat_version and compat_version != version: - symlink(shared_lib_link, '{0}.{1}'.format(shared_lib_base, - compat_version)) + symlink(shared_lib_link, "{0}.{1}".format(shared_lib_base, compat_version)) return compiler(*compiler_args, output=compiler_output) @@ -694,19 +701,17 @@ def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, def get_rpath_deps(pkg): """Return immediate or transitive RPATHs depending on the package.""" if pkg.transitive_rpaths: - return [d for d in pkg.spec.traverse(root=False, deptype=('link'))] + return [d for d in pkg.spec.traverse(root=False, deptype=("link"))] else: - return pkg.spec.dependencies(deptype='link') + return pkg.spec.dependencies(deptype="link") def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" rpaths = [pkg.prefix.lib, pkg.prefix.lib64] deps = get_rpath_deps(pkg) - rpaths.extend(d.prefix.lib for d in deps - if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in deps - if os.path.isdir(d.prefix.lib64)) + rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib)) + rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. if pkg.compiler.modules and len(pkg.compiler.modules) > 1: @@ -752,8 +757,9 @@ def parent_class_modules(cls): Includes cls.__module__ """ - if (not issubclass(cls, spack.package_base.PackageBase) or - issubclass(spack.package_base.PackageBase, cls)): + if not issubclass(cls, spack.package_base.PackageBase) or issubclass( + spack.package_base.PackageBase, cls + ): return [] result = [] module = sys.modules.get(cls.__module__) @@ -779,12 +785,10 @@ def load_external_modules(pkg): load_module(external_module) -def setup_package(pkg, dirty, context='build'): +def setup_package(pkg, dirty, context="build"): """Execute all environment setup routines.""" - if context not in ['build', 'test']: - raise ValueError( - "'context' must be one of ['build', 'test'] - got: {0}" - .format(context)) + if context not in ["build", "test"]: + raise ValueError("'context' must be one of ['build', 'test'] - got: {0}".format(context)) set_module_variables_for_package(pkg) @@ -794,37 +798,37 @@ def setup_package(pkg, dirty, context='build'): env_mods = EnvironmentModifications() # setup compilers for build contexts - need_compiler = context == 'build' or (context == 'test' and - pkg.test_requires_compiler) + need_compiler = context == "build" or (context == "test" and pkg.test_requires_compiler) if need_compiler: set_compiler_environment_variables(pkg, env_mods) set_wrapper_variables(pkg, env_mods) - env_mods.extend(modifications_from_dependencies( - pkg.spec, context, custom_mods_only=False)) + env_mods.extend(modifications_from_dependencies(pkg.spec, context, custom_mods_only=False)) # architecture specific setup platform = spack.platforms.by_name(pkg.spec.architecture.platform) target = platform.target(pkg.spec.architecture.target) platform.setup_platform_environment(pkg, env_mods) - if context == 'build': + if context == "build": pkg.setup_build_environment(env_mods) - if (not dirty) and (not env_mods.is_unset('CPATH')): - tty.debug("A dependency has updated CPATH, this may lead pkg-" - "config to assume that the package is part of the system" - " includes and omit it when invoked with '--cflags'.") - elif context == 'test': + if (not dirty) and (not env_mods.is_unset("CPATH")): + tty.debug( + "A dependency has updated CPATH, this may lead pkg-" + "config to assume that the package is part of the system" + " includes and omit it when invoked with '--cflags'." + ) + elif context == "test": env_mods.extend( inspect_path( pkg.spec.prefix, spack.user_environment.prefix_inspections(pkg.spec.platform), - exclude=is_system_path + exclude=is_system_path, ) ) pkg.setup_run_environment(env_mods) - env_mods.prepend_path('PATH', '.') + env_mods.prepend_path("PATH", ".") # First apply the clean environment changes env_base.apply_modifications() @@ -840,8 +844,8 @@ def setup_package(pkg, dirty, context='build'): # unnecessary on_cray, _ = _on_cray() if on_cray and not dirty: - for mod in ['cray-mpich', 'cray-libsci']: - module('unload', mod) + for mod in ["cray-mpich", "cray-libsci"]: + module("unload", mod) if target.module_name: load_module(target.module_name) @@ -850,8 +854,7 @@ def setup_package(pkg, dirty, context='build'): implicit_rpaths = pkg.compiler.implicit_rpaths() if implicit_rpaths: - env_mods.set('SPACK_COMPILER_IMPLICIT_RPATHS', - ':'.join(implicit_rpaths)) + env_mods.set("SPACK_COMPILER_IMPLICIT_RPATHS", ":".join(implicit_rpaths)) # Make sure nothing's strange about the Spack environment. validate(env_mods, tty.warn) @@ -867,14 +870,14 @@ def _make_runnable(pkg, env): # environment variable prefix = pkg.prefix - for dirname in ['bin', 'bin64']: + for dirname in ["bin", "bin64"]: bin_dir = os.path.join(prefix, dirname) if os.path.isdir(bin_dir): - env.prepend_path('PATH', bin_dir) + env.prepend_path("PATH", bin_dir) def modifications_from_dependencies( - spec, context, custom_mods_only=True, set_package_py_globals=True + spec, context, custom_mods_only=True, set_package_py_globals=True ): """Returns the environment modifications that are required by the dependencies of a spec and also applies modifications @@ -914,60 +917,57 @@ def modifications_from_dependencies( package.py files (this may be problematic when using buildcaches that have been built on a different but compatible OS) """ - if context not in ['build', 'run', 'test']: + if context not in ["build", "run", "test"]: raise ValueError( - "Expecting context to be one of ['build', 'run', 'test'], " - "got: {0}".format(context)) + "Expecting context to be one of ['build', 'run', 'test'], " "got: {0}".format(context) + ) env = EnvironmentModifications() # Note: see computation of 'custom_mod_deps' and 'exe_deps' later in this # function; these sets form the building blocks of those collections. - build_deps = set(spec.dependencies(deptype=('build', 'test'))) - link_deps = set(spec.traverse(root=False, deptype='link')) + build_deps = set(spec.dependencies(deptype=("build", "test"))) + link_deps = set(spec.traverse(root=False, deptype="link")) build_link_deps = build_deps | link_deps build_and_supporting_deps = set() for build_dep in build_deps: - build_and_supporting_deps.update(build_dep.traverse(deptype='run')) - run_and_supporting_deps = set( - spec.traverse(root=False, deptype=('run', 'link'))) + build_and_supporting_deps.update(build_dep.traverse(deptype="run")) + run_and_supporting_deps = set(spec.traverse(root=False, deptype=("run", "link"))) test_and_supporting_deps = set() - for test_dep in set(spec.dependencies(deptype='test')): - test_and_supporting_deps.update(test_dep.traverse(deptype='run')) + for test_dep in set(spec.dependencies(deptype="test")): + test_and_supporting_deps.update(test_dep.traverse(deptype="run")) # All dependencies that might have environment modifications to apply custom_mod_deps = set() - if context == 'build': + if context == "build": custom_mod_deps.update(build_and_supporting_deps) # Tests may be performed after build custom_mod_deps.update(test_and_supporting_deps) else: # test/run context custom_mod_deps.update(run_and_supporting_deps) - if context == 'test': + if context == "test": custom_mod_deps.update(test_and_supporting_deps) custom_mod_deps.update(link_deps) # Determine 'exe_deps': the set of packages with binaries we want to use - if context == 'build': + if context == "build": exe_deps = build_and_supporting_deps | test_and_supporting_deps - elif context == 'run': - exe_deps = set(spec.traverse(deptype='run')) - elif context == 'test': + elif context == "run": + exe_deps = set(spec.traverse(deptype="run")) + elif context == "test": exe_deps = test_and_supporting_deps def default_modifications_for_dep(dep): - if (dep in build_link_deps and - not is_system_path(dep.prefix) and - context == 'build'): + if dep in build_link_deps and not is_system_path(dep.prefix) and context == "build": prefix = dep.prefix - env.prepend_path('CMAKE_PREFIX_PATH', prefix) + env.prepend_path("CMAKE_PREFIX_PATH", prefix) - for directory in ('lib', 'lib64', 'share'): - pcdir = os.path.join(prefix, directory, 'pkgconfig') + for directory in ("lib", "lib64", "share"): + pcdir = os.path.join(prefix, directory, "pkgconfig") if os.path.isdir(pcdir): - env.prepend_path('PKG_CONFIG_PATH', pcdir) + env.prepend_path("PKG_CONFIG_PATH", pcdir) if dep in exe_deps and not is_system_path(dep.prefix): _make_runnable(dep, env) @@ -990,7 +990,7 @@ def modifications_from_dependencies( set_module_variables_for_package(dpkg) # Allow dependencies to modify the module dpkg.setup_dependent_package(spec.package.module, spec) - if context == 'build': + if context == "build": dpkg.setup_dependent_build_environment(env, spec) else: dpkg.setup_dependent_run_environment(env, spec) @@ -999,11 +999,11 @@ def modifications_from_dependencies( # The Spec.traverse method provides this: i.e. in addition to # the post-order semantics, it also guarantees a fixed traversal order # among dependencies which are not constrained by post-order semantics. - for dspec in spec.traverse(root=False, order='post'): + for dspec in spec.traverse(root=False, order="post"): if dspec.external: add_modifications_for_dep(dspec) - for dspec in spec.traverse(root=False, order='post'): + for dspec in spec.traverse(root=False, order="post"): # Default env modifications for non-external packages can override # custom modifications of external packages (this can only occur # for modifications to PATH, CMAKE_PREFIX_PATH, and PKG_CONFIG_PATH) @@ -1017,14 +1017,14 @@ def get_cmake_prefix_path(pkg): # Note that unlike modifications_from_dependencies, this does not include # any edits to CMAKE_PREFIX_PATH defined in custom # setup_dependent_build_environment implementations of dependency packages - build_deps = set(pkg.spec.dependencies(deptype=('build', 'test'))) - link_deps = set(pkg.spec.traverse(root=False, deptype=('link'))) + build_deps = set(pkg.spec.dependencies(deptype=("build", "test"))) + link_deps = set(pkg.spec.traverse(root=False, deptype=("link"))) build_link_deps = build_deps | link_deps spack_built = [] externals = [] # modifications_from_dependencies updates CMAKE_PREFIX_PATH by first # prepending all externals and then all non-externals - for dspec in pkg.spec.traverse(root=False, order='post'): + for dspec in pkg.spec.traverse(root=False, order="post"): if dspec in build_link_deps: if dspec.external: externals.insert(0, dspec) @@ -1032,15 +1032,15 @@ def get_cmake_prefix_path(pkg): spack_built.insert(0, dspec) ordered_build_link_deps = spack_built + externals - build_link_prefixes = filter_system_paths( - x.prefix for x in ordered_build_link_deps) + build_link_prefixes = filter_system_paths(x.prefix for x in ordered_build_link_deps) return build_link_prefixes -def _setup_pkg_and_run(serialized_pkg, function, kwargs, child_pipe, - input_multiprocess_fd, jsfd1, jsfd2): +def _setup_pkg_and_run( + serialized_pkg, function, kwargs, child_pipe, input_multiprocess_fd, jsfd1, jsfd2 +): - context = kwargs.get('context', 'build') + context = kwargs.get("context", "build") try: # We are in the child process. Python sets sys.stdin to @@ -1053,10 +1053,11 @@ def _setup_pkg_and_run(serialized_pkg, function, kwargs, child_pipe, pkg = serialized_pkg.restore() - if not kwargs.get('fake', False): - kwargs['unmodified_env'] = os.environ.copy() - kwargs['env_modifications'] = setup_package( - pkg, dirty=kwargs.get('dirty', False), context=context) + if not kwargs.get("fake", False): + kwargs["unmodified_env"] = os.environ.copy() + kwargs["env_modifications"] = setup_package( + pkg, dirty=kwargs.get("dirty", False), context=context + ) return_value = function(pkg, kwargs) child_pipe.send(return_value) @@ -1077,25 +1078,30 @@ def _setup_pkg_and_run(serialized_pkg, function, kwargs, child_pipe, package_context = get_package_context(tb) logfile = None - if context == 'build': + if context == "build": try: - if hasattr(pkg, 'log_path'): + if hasattr(pkg, "log_path"): logfile = pkg.log_path except NameError: # 'pkg' is not defined yet pass - elif context == 'test': + elif context == "test": logfile = os.path.join( - pkg.test_suite.stage, - spack.install_test.TestSuite.test_log_name(pkg.spec)) + pkg.test_suite.stage, spack.install_test.TestSuite.test_log_name(pkg.spec) + ) # make a pickleable exception to send to parent. msg = "%s: %s" % (exc_type.__name__, str(exc)) - ce = ChildError(msg, - exc_type.__module__, - exc_type.__name__, - tb_string, logfile, context, package_context) + ce = ChildError( + msg, + exc_type.__module__, + exc_type.__name__, + tb_string, + logfile, + context, + package_context, + ) child_pipe.send(ce) finally: @@ -1152,21 +1158,28 @@ def start_build_process(pkg, function, kwargs): try: # Forward sys.stdin when appropriate, to allow toggling verbosity - if sys.platform != "win32" and sys.stdin.isatty() and hasattr(sys.stdin, - 'fileno'): + if sys.platform != "win32" and sys.stdin.isatty() and hasattr(sys.stdin, "fileno"): input_fd = os.dup(sys.stdin.fileno()) input_multiprocess_fd = MultiProcessFd(input_fd) - mflags = os.environ.get('MAKEFLAGS', False) + mflags = os.environ.get("MAKEFLAGS", False) if mflags: - m = re.search(r'--jobserver-[^=]*=(\d),(\d)', mflags) + m = re.search(r"--jobserver-[^=]*=(\d),(\d)", mflags) if m: jobserver_fd1 = MultiProcessFd(int(m.group(1))) jobserver_fd2 = MultiProcessFd(int(m.group(2))) p = multiprocessing.Process( target=_setup_pkg_and_run, - args=(serialized_pkg, function, kwargs, child_pipe, - input_multiprocess_fd, jobserver_fd1, jobserver_fd2)) + args=( + serialized_pkg, + function, + kwargs, + child_pipe, + input_multiprocess_fd, + jobserver_fd1, + jobserver_fd2, + ), + ) p.start() @@ -1217,6 +1230,7 @@ def get_package_context(traceback, context=3): from there. """ + def make_stack(tb, stack=None): """Tracebacks come out of the system in caller -> callee order. Return an array in callee -> caller order so we can traverse it.""" @@ -1231,19 +1245,19 @@ def get_package_context(traceback, context=3): for tb in stack: frame = tb.tb_frame - if 'self' in frame.f_locals: + if "self" in frame.f_locals: # Find the first proper subclass of PackageBase. - obj = frame.f_locals['self'] + obj = frame.f_locals["self"] if isinstance(obj, spack.package_base.PackageBase): break # We found obj, the Package implementation we care about. # Point out the location in the install method where we failed. lines = [ - '{0}:{1:d}, in {2}:'.format( + "{0}:{1:d}, in {2}:".format( inspect.getfile(frame.f_code), frame.f_lineno - 1, # subtract 1 because f_lineno is 0-indexed - frame.f_code.co_name + frame.f_code.co_name, ) ] @@ -1254,16 +1268,15 @@ def get_package_context(traceback, context=3): # Subtract 1 because f_lineno is 0-indexed. fun_lineno = frame.f_lineno - start - 1 start_ctx = max(0, fun_lineno - context) - sourcelines = sourcelines[start_ctx:fun_lineno + context + 1] + sourcelines = sourcelines[start_ctx : fun_lineno + context + 1] for i, line in enumerate(sourcelines): is_error = start_ctx + i == fun_lineno - mark = '>> ' if is_error else ' ' + mark = ">> " if is_error else " " # Add start to get lineno relative to start of file, not function. - marked = ' {0}{1:-6d}{2}'.format( - mark, start + start_ctx + i, line.rstrip()) + marked = " {0}{1:-6d}{2}".format(mark, start + start_ctx + i, line.rstrip()) if is_error: - marked = colorize('@R{%s}' % cescape(marked)) + marked = colorize("@R{%s}" % cescape(marked)) lines.append(marked) return lines @@ -1311,12 +1324,12 @@ class ChildError(InstallError): mode with spack -d. """ + # List of errors considered "build errors", for which we'll show log # context instead of Python context. - build_errors = [('spack.util.executable', 'ProcessError')] + build_errors = [("spack.util.executable", "ProcessError")] - def __init__(self, msg, module, classname, traceback_string, log_name, - log_type, context): + def __init__(self, msg, module, classname, traceback_string, log_name, log_type, context): super(ChildError, self).__init__(msg) self.module = module self.name = classname @@ -1328,7 +1341,7 @@ class ChildError(InstallError): @property def long_message(self): out = StringIO() - out.write(self._long_message if self._long_message else '') + out.write(self._long_message if self._long_message else "") have_log = self.log_name and os.path.exists(self.log_name) @@ -1342,16 +1355,16 @@ class ChildError(InstallError): # The error happened in the Python code, so try to show # some context from the Package itself. if self.context: - out.write('\n') - out.write('\n'.join(self.context)) - out.write('\n') + out.write("\n") + out.write("\n".join(self.context)) + out.write("\n") if out.getvalue(): - out.write('\n') + out.write("\n") if have_log: - out.write('See {0} log for details:\n'.format(self.log_type)) - out.write(' {0}\n'.format(self.log_name)) + out.write("See {0} log for details:\n".format(self.log_type)) + out.write(" {0}\n".format(self.log_name)) return out.getvalue() @@ -1371,7 +1384,8 @@ class ChildError(InstallError): self.traceback, self.log_name, self.log_type, - self.context) + self.context, + ) def _make_child_error(msg, module, name, traceback, log, log_type, context): @@ -1381,6 +1395,7 @@ def _make_child_error(msg, module, name, traceback, log, log_type, context): class StopPhase(spack.error.SpackError): """Pickle-able exception to control stopped builds.""" + def __reduce__(self): return _make_stop_phase, (self.message, self.long_message) @@ -1400,9 +1415,7 @@ def write_log_summary(out, log_type, log, last=None): nerr = last # If errors are found, only display errors - out.write( - "\n%s found in %s log:\n" % - (plural(nerr, 'error'), log_type)) + out.write("\n%s found in %s log:\n" % (plural(nerr, "error"), log_type)) out.write(make_log_context(errors)) elif nwar > 0: if last and nwar > last: @@ -1410,7 +1423,5 @@ def write_log_summary(out, log_type, log, last=None): nwar = last # If no errors are found but warnings are, display warnings - out.write( - "\n%s found in %s log:\n" % - (plural(nwar, 'warning'), log_type)) + out.write("\n%s found in %s log:\n" % (plural(nwar, "warning"), log_type)) out.write(make_log_context(warnings)) |