From 39c9bbfbbb26d35be44fa69b2b7d536f9539d435 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 15 Aug 2018 22:55:51 -0700 Subject: imports: spack uses importlib instead of imp when available - `imp` is deprecated and seems to have started having some weird issues on certain Linux versions. - In particular, the file argument to `load_source` is ignored on arch linux with Python 3.7. - `imp` is the only way to do imports in 2.6, so we'll keep it around for now and use it if importlib won't work. - `importlib` is the new import system, and it allows us to get lower-level access to the import implementation. - This consolidates all import logic into `spack.util.imp`, and make it use `importlib` if it's avialable. --- lib/spack/spack/compilers/__init__.py | 5 +- lib/spack/spack/directives.py | 7 +-- lib/spack/spack/hooks/__init__.py | 4 +- lib/spack/spack/repo.py | 34 ++-------- lib/spack/spack/util/imp/__init__.py | 41 ++++++++++++ lib/spack/spack/util/imp/imp_importer.py | 86 ++++++++++++++++++++++++++ lib/spack/spack/util/imp/importlib_importer.py | 61 ++++++++++++++++++ 7 files changed, 199 insertions(+), 39 deletions(-) create mode 100644 lib/spack/spack/util/imp/__init__.py create mode 100644 lib/spack/spack/util/imp/imp_importer.py create mode 100644 lib/spack/spack/util/imp/importlib_importer.py (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 45245e56c4..e5d03c990a 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -26,7 +26,6 @@ system and configuring Spack to use multiple compilers. """ import os -import imp from llnl.util.lang import list_modules @@ -35,7 +34,7 @@ import spack.error import spack.spec import spack.config import spack.architecture - +import spack.util.imp as simp from spack.util.naming import mod_to_class _imported_compilers_module = 'spack.compilers' @@ -361,7 +360,7 @@ def class_for_compiler_name(compiler_name): assert(supported(compiler_name)) file_path = os.path.join(spack.paths.compilers_path, compiler_name + ".py") - compiler_mod = imp.load_source(_imported_compilers_module, file_path) + compiler_mod = simp.load_source(_imported_compilers_module, file_path) cls = getattr(compiler_mod, mod_to_class(compiler_name)) # make a note of the name in the module so we can get to it easily. diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 5137320e98..bd843ee8d6 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -48,7 +48,6 @@ The available directives are: import collections import functools -import inspect import os.path import re from six import string_types @@ -115,11 +114,11 @@ class DirectiveMeta(type): def __init__(cls, name, bases, attr_dict): # The class is being created: if it is a package we must ensure # that the directives are called on the class to set it up - module = inspect.getmodule(cls) - if 'spack.pkg' in module.__name__: + + if 'spack.pkg' in cls.__module__: # Package name as taken # from llnl.util.lang.get_calling_module_name - pkg_name = module.__name__.split('.')[-1] + pkg_name = cls.__module__.split('.')[-1] setattr(cls, 'name', pkg_name) # Ensure the presence of the dictionaries associated diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 8a4b8ada76..b23b641b77 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -41,10 +41,10 @@ systems (e.g. modules, dotkit, etc.) or to add other custom features. """ -import imp import os.path import spack.paths +import spack.util.imp as simp from llnl.util.lang import memoized, list_modules @@ -54,7 +54,7 @@ def all_hook_modules(): for name in list_modules(spack.paths.hooks_path): mod_name = __name__ + '.' + name path = os.path.join(spack.paths.hooks_path, name) + ".py" - mod = imp.load_source(mod_name, path) + mod = simp.load_source(mod_name, path) modules.append(mod) return modules diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index ec09262541..980fb0cbab 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -29,10 +29,8 @@ import shutil import errno import sys import inspect -import imp import re import traceback -import tempfile import json from contextlib import contextmanager from six import string_types @@ -54,11 +52,13 @@ import spack.config import spack.caches import spack.error import spack.spec +import spack.util.imp as simp from spack.provider_index import ProviderIndex from spack.util.path import canonicalize_path from spack.util.naming import NamespaceTrie, valid_module_name from spack.util.naming import mod_to_class, possible_spack_module_names + #: Super-namespace for all packages. #: Package modules are imported as spack.pkg... repo_namespace = 'spack.pkg' @@ -994,9 +994,8 @@ class Repo(object): fullname = "%s.%s" % (self.full_namespace, pkg_name) try: - with import_lock(): - with prepend_open(file_path, text=_package_prepend) as f: - module = imp.load_source(fullname, file_path, f) + module = simp.load_source(fullname, file_path, + prepend=_package_prepend) except SyntaxError as e: # SyntaxError strips the path from the filename so we need to # manually construct the error message in order to give the @@ -1146,31 +1145,6 @@ def set_path(repo): return append -@contextmanager -def import_lock(): - imp.acquire_lock() - yield - imp.release_lock() - - -@contextmanager -def prepend_open(f, *args, **kwargs): - """Open a file for reading, but prepend with some text prepended - - Arguments are same as for ``open()``, with one keyword argument, - ``text``, specifying the text to prepend. - """ - text = kwargs.get('text', None) - - with open(f, *args) as f: - with tempfile.NamedTemporaryFile(mode='w+') as tf: - if text: - tf.write(text + '\n') - tf.write(f.read()) - tf.seek(0) - yield tf.file - - @contextmanager def swap(repo_path): """Temporarily use another RepoPath.""" diff --git a/lib/spack/spack/util/imp/__init__.py b/lib/spack/spack/util/imp/__init__.py new file mode 100644 index 0000000000..86392c3f30 --- /dev/null +++ b/lib/spack/spack/util/imp/__init__.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Consolidated module for all imports done by Spack. + +Many parts of Spack have to import Python code. This utility package +wraps Spack's interface with Python's import system. + +We do this because Python's import system is confusing and changes from +Python version to Python version, and we should be able to adapt our +approach to the underlying implementation. + +Currently, this uses ``importlib.machinery`` where available and ``imp`` +when ``importlib`` is not completely usable. +""" + +try: + from .importlib_importer import load_source # noqa +except ImportError: + from .imp_importer import load_source # noqa diff --git a/lib/spack/spack/util/imp/imp_importer.py b/lib/spack/spack/util/imp/imp_importer.py new file mode 100644 index 0000000000..b023517c4c --- /dev/null +++ b/lib/spack/spack/util/imp/imp_importer.py @@ -0,0 +1,86 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Implementation of Spack imports that uses imp underneath. + +``imp`` is deprecated in newer versions of Python, but is the only option +in Python 2.6. +""" +import imp +import tempfile +from contextlib import contextmanager + + +@contextmanager +def import_lock(): + imp.acquire_lock() + yield + imp.release_lock() + + +def load_source(full_name, path, prepend=None): + """Import a Python module from source. + + Load the source file and add it to ``sys.modules``. + + Args: + full_name (str): full name of the module to be loaded + path (str): path to the file that should be loaded + prepend (str, optional): some optional code to prepend to the + loaded module; e.g., can be used to inject import statements + + Returns: + (ModuleType): the loaded module + """ + with import_lock(): + if prepend is None: + return imp.load_source(full_name, path) + else: + with prepend_open(path, text=prepend) as f: + return imp.load_source(full_name, path, f) + + +@contextmanager +def prepend_open(f, *args, **kwargs): + """Open a file for reading, but prepend with some text prepended + + Arguments are same as for ``open()``, with one keyword argument, + ``text``, specifying the text to prepend. + + We have to write and read a tempfile for the ``imp``-based importer, + as the ``file`` argument to ``imp.load_source()`` requires a + low-level file handle. + + See the ``importlib``-based importer for a faster way to do this in + later versions of python. + """ + text = kwargs.get('text', None) + + with open(f, *args) as f: + with tempfile.NamedTemporaryFile(mode='w+') as tf: + if text: + tf.write(text + '\n') + tf.write(f.read()) + tf.seek(0) + yield tf.file diff --git a/lib/spack/spack/util/imp/importlib_importer.py b/lib/spack/spack/util/imp/importlib_importer.py new file mode 100644 index 0000000000..cacf959817 --- /dev/null +++ b/lib/spack/spack/util/imp/importlib_importer.py @@ -0,0 +1,61 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Implementation of Spack imports that uses importlib underneath. + +``importlib`` is only fully implemented in Python 3. +""" +from importlib.machinery import SourceFileLoader + + +class PrependFileLoader(SourceFileLoader): + def __init__(self, full_name, path, prepend=None): + super(PrependFileLoader, self).__init__(full_name, path) + self.prepend = prepend + + def get_data(self, path): + data = super(PrependFileLoader, self).get_data(path) + if path != self.path or self.prepend is None: + return data + else: + return self.prepend.encode() + b"\n" + data + + +def load_source(full_name, path, prepend=None): + """Import a Python module from source. + + Load the source file and add it to ``sys.modules``. + + Args: + full_name (str): full name of the module to be loaded + path (str): path to the file that should be loaded + prepend (str, optional): some optional code to prepend to the + loaded module; e.g., can be used to inject import statements + + Returns: + (ModuleType): the loaded module + """ + # use our custom loader + loader = PrependFileLoader(full_name, path, prepend) + return loader.load_module() -- cgit v1.2.3-70-g09d2