From 70d5d234db670532afa2644a030f488d65181249 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 24 Nov 2021 10:21:35 +0100 Subject: Update Jinja2 to v2.11.3 and MarkupSafe to v1.1.1 (#27264) --- lib/spack/external/__init__.py | 4 +- lib/spack/external/jinja2/LICENSE.rst | 28 + lib/spack/external/jinja2/__init__.py | 123 +-- lib/spack/external/jinja2/_compat.py | 67 +- lib/spack/external/jinja2/_identifier.py | 6 +- lib/spack/external/jinja2/asyncfilters.py | 70 +- lib/spack/external/jinja2/asyncsupport.py | 186 ++-- lib/spack/external/jinja2/bccache.py | 142 ++- lib/spack/external/jinja2/compiler.py | 1242 +++++++++++++++------------ lib/spack/external/jinja2/constants.py | 15 +- lib/spack/external/jinja2/debug.py | 522 +++++------ lib/spack/external/jinja2/defaults.py | 68 +- lib/spack/external/jinja2/environment.py | 592 +++++++------ lib/spack/external/jinja2/exceptions.py | 73 +- lib/spack/external/jinja2/ext.py | 387 +++++---- lib/spack/external/jinja2/filters.py | 716 +++++++++------ lib/spack/external/jinja2/idtracking.py | 58 +- lib/spack/external/jinja2/lexer.py | 775 ++++++++++------- lib/spack/external/jinja2/loaders.py | 135 +-- lib/spack/external/jinja2/meta.py | 33 +- lib/spack/external/jinja2/nativetypes.py | 248 ++---- lib/spack/external/jinja2/nodes.py | 427 +++++---- lib/spack/external/jinja2/optimizer.py | 60 +- lib/spack/external/jinja2/parser.py | 568 ++++++------ lib/spack/external/jinja2/runtime.py | 704 +++++++++------ lib/spack/external/jinja2/sandbox.py | 239 +++--- lib/spack/external/jinja2/tests.py | 145 ++-- lib/spack/external/jinja2/utils.py | 410 +++++---- lib/spack/external/jinja2/visitor.py | 14 +- lib/spack/external/markupsafe/AUTHORS | 13 - lib/spack/external/markupsafe/LICENSE | 33 - lib/spack/external/markupsafe/LICENSE.rst | 28 + lib/spack/external/markupsafe/README.rst | 138 +-- lib/spack/external/markupsafe/__init__.py | 273 +++--- lib/spack/external/markupsafe/_compat.py | 23 +- lib/spack/external/markupsafe/_constants.py | 517 ++++++----- lib/spack/external/markupsafe/_native.py | 67 +- 37 files changed, 5002 insertions(+), 4147 deletions(-) create mode 100644 lib/spack/external/jinja2/LICENSE.rst delete mode 100644 lib/spack/external/markupsafe/AUTHORS delete mode 100644 lib/spack/external/markupsafe/LICENSE create mode 100644 lib/spack/external/markupsafe/LICENSE.rst (limited to 'lib') diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index a1133e200f..1f3470b95d 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -44,7 +44,7 @@ jinja2 * Homepage: https://pypi.python.org/pypi/Jinja2 * Usage: A modern and designer-friendly templating language for Python. -* Version: 2.10 +* Version: 2.11.3 (last version supporting Python 2.7) jsonschema ---------- @@ -62,7 +62,7 @@ markupsafe * Homepage: https://pypi.python.org/pypi/MarkupSafe * Usage: Implements a XML/HTML/XHTML Markup safe string for Python. -* Version: 1.0 +* Version: 1.1.1 (last version supporting Python 2.7) py -- diff --git a/lib/spack/external/jinja2/LICENSE.rst b/lib/spack/external/jinja2/LICENSE.rst new file mode 100644 index 0000000000..c37cae49ec --- /dev/null +++ b/lib/spack/external/jinja2/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2007 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/spack/external/jinja2/__init__.py b/lib/spack/external/jinja2/__init__.py index 42aa763d57..f17866f6c4 100644 --- a/lib/spack/external/jinja2/__init__.py +++ b/lib/spack/external/jinja2/__init__.py @@ -1,83 +1,44 @@ # -*- coding: utf-8 -*- +"""Jinja is a template engine written in pure Python. It provides a +non-XML syntax that supports inline expressions and an optional +sandboxed environment. """ - jinja2 - ~~~~~~ - - Jinja2 is a template engine written in pure Python. It provides a - Django inspired non-XML syntax but supports inline expressions and - an optional sandboxed environment. - - Nutshell - -------- - - Here a small example of a Jinja2 template:: - - {% extends 'base.html' %} - {% block title %}Memberlist{% endblock %} - {% block content %} - - {% endblock %} - - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -__docformat__ = 'restructuredtext en' -__version__ = '2.10' - -# high level interface -from jinja2.environment import Environment, Template - -# loaders -from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ - DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ - ModuleLoader - -# bytecode caches -from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ - MemcachedBytecodeCache - -# undefined types -from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \ - make_logging_undefined - -# exceptions -from jinja2.exceptions import TemplateError, UndefinedError, \ - TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \ - TemplateAssertionError, TemplateRuntimeError - -# decorators and public utilities -from jinja2.filters import environmentfilter, contextfilter, \ - evalcontextfilter -from jinja2.utils import Markup, escape, clear_caches, \ - environmentfunction, evalcontextfunction, contextfunction, \ - is_undefined, select_autoescape - -__all__ = [ - 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', - 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader', - 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', - 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', - 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', - 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', - 'TemplateRuntimeError', - 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', - 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined', - 'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined', - 'select_autoescape', -] - - -def _patch_async(): - from jinja2.utils import have_async_gen - if have_async_gen: - from jinja2.asyncsupport import patch_all - patch_all() - - -_patch_async() -del _patch_async +from markupsafe import escape +from markupsafe import Markup + +from .bccache import BytecodeCache +from .bccache import FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache +from .environment import Environment +from .environment import Template +from .exceptions import TemplateAssertionError +from .exceptions import TemplateError +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .filters import contextfilter +from .filters import environmentfilter +from .filters import evalcontextfilter +from .loaders import BaseLoader +from .loaders import ChoiceLoader +from .loaders import DictLoader +from .loaders import FileSystemLoader +from .loaders import FunctionLoader +from .loaders import ModuleLoader +from .loaders import PackageLoader +from .loaders import PrefixLoader +from .runtime import ChainableUndefined +from .runtime import DebugUndefined +from .runtime import make_logging_undefined +from .runtime import StrictUndefined +from .runtime import Undefined +from .utils import clear_caches +from .utils import contextfunction +from .utils import environmentfunction +from .utils import evalcontextfunction +from .utils import is_undefined +from .utils import select_autoescape + +__version__ = "2.11.3" diff --git a/lib/spack/external/jinja2/_compat.py b/lib/spack/external/jinja2/_compat.py index 61d85301a4..1f044954a0 100644 --- a/lib/spack/external/jinja2/_compat.py +++ b/lib/spack/external/jinja2/_compat.py @@ -1,22 +1,12 @@ # -*- coding: utf-8 -*- -""" - jinja2._compat - ~~~~~~~~~~~~~~ - - Some py2/py3 compatibility support based on a stripped down - version of six so we don't have to depend on a specific version - of it. - - :copyright: Copyright 2013 by the Jinja team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" +# flake8: noqa +import marshal import sys PY2 = sys.version_info[0] == 2 -PYPY = hasattr(sys, 'pypy_translation_info') +PYPY = hasattr(sys, "pypy_translation_info") _identity = lambda x: x - if not PY2: unichr = chr range_type = range @@ -30,6 +20,7 @@ if not PY2: import pickle from io import BytesIO, StringIO + NativeStringIO = StringIO def reraise(tp, value, tb=None): @@ -46,6 +37,9 @@ if not PY2: implements_to_string = _identity encode_filename = _identity + marshal_dump = marshal.dump + marshal_load = marshal.load + else: unichr = unichr text_type = unicode @@ -59,11 +53,13 @@ else: import cPickle as pickle from cStringIO import StringIO as BytesIO, StringIO + NativeStringIO = BytesIO - exec('def reraise(tp, value, tb=None):\n raise tp, value, tb') + exec("def reraise(tp, value, tb=None):\n raise tp, value, tb") from itertools import imap, izip, ifilter + intern = intern def implements_iterator(cls): @@ -73,14 +69,25 @@ else: def implements_to_string(cls): cls.__unicode__ = cls.__str__ - cls.__str__ = lambda x: x.__unicode__().encode('utf-8') + cls.__str__ = lambda x: x.__unicode__().encode("utf-8") return cls def encode_filename(filename): if isinstance(filename, unicode): - return filename.encode('utf-8') + return filename.encode("utf-8") return filename + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" @@ -90,10 +97,36 @@ def with_metaclass(meta, *bases): class metaclass(type): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) try: from urllib.parse import quote_from_bytes as url_quote except ImportError: from urllib import quote as url_quote + + +try: + from collections import abc +except ImportError: + import collections as abc + + +try: + from os import fspath +except ImportError: + try: + from pathlib import PurePath + except ImportError: + PurePath = None + + def fspath(path): + if hasattr(path, "__fspath__"): + return path.__fspath__() + + # Python 3.5 doesn't have __fspath__ yet, use str. + if PurePath is not None and isinstance(path, PurePath): + return str(path) + + return path diff --git a/lib/spack/external/jinja2/_identifier.py b/lib/spack/external/jinja2/_identifier.py index 2eac35d5c3..224d5449d1 100644 --- a/lib/spack/external/jinja2/_identifier.py +++ b/lib/spack/external/jinja2/_identifier.py @@ -1,2 +1,6 @@ +import re + # generated by scripts/generate_identifier_pattern.py -pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯' +pattern = re.compile( + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 +) diff --git a/lib/spack/external/jinja2/asyncfilters.py b/lib/spack/external/jinja2/asyncfilters.py index 5c1f46d7fa..3d98dbcc00 100644 --- a/lib/spack/external/jinja2/asyncfilters.py +++ b/lib/spack/external/jinja2/asyncfilters.py @@ -1,12 +1,13 @@ from functools import wraps -from jinja2.asyncsupport import auto_aiter -from jinja2 import filters +from . import filters +from .asyncsupport import auto_aiter +from .asyncsupport import auto_await async def auto_to_seq(value): seq = [] - if hasattr(value, '__aiter__'): + if hasattr(value, "__aiter__"): async for item in value: seq.append(item) else: @@ -16,8 +17,7 @@ async def auto_to_seq(value): async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): - seq, func = filters.prepare_select_or_reject( - args, kwargs, modfunc, lookup_attr) + seq, func = filters.prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) if seq: async for item in auto_aiter(seq): if func(item): @@ -26,14 +26,19 @@ async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): def dualfilter(normal_filter, async_filter): wrap_evalctx = False - if getattr(normal_filter, 'environmentfilter', False): - is_async = lambda args: args[0].is_async + if getattr(normal_filter, "environmentfilter", False) is True: + + def is_async(args): + return args[0].is_async + wrap_evalctx = False else: - if not getattr(normal_filter, 'evalcontextfilter', False) and \ - not getattr(normal_filter, 'contextfilter', False): - wrap_evalctx = True - is_async = lambda args: args[0].environment.is_async + has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True + has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True + wrap_evalctx = not has_evalctxfilter and not has_ctxfilter + + def is_async(args): + return args[0].environment.is_async @wraps(normal_filter) def wrapper(*args, **kwargs): @@ -55,6 +60,7 @@ def dualfilter(normal_filter, async_filter): def asyncfiltervariant(original): def decorator(f): return dualfilter(original, f) + return decorator @@ -63,19 +69,22 @@ async def do_first(environment, seq): try: return await auto_aiter(seq).__anext__() except StopAsyncIteration: - return environment.undefined('No first item, sequence was empty.') + return environment.undefined("No first item, sequence was empty.") @asyncfiltervariant(filters.do_groupby) async def do_groupby(environment, value, attribute): expr = filters.make_attrgetter(environment, attribute) - return [filters._GroupTuple(key, await auto_to_seq(values)) - for key, values in filters.groupby(sorted( - await auto_to_seq(value), key=expr), expr)] + return [ + filters._GroupTuple(key, await auto_to_seq(values)) + for key, values in filters.groupby( + sorted(await auto_to_seq(value), key=expr), expr + ) + ] @asyncfiltervariant(filters.do_join) -async def do_join(eval_ctx, value, d=u'', attribute=None): +async def do_join(eval_ctx, value, d=u"", attribute=None): return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) @@ -109,7 +118,7 @@ async def do_map(*args, **kwargs): seq, func = filters.prepare_map(args, kwargs) if seq: async for item in auto_aiter(seq): - yield func(item) + yield await auto_await(func(item)) @asyncfiltervariant(filters.do_sum) @@ -118,7 +127,10 @@ async def do_sum(environment, iterable, attribute=None, start=0): if attribute is not None: func = filters.make_attrgetter(environment, attribute) else: - func = lambda x: x + + def func(x): + return x + async for item in auto_aiter(iterable): rv += func(item) return rv @@ -130,17 +142,17 @@ async def do_slice(value, slices, fill_with=None): ASYNC_FILTERS = { - 'first': do_first, - 'groupby': do_groupby, - 'join': do_join, - 'list': do_list, + "first": do_first, + "groupby": do_groupby, + "join": do_join, + "list": do_list, # we intentionally do not support do_last because that would be # ridiculous - 'reject': do_reject, - 'rejectattr': do_rejectattr, - 'map': do_map, - 'select': do_select, - 'selectattr': do_selectattr, - 'sum': do_sum, - 'slice': do_slice, + "reject": do_reject, + "rejectattr": do_rejectattr, + "map": do_map, + "select": do_select, + "selectattr": do_selectattr, + "sum": do_sum, + "slice": do_slice, } diff --git a/lib/spack/external/jinja2/asyncsupport.py b/lib/spack/external/jinja2/asyncsupport.py index b1e7b5ce9a..78ba3739d8 100644 --- a/lib/spack/external/jinja2/asyncsupport.py +++ b/lib/spack/external/jinja2/asyncsupport.py @@ -1,29 +1,27 @@ # -*- coding: utf-8 -*- +"""The code for async support. Importing this patches Jinja on supported +Python versions. """ - jinja2.asyncsupport - ~~~~~~~~~~~~~~~~~~~ - - Has all the code for async support which is implemented as a patch - for supported Python versions. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import sys import asyncio import inspect from functools import update_wrapper -from jinja2.utils import concat, internalcode, Markup -from jinja2.environment import TemplateModule -from jinja2.runtime import LoopContextBase, _last_iteration +from markupsafe import Markup + +from .environment import TemplateModule +from .runtime import LoopContext +from .utils import concat +from .utils import internalcode +from .utils import missing async def concat_async(async_gen): rv = [] + async def collect(): async for event in async_gen: rv.append(event) + await collect() return concat(rv) @@ -34,10 +32,7 @@ async def generate_async(self, *args, **kwargs): async for event in self.root_render_func(self.new_context(vars)): yield event except Exception: - exc_info = sys.exc_info() - else: - return - yield self.environment.handle_exception(exc_info, True) + yield self.environment.handle_exception() def wrap_generate_func(original_generate): @@ -48,17 +43,18 @@ def wrap_generate_func(original_generate): yield loop.run_until_complete(async_gen.__anext__()) except StopAsyncIteration: pass + def generate(self, *args, **kwargs): if not self.environment.is_async: return original_generate(self, *args, **kwargs) return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) + return update_wrapper(generate, original_generate) async def render_async(self, *args, **kwargs): if not self.environment.is_async: - raise RuntimeError('The environment was not created with async mode ' - 'enabled.') + raise RuntimeError("The environment was not created with async mode enabled.") vars = dict(*args, **kwargs) ctx = self.new_context(vars) @@ -66,8 +62,7 @@ async def render_async(self, *args, **kwargs): try: return await concat_async(self.root_render_func(ctx)) except Exception: - exc_info = sys.exc_info() - return self.environment.handle_exception(exc_info, True) + return self.environment.handle_exception() def wrap_render_func(original_render): @@ -76,6 +71,7 @@ def wrap_render_func(original_render): return original_render(self, *args, **kwargs) loop = asyncio.get_event_loop() return loop.run_until_complete(self.render_async(*args, **kwargs)) + return update_wrapper(render, original_render) @@ -109,6 +105,7 @@ def wrap_macro_invoke(original_invoke): if not self._environment.is_async: return original_invoke(self, arguments, autoescape) return async_invoke(self, arguments, autoescape) + return update_wrapper(_invoke, original_invoke) @@ -124,9 +121,9 @@ def wrap_default_module(original_default_module): @internalcode def _get_default_module(self): if self.environment.is_async: - raise RuntimeError('Template module attribute is unavailable ' - 'in async mode') + raise RuntimeError("Template module attribute is unavailable in async mode") return original_default_module(self) + return _get_default_module @@ -139,30 +136,30 @@ async def make_module_async(self, vars=None, shared=False, locals=None): def patch_template(): - from jinja2 import Template + from . import Template + Template.generate = wrap_generate_func(Template.generate) - Template.generate_async = update_wrapper( - generate_async, Template.generate_async) - Template.render_async = update_wrapper( - render_async, Template.render_async) + Template.generate_async = update_wrapper(generate_async, Template.generate_async) + Template.render_async = update_wrapper(render_async, Template.render_async) Template.render = wrap_render_func(Template.render) - Template._get_default_module = wrap_default_module( - Template._get_default_module) + Template._get_default_module = wrap_default_module(Template._get_default_module) Template._get_default_module_async = get_default_module_async Template.make_module_async = update_wrapper( - make_module_async, Template.make_module_async) + make_module_async, Template.make_module_async + ) def patch_runtime(): - from jinja2.runtime import BlockReference, Macro - BlockReference.__call__ = wrap_block_reference_call( - BlockReference.__call__) + from .runtime import BlockReference, Macro + + BlockReference.__call__ = wrap_block_reference_call(BlockReference.__call__) Macro._invoke = wrap_macro_invoke(Macro._invoke) def patch_filters(): - from jinja2.filters import FILTERS - from jinja2.asyncfilters import ASYNC_FILTERS + from .filters import FILTERS + from .asyncfilters import ASYNC_FILTERS + FILTERS.update(ASYNC_FILTERS) @@ -179,7 +176,7 @@ async def auto_await(value): async def auto_aiter(iterable): - if hasattr(iterable, '__aiter__'): + if hasattr(iterable, "__aiter__"): async for item in iterable: yield item return @@ -187,70 +184,81 @@ async def auto_aiter(iterable): yield item -class AsyncLoopContext(LoopContextBase): - - def __init__(self, async_iterator, undefined, after, length, recurse=None, - depth0=0): - LoopContextBase.__init__(self, undefined, recurse, depth0) - self._async_iterator = async_iterator - self._after = after - self._length = length +class AsyncLoopContext(LoopContext): + _to_iterator = staticmethod(auto_aiter) @property - def length(self): - if self._length is None: - raise TypeError('Loop length for some iterators cannot be ' - 'lazily calculated in async mode') + async def length(self): + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = [x async for x in self._iterator] + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + return self._length - def __aiter__(self): - return AsyncLoopContextIterator(self) + @property + async def revindex0(self): + return await self.length - self.index + @property + async def revindex(self): + return await self.length - self.index0 + + async def _peek_next(self): + if self._after is not missing: + return self._after + + try: + self._after = await self._iterator.__anext__() + except StopAsyncIteration: + self._after = missing -class AsyncLoopContextIterator(object): - __slots__ = ('context',) + return self._after - def __init__(self, context): - self.context = context + @property + async def last(self): + return await self._peek_next() is missing + + @property + async def nextitem(self): + rv = await self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv def __aiter__(self): return self async def __anext__(self): - ctx = self.context - ctx.index0 += 1 - if ctx._after is _last_iteration: - raise StopAsyncIteration() - ctx._before = ctx._current - ctx._current = ctx._after - try: - ctx._after = await ctx._async_iterator.__anext__() - except StopAsyncIteration: - ctx._after = _last_iteration - return ctx._current, ctx + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = await self._iterator.__anext__() + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): - # Length is more complicated and less efficient in async mode. The - # reason for this is that we cannot know if length will be used - # upfront but because length is a property we cannot lazily execute it - # later. This means that we need to buffer it up and measure :( - # - # We however only do this for actual iterators, not for async - # iterators as blocking here does not seem like the best idea in the - # world. - try: - length = len(iterable) - except (TypeError, AttributeError): - if not hasattr(iterable, '__aiter__'): - iterable = tuple(iterable) - length = len(iterable) - else: - length = None - async_iterator = auto_aiter(iterable) - try: - after = await async_iterator.__anext__() - except StopAsyncIteration: - after = _last_iteration - return AsyncLoopContext(async_iterator, undefined, after, length, recurse, - depth0) + import warnings + + warnings.warn( + "This template must be recompiled with at least Jinja 2.11, or" + " it will fail in 3.0.", + DeprecationWarning, + stacklevel=2, + ) + return AsyncLoopContext(iterable, undefined, recurse, depth0) + + +patch_all() diff --git a/lib/spack/external/jinja2/bccache.py b/lib/spack/external/jinja2/bccache.py index 080e527cab..9c0661030f 100644 --- a/lib/spack/external/jinja2/bccache.py +++ b/lib/spack/external/jinja2/bccache.py @@ -1,60 +1,37 @@ # -*- coding: utf-8 -*- -""" - jinja2.bccache - ~~~~~~~~~~~~~~ - - This module implements the bytecode cache system Jinja is optionally - using. This is useful if you have very complex template situations and - the compiliation of all those templates slow down your application too - much. - - Situations where this is useful are often forking web applications that - are initialized on the first request. +"""The optional bytecode cache system. This is useful if you have very +complex template situations and the compilation of all those templates +slows down your application too much. - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. +Situations where this is useful are often forking web applications that +are initialized on the first request. """ -from os import path, listdir +import errno +import fnmatch import os -import sys import stat -import errno -import marshal +import sys import tempfile -import fnmatch from hashlib import sha1 -from jinja2.utils import open_if_exists -from jinja2._compat import BytesIO, pickle, PY2, text_type - - -# marshal works better on 3.x, one hack less required -if not PY2: - marshal_dump = marshal.dump - marshal_load = marshal.load -else: - - def marshal_dump(code, f): - if isinstance(f, file): - marshal.dump(code, f) - else: - f.write(marshal.dumps(code)) - - def marshal_load(f): - if isinstance(f, file): - return marshal.load(f) - return marshal.loads(f.read()) - - -bc_version = 3 - -# magic version used to only change with new jinja versions. With 2.6 -# we change this to also take Python version changes into account. The -# reason for this is that Python tends to segfault if fed earlier bytecode -# versions because someone thought it would be a good idea to reuse opcodes -# or make Python incompatible with earlier versions. -bc_magic = 'j2'.encode('ascii') + \ - pickle.dumps(bc_version, 2) + \ - pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) +from os import listdir +from os import path + +from ._compat import BytesIO +from ._compat import marshal_dump +from ._compat import marshal_load +from ._compat import pickle +from ._compat import text_type +from .utils import open_if_exists + +bc_version = 4 +# Magic bytes to identify Jinja bytecode cache files. Contains the +# Python major and minor version to avoid loading incompatible bytecode +# if a project upgrades its Python version. +bc_magic = ( + b"j2" + + pickle.dumps(bc_version, 2) + + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) +) class Bucket(object): @@ -98,7 +75,7 @@ class Bucket(object): def write_bytecode(self, f): """Dump the bytecode into the file or file like object passed.""" if self.code is None: - raise TypeError('can\'t write empty bucket') + raise TypeError("can't write empty bucket") f.write(bc_magic) pickle.dump(self.checksum, f, 2) marshal_dump(self.code, f) @@ -140,7 +117,7 @@ class BytecodeCache(object): bucket.write_bytecode(f) A more advanced version of a filesystem based bytecode cache is part of - Jinja2. + Jinja. """ def load_bytecode(self, bucket): @@ -158,24 +135,24 @@ class BytecodeCache(object): raise NotImplementedError() def clear(self): - """Clears the cache. This method is not used by Jinja2 but should be + """Clears the cache. This method is not used by Jinja but should be implemented to allow applications to clear the bytecode cache used by a particular environment. """ def get_cache_key(self, name, filename=None): """Returns the unique hash key for this template name.""" - hash = sha1(name.encode('utf-8')) + hash = sha1(name.encode("utf-8")) if filename is not None: - filename = '|' + filename + filename = "|" + filename if isinstance(filename, text_type): - filename = filename.encode('utf-8') + filename = filename.encode("utf-8") hash.update(filename) return hash.hexdigest() def get_source_checksum(self, source): """Returns a checksum for the source.""" - return sha1(source.encode('utf-8')).hexdigest() + return sha1(source.encode("utf-8")).hexdigest() def get_bucket(self, environment, name, filename, source): """Return a cache bucket for the given template. All arguments are @@ -210,7 +187,7 @@ class FileSystemBytecodeCache(BytecodeCache): This bytecode cache supports clearing of the cache using the clear method. """ - def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + def __init__(self, directory=None, pattern="__jinja2_%s.cache"): if directory is None: directory = self._get_default_cache_dir() self.directory = directory @@ -218,19 +195,21 @@ class FileSystemBytecodeCache(BytecodeCache): def _get_default_cache_dir(self): def _unsafe_dir(): - raise RuntimeError('Cannot determine safe temp directory. You ' - 'need to explicitly provide one.') + raise RuntimeError( + "Cannot determine safe temp directory. You " + "need to explicitly provide one." + ) tmpdir = tempfile.gettempdir() # On windows the temporary directory is used specific unless # explicitly forced otherwise. We can just use that. - if os.name == 'nt': + if os.name == "nt": return tmpdir - if not hasattr(os, 'getuid'): + if not hasattr(os, "getuid"): _unsafe_dir() - dirname = '_jinja2-cache-%d' % os.getuid() + dirname = "_jinja2-cache-%d" % os.getuid() actual_dir = os.path.join(tmpdir, dirname) try: @@ -241,18 +220,22 @@ class FileSystemBytecodeCache(BytecodeCache): try: os.chmod(actual_dir, stat.S_IRWXU) actual_dir_stat = os.lstat(actual_dir) - if actual_dir_stat.st_uid != os.getuid() \ - or not stat.S_ISDIR(actual_dir_stat.st_mode) \ - or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): _unsafe_dir() except OSError as e: if e.errno != errno.EEXIST: raise actual_dir_stat = os.lstat(actual_dir) - if actual_dir_stat.st_uid != os.getuid() \ - or not stat.S_ISDIR(actual_dir_stat.st_mode) \ - or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): _unsafe_dir() return actual_dir @@ -261,7 +244,7 @@ class FileSystemBytecodeCache(BytecodeCache): return path.join(self.directory, self.pattern % bucket.key) def load_bytecode(self, bucket): - f = open_if_exists(self._get_cache_filename(bucket), 'rb') + f = open_if_exists(self._get_cache_filename(bucket), "rb") if f is not None: try: bucket.load_bytecode(f) @@ -269,7 +252,7 @@ class FileSystemBytecodeCache(BytecodeCache): f.close() def dump_bytecode(self, bucket): - f = open(self._get_cache_filename(bucket), 'wb') + f = open(self._get_cache_filename(bucket), "wb") try: bucket.write_bytecode(f) finally: @@ -280,7 +263,8 @@ class FileSystemBytecodeCache(BytecodeCache): # write access on the file system and the function does not exist # normally. from os import remove - files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + + files = fnmatch.filter(listdir(self.directory), self.pattern % "*") for filename in files: try: remove(path.join(self.directory, filename)) @@ -296,9 +280,8 @@ class MemcachedBytecodeCache(BytecodeCache): Libraries compatible with this class: - - `werkzeug `_.contrib.cache - - `python-memcached `_ - - `cmemcache `_ + - `cachelib `_ + - `python-memcached `_ (Unfortunately the django cache interface is not compatible because it does not support storing binary data, only unicode. You can however pass @@ -334,8 +317,13 @@ class MemcachedBytecodeCache(BytecodeCache): `ignore_memcache_errors` parameter. """ - def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, - ignore_memcache_errors=True): + def __init__( + self, + client, + prefix="jinja2/bytecode/", + timeout=None, + ignore_memcache_errors=True, + ): self.client = client self.prefix = prefix self.timeout = timeout diff --git a/lib/spack/external/jinja2/compiler.py b/lib/spack/external/jinja2/compiler.py index d534a82739..63297b42c3 100644 --- a/lib/spack/external/jinja2/compiler.py +++ b/lib/spack/external/jinja2/compiler.py @@ -1,59 +1,62 @@ # -*- coding: utf-8 -*- -""" - jinja2.compiler - ~~~~~~~~~~~~~~~ - - Compiles nodes into python code. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" +"""Compiles nodes from the parser into Python code.""" +from collections import namedtuple +from functools import update_wrapper from itertools import chain -from copy import deepcopy from keyword import iskeyword as is_python_keyword -from functools import update_wrapper -from jinja2 import nodes -from jinja2.nodes import EvalContext -from jinja2.visitor import NodeVisitor -from jinja2.optimizer import Optimizer -from jinja2.exceptions import TemplateAssertionError -from jinja2.utils import Markup, concat, escape -from jinja2._compat import range_type, text_type, string_types, \ - iteritems, NativeStringIO, imap, izip -from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \ - VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED +from markupsafe import escape +from markupsafe import Markup + +from . import nodes +from ._compat import imap +from ._compat import iteritems +from ._compat import izip +from ._compat import NativeStringIO +from ._compat import range_type +from ._compat import string_types +from ._compat import text_type +from .exceptions import TemplateAssertionError +from .idtracking import Symbols +from .idtracking import VAR_LOAD_ALIAS +from .idtracking import VAR_LOAD_PARAMETER +from .idtracking import VAR_LOAD_RESOLVE +from .idtracking import VAR_LOAD_UNDEFINED +from .nodes import EvalContext +from .optimizer import Optimizer +from .utils import concat +from .visitor import NodeVisitor operators = { - 'eq': '==', - 'ne': '!=', - 'gt': '>', - 'gteq': '>=', - 'lt': '<', - 'lteq': '<=', - 'in': 'in', - 'notin': 'not in' + "eq": "==", + "ne": "!=", + "gt": ">", + "gteq": ">=", + "lt": "<", + "lteq": "<=", + "in": "in", + "notin": "not in", } # what method to iterate over items do we want to use for dict iteration # in generated code? on 2.x let's go with iteritems, on 3.x with items -if hasattr(dict, 'iteritems'): - dict_item_iter = 'iteritems' +if hasattr(dict, "iteritems"): + dict_item_iter = "iteritems" else: - dict_item_iter = 'items' + dict_item_iter = "items" -code_features = ['division'] +code_features = ["division"] # does this python version support generator stops? (PEP 0479) try: - exec('from __future__ import generator_stop') - code_features.append('generator_stop') + exec("from __future__ import generator_stop") + code_features.append("generator_stop") except SyntaxError: pass # does this python version support yield from? try: - exec('def f(): yield from x()') + exec("def f(): yield from x()") except SyntaxError: supports_yield_from = False else: @@ -68,17 +71,19 @@ def optimizeconst(f): if new_node != node: return self.visit(new_node, frame) return f(self, node, frame, **kwargs) + return update_wrapper(new_func, f) -def generate(node, environment, name, filename, stream=None, - defer_init=False, optimized=True): +def generate( + node, environment, name, filename, stream=None, defer_init=False, optimized=True +): """Generate the python source for a node tree.""" if not isinstance(node, nodes.Template): - raise TypeError('Can\'t compile non template nodes') - generator = environment.code_generator_class(environment, name, filename, - stream, defer_init, - optimized) + raise TypeError("Can't compile non template nodes") + generator = environment.code_generator_class( + environment, name, filename, stream, defer_init, optimized + ) generator.visit(node) if stream is None: return generator.stream.getvalue() @@ -119,7 +124,6 @@ def find_undeclared(nodes, names): class MacroRef(object): - def __init__(self, node): self.node = node self.accesses_caller = False @@ -132,8 +136,7 @@ class Frame(object): def __init__(self, eval_ctx, parent=None, level=None): self.eval_ctx = eval_ctx - self.symbols = Symbols(parent and parent.symbols or None, - level=level) + self.symbols = Symbols(parent and parent.symbols or None, level=level) # a toplevel frame is the root + soft frames such as if conditions. self.toplevel = False @@ -223,7 +226,7 @@ class UndeclaredNameVisitor(NodeVisitor): self.undeclared = set() def visit_Name(self, node): - if node.ctx == 'load' and node.name in self.names: + if node.ctx == "load" and node.name in self.names: self.undeclared.add(node.name) if self.undeclared == self.names: raise VisitorExit() @@ -242,9 +245,9 @@ class CompilerExit(Exception): class CodeGenerator(NodeVisitor): - - def __init__(self, environment, name, filename, stream=None, - defer_init=False, optimized=True): + def __init__( + self, environment, name, filename, stream=None, defer_init=False, optimized=True + ): if stream is None: stream = NativeStringIO() self.environment = environment @@ -306,7 +309,7 @@ class CodeGenerator(NodeVisitor): self._param_def_block = [] # Tracks the current context. - self._context_reference_stack = ['context'] + self._context_reference_stack = ["context"] # -- Various compilation helpers @@ -317,30 +320,30 @@ class CodeGenerator(NodeVisitor): def temporary_identifier(self): """Get a new unique identifier.""" self._last_identifier += 1 - return 't_%d' % self._last_identifier + return "t_%d" % self._last_identifier def buffer(self, frame): """Enable buffering for the frame from that point onwards.""" frame.buffer = self.temporary_identifier() - self.writeline('%s = []' % frame.buffer) + self.writeline("%s = []" % frame.buffer) def return_buffer_contents(self, frame, force_unescaped=False): """Return the buffer contents of the frame.""" if not force_unescaped: if frame.eval_ctx.volatile: - self.writeline('if context.eval_ctx.autoescape:') + self.writeline("if context.eval_ctx.autoescape:") self.indent() - self.writeline('return Markup(concat(%s))' % frame.buffer) + self.writeline("return Markup(concat(%s))" % frame.buffer) self.outdent() - self.writeline('else:') + self.writeline("else:") self.indent() - self.writeline('return concat(%s)' % frame.buffer) + self.writeline("return concat(%s)" % frame.buffer) self.outdent() return elif frame.eval_ctx.autoescape: - self.writeline('return Markup(concat(%s))' % frame.buffer) + self.writeline("return Markup(concat(%s))" % frame.buffer) return - self.writeline('return concat(%s)' % frame.buffer) + self.writeline("return concat(%s)" % frame.buffer) def indent(self): """Indent by one.""" @@ -353,14 +356,14 @@ class CodeGenerator(NodeVisitor): def start_write(self, frame, node=None): """Yield or write into the frame buffer.""" if frame.buffer is None: - self.writeline('yield ', node) + self.writeline("yield ", node) else: - self.writeline('%s.append(' % frame.buffer, node) + self.writeline("%s.append(" % frame.buffer, node) def end_write(self, frame): """End the writing process started by `start_write`.""" if frame.buffer is not None: - self.write(')') + self.write(")") def simple_write(self, s, frame, node=None): """Simple shortcut for start_write + write + end_write.""" @@ -373,7 +376,7 @@ class CodeGenerator(NodeVisitor): is no buffer a dummy ``if 0: yield None`` is written automatically. """ try: - self.writeline('pass') + self.writeline("pass") for node in nodes: self.visit(node, frame) except CompilerExit: @@ -383,14 +386,13 @@ class CodeGenerator(NodeVisitor): """Write a string into the output stream.""" if self._new_lines: if not self._first_write: - self.stream.write('\n' * self._new_lines) + self.stream.write("\n" * self._new_lines) self.code_lineno += self._new_lines if self._write_debug_info is not None: - self.debug_info.append((self._write_debug_info, - self.code_lineno)) + self.debug_info.append((self._write_debug_info, self.code_lineno)) self._write_debug_info = None self._first_write = False - self.stream.write(' ' * self._indentation) + self.stream.write(" " * self._indentation) self._new_lines = 0 self.stream.write(x) @@ -410,7 +412,7 @@ class CodeGenerator(NodeVisitor): """Writes a function call to the stream for the current node. A leading comma is added automatically. The extra keyword arguments may not include python keywords otherwise a syntax - error could occour. The extra keyword arguments should be given + error could occur. The extra keyword arguments should be given as python dict. """ # if any of the given keyword arguments is a python keyword @@ -422,41 +424,41 @@ class CodeGenerator(NodeVisitor): break for arg in node.args: - self.write(', ') + self.write(", ") self.visit(arg, frame) if not kwarg_workaround: for kwarg in node.kwargs: - self.write(', ') + self.write(", ") self.visit(kwarg, frame) if extra_kwargs is not None: for key, value in iteritems(extra_kwargs): - self.write(', %s=%s' % (key, value)) + self.write(", %s=%s" % (key, value)) if node.dyn_args: - self.write(', *') + self.write(", *") self.visit(node.dyn_args, frame) if kwarg_workaround: if node.dyn_kwargs is not None: - self.write(', **dict({') + self.write(", **dict({") else: - self.write(', **{') + self.write(", **{") for kwarg in node.kwargs: - self.write('%r: ' % kwarg.key) + self.write("%r: " % kwarg.key) self.visit(kwarg.value, frame) - self.write(', ') + self.write(", ") if extra_kwargs is not None: for key, value in iteritems(extra_kwargs): - self.write('%r: %s, ' % (key, value)) + self.write("%r: %s, " % (key, value)) if node.dyn_kwargs is not None: - self.write('}, **') + self.write("}, **") self.visit(node.dyn_kwargs, frame) - self.write(')') + self.write(")") else: - self.write('}') + self.write("}") elif node.dyn_kwargs is not None: - self.write(', **') + self.write(", **") self.visit(node.dyn_kwargs, frame) def pull_dependencies(self, nodes): @@ -464,13 +466,14 @@ class CodeGenerator(NodeVisitor): visitor = DependencyFinderVisitor() for node in nodes: visitor.visit(node) - for dependency in 'filters', 'tests': + for dependency in "filters", "tests": mapping = getattr(self, dependency) for name in getattr(visitor, dependency): if name not in mapping: mapping[name] = self.temporary_identifier() - self.writeline('%s = environment.%s[%r]' % - (mapping[name], dependency, name)) + self.writeline( + "%s = environment.%s[%r]" % (mapping[name], dependency, name) + ) def enter_frame(self, frame): undefs = [] @@ -478,16 +481,15 @@ class CodeGenerator(NodeVisitor): if action == VAR_LOAD_PARAMETER: pass elif action == VAR_LOAD_RESOLVE: - self.writeline('%s = %s(%r)' % - (target, self.get_resolve_func(), param)) + self.writeline("%s = %s(%r)" % (target, self.get_resolve_func(), param)) elif action == VAR_LOAD_ALIAS: - self.writeline('%s = %s' % (target, param)) + self.writeline("%s = %s" % (target, param)) elif action == VAR_LOAD_UNDEFINED: undefs.append(target) else: - raise NotImplementedError('unknown load instruction') + raise NotImplementedError("unknown load instruction") if undefs: - self.writeline('%s = missing' % ' = '.join(undefs)) + self.writeline("%s = missing" % " = ".join(undefs)) def leave_frame(self, frame, with_python_scope=False): if not with_python_scope: @@ -495,12 +497,12 @@ class CodeGenerator(NodeVisitor): for target, _ in iteritems(frame.symbols.loads): undefs.append(target) if undefs: - self.writeline('%s = missing' % ' = '.join(undefs)) + self.writeline("%s = missing" % " = ".join(undefs)) def func(self, name): if self.environment.is_async: - return 'async def %s' % name - return 'def %s' % name + return "async def %s" % name + return "def %s" % name def macro_body(self, node, frame): """Dump the function def of a macro or call block.""" @@ -512,16 +514,16 @@ class CodeGenerator(NodeVisitor): skip_special_params = set() args = [] for idx, arg in enumerate(node.args): - if arg.name == 'caller': + if arg.name == "caller": explicit_caller = idx - if arg.name in ('kwargs', 'varargs'): + if arg.name in ("kwargs", "varargs"): skip_special_params.add(arg.name) args.append(frame.symbols.ref(arg.name)) - undeclared = find_undeclared(node.body, ('caller', 'kwargs', 'varargs')) + undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) - if 'caller' in undeclared: - # In older Jinja2 versions there was a bug that allowed caller + if "caller" in undeclared: + # In older Jinja versions there was a bug that allowed caller # to retain the special behavior even if it was mentioned in # the argument list. However thankfully this was only really # working if it was the last argument. So we are explicitly @@ -531,23 +533,26 @@ class CodeGenerator(NodeVisitor): try: node.defaults[explicit_caller - len(node.args)] except IndexError: - self.fail('When defining macros or call blocks the ' - 'special "caller" argument must be omitted ' - 'or be given a default.', node.lineno) + self.fail( + "When defining macros or call blocks the " + 'special "caller" argument must be omitted ' + "or be given a default.", + node.lineno, + ) else: - args.append(frame.symbols.declare_parameter('caller')) + args.append(frame.symbols.declare_parameter("caller")) macro_ref.accesses_caller = True - if 'kwargs' in undeclared and not 'kwargs' in skip_special_params: - args.append(frame.symbols.declare_parameter('kwargs')) + if "kwargs" in undeclared and "kwargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("kwargs")) macro_ref.accesses_kwargs = True - if 'varargs' in undeclared and not 'varargs' in skip_special_params: - args.append(frame.symbols.declare_parameter('varargs')) + if "varargs" in undeclared and "varargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("varargs")) macro_ref.accesses_varargs = True # macros are delayed, they never require output checks frame.require_output_check = False frame.symbols.analyze_node(node) - self.writeline('%s(%s):' % (self.func('macro'), ', '.join(args)), node) + self.writeline("%s(%s):" % (self.func("macro"), ", ".join(args)), node) self.indent() self.buffer(frame) @@ -556,17 +561,17 @@ class CodeGenerator(NodeVisitor): self.push_parameter_definitions(frame) for idx, arg in enumerate(node.args): ref = frame.symbols.ref(arg.name) - self.writeline('if %s is missing:' % ref) + self.writeline("if %s is missing:" % ref) self.indent() try: default = node.defaults[idx - len(node.args)] except IndexError: - self.writeline('%s = undefined(%r, name=%r)' % ( - ref, - 'parameter %r was not provided' % arg.name, - arg.name)) + self.writeline( + "%s = undefined(%r, name=%r)" + % (ref, "parameter %r was not provided" % arg.name, arg.name) + ) else: - self.writeline('%s = ' % ref) + self.writeline("%s = " % ref) self.visit(default, frame) self.mark_parameter_stored(ref) self.outdent() @@ -581,35 +586,46 @@ class CodeGenerator(NodeVisitor): def macro_def(self, macro_ref, frame): """Dump the macro definition for the def created by macro_body.""" - arg_tuple = ', '.join(repr(x.name) for x in macro_ref.node.args) - name = getattr(macro_ref.node, 'name', None) + arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, "name", None) if len(macro_ref.node.args) == 1: - arg_tuple += ',' - self.write('Macro(environment, macro, %r, (%s), %r, %r, %r, ' - 'context.eval_ctx.autoescape)' % - (name, arg_tuple, macro_ref.accesses_kwargs, - macro_ref.accesses_varargs, macro_ref.accesses_caller)) + arg_tuple += "," + self.write( + "Macro(environment, macro, %r, (%s), %r, %r, %r, " + "context.eval_ctx.autoescape)" + % ( + name, + arg_tuple, + macro_ref.accesses_kwargs, + macro_ref.accesses_varargs, + macro_ref.accesses_caller, + ) + ) def position(self, node): """Return a human readable position for the node.""" - rv = 'line %d' % node.lineno + rv = "line %d" % node.lineno if self.name is not None: - rv += ' in ' + repr(self.name) + rv += " in " + repr(self.name) return rv def dump_local_context(self, frame): - return '{%s}' % ', '.join( - '%r: %s' % (name, target) for name, target - in iteritems(frame.symbols.dump_stores())) + return "{%s}" % ", ".join( + "%r: %s" % (name, target) + for name, target in iteritems(frame.symbols.dump_stores()) + ) def write_commons(self): """Writes a common preamble that is used by root and block functions. Primarily this sets up common local helpers and enforces a generator through a dead branch. """ - self.writeline('resolve = context.resolve_or_missing') - self.writeline('undefined = environment.undefined') - self.writeline('if 0: yield None') + self.writeline("resolve = context.resolve_or_missing") + self.writeline("undefined = environment.undefined") + # always use the standard Undefined class for the implicit else of + # conditional expressions + self.writeline("cond_expr_undefined = Undefined") + self.writeline("if 0: yield None") def push_parameter_definitions(self, frame): """Pushes all parameter targets from the given frame into a local @@ -642,12 +658,12 @@ class CodeGenerator(NodeVisitor): def get_resolve_func(self): target = self._context_reference_stack[-1] - if target == 'context': - return 'resolve' - return '%s.resolve' % target + if target == "context": + return "resolve" + return "%s.resolve" % target def derive_context(self, frame): - return '%s.derived(%s)' % ( + return "%s.derived(%s)" % ( self.get_context_ref(), self.dump_local_context(frame), ) @@ -669,44 +685,48 @@ class CodeGenerator(NodeVisitor): vars = self._assign_stack.pop() if not frame.toplevel or not vars: return - public_names = [x for x in vars if x[:1] != '_'] + public_names = [x for x in vars if x[:1] != "_"] if len(vars) == 1: name = next(iter(vars)) ref = frame.symbols.ref(name) - self.writeline('context.vars[%r] = %s' % (name, ref)) + self.writeline("context.vars[%r] = %s" % (name, ref)) else: - self.writeline('context.vars.update({') + self.writeline("context.vars.update({") for idx, name in enumerate(vars): if idx: - self.write(', ') + self.write(", ") ref = frame.symbols.ref(name) - self.write('%r: %s' % (name, ref)) - self.write('})') + self.write("%r: %s" % (name, ref)) + self.write("})") if public_names: if len(public_names) == 1: - self.writeline('context.exported_vars.add(%r)' % - public_names[0]) + self.writeline("context.exported_vars.add(%r)" % public_names[0]) else: - self.writeline('context.exported_vars.update((%s))' % - ', '.join(imap(repr, public_names))) + self.writeline( + "context.exported_vars.update((%s))" + % ", ".join(imap(repr, public_names)) + ) # -- Statement Visitors def visit_Template(self, node, frame=None): - assert frame is None, 'no root frame allowed' + assert frame is None, "no root frame allowed" eval_ctx = EvalContext(self.environment, self.name) - from jinja2.runtime import __all__ as exported - self.writeline('from __future__ import %s' % ', '.join(code_features)) - self.writeline('from jinja2.runtime import ' + ', '.join(exported)) + from .runtime import exported + + self.writeline("from __future__ import %s" % ", ".join(code_features)) + self.writeline("from jinja2.runtime import " + ", ".join(exported)) if self.environment.is_async: - self.writeline('from jinja2.asyncsupport import auto_await, ' - 'auto_aiter, make_async_loop_context') + self.writeline( + "from jinja2.asyncsupport import auto_await, " + "auto_aiter, AsyncLoopContext" + ) # if we want a deferred initialization we cannot move the # environment into a local name - envenv = not self.defer_init and ', environment=environment' or '' + envenv = not self.defer_init and ", environment=environment" or "" # do we have an extends tag at all? If not, we can save some # overhead by just not processing any inheritance code. @@ -715,7 +735,7 @@ class CodeGenerator(NodeVisitor): # find all blocks for block in node.find_all(nodes.Block): if block.name in self.blocks: - self.fail('block %r defined twice' % block.name, block.lineno) + self.fail("block %r defined twice" % block.name, block.lineno) self.blocks[block.name] = block # find all imports and import them @@ -723,32 +743,32 @@ class CodeGenerator(NodeVisitor): if import_.importname not in self.import_aliases: imp = import_.importname self.import_aliases[imp] = alias = self.temporary_identifier() - if '.' in imp: - module, obj = imp.rsplit('.', 1) - self.writeline('from %s import %s as %s' % - (module, obj, alias)) + if "." in imp: + module, obj = imp.rsplit(".", 1) + self.writeline("from %s import %s as %s" % (module, obj, alias)) else: - self.writeline('import %s as %s' % (imp, alias)) + self.writeline("import %s as %s" % (imp, alias)) # add the load name - self.writeline('name = %r' % self.name) + self.writeline("name = %r" % self.name) # generate the root render function. - self.writeline('%s(context, missing=missing%s):' % - (self.func('root'), envenv), extra=1) + self.writeline( + "%s(context, missing=missing%s):" % (self.func("root"), envenv), extra=1 + ) self.indent() self.write_commons() # process the root frame = Frame(eval_ctx) - if 'self' in find_undeclared(node.body, ('self',)): - ref = frame.symbols.declare_parameter('self') - self.writeline('%s = TemplateReference(context)' % ref) + if "self" in find_undeclared(node.body, ("self",)): + ref = frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) frame.symbols.analyze_node(node) frame.toplevel = frame.rootlevel = True frame.require_output_check = have_extends and not self.has_known_extends if have_extends: - self.writeline('parent_template = None') + self.writeline("parent_template = None") self.enter_frame(frame) self.pull_dependencies(node.body) self.blockvisit(node.body, frame) @@ -759,39 +779,42 @@ class CodeGenerator(NodeVisitor): if have_extends: if not self.has_known_extends: self.indent() - self.writeline('if parent_template is not None:') + self.writeline("if parent_template is not None:") self.indent() if supports_yield_from and not self.environment.is_async: - self.writeline('yield from parent_template.' - 'root_render_func(context)') + self.writeline("yield from parent_template.root_render_func(context)") else: - self.writeline('%sfor event in parent_template.' - 'root_render_func(context):' % - (self.environment.is_async and 'async ' or '')) + self.writeline( + "%sfor event in parent_template." + "root_render_func(context):" + % (self.environment.is_async and "async " or "") + ) self.indent() - self.writeline('yield event') + self.writeline("yield event") self.outdent() self.outdent(1 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. for name, block in iteritems(self.blocks): - self.writeline('%s(context, missing=missing%s):' % - (self.func('block_' + name), envenv), - block, 1) + self.writeline( + "%s(context, missing=missing%s):" + % (self.func("block_" + name), envenv), + block, + 1, + ) self.indent() self.write_commons() # It's important that we do not make this frame a child of the # toplevel template. This would cause a variety of # interesting issues with identifier tracking. block_frame = Frame(eval_ctx) - undeclared = find_undeclared(block.body, ('self', 'super')) - if 'self' in undeclared: - ref = block_frame.symbols.declare_parameter('self') - self.writeline('%s = TemplateReference(context)' % ref) - if 'super' in undeclared: - ref = block_frame.symbols.declare_parameter('super') - self.writeline('%s = context.super(%r, ' - 'block_%s)' % (ref, name, name)) + undeclared = find_undeclared(block.body, ("self", "super")) + if "self" in undeclared: + ref = block_frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) + if "super" in undeclared: + ref = block_frame.symbols.declare_parameter("super") + self.writeline("%s = context.super(%r, block_%s)" % (ref, name, name)) block_frame.symbols.analyze_node(block) block_frame.block = name self.enter_frame(block_frame) @@ -800,13 +823,15 @@ class CodeGenerator(NodeVisitor): self.leave_frame(block_frame, with_python_scope=True) self.outdent() - self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x) - for x in self.blocks), - extra=1) + self.writeline( + "blocks = {%s}" % ", ".join("%r: block_%s" % (x, x) for x in self.blocks), + extra=1, + ) # add a function that returns the debug info - self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x - in self.debug_info)) + self.writeline( + "debug_info = %r" % "&".join("%s=%s" % x for x in self.debug_info) + ) def visit_Block(self, node, frame): """Call a block and register it for the template.""" @@ -817,7 +842,7 @@ class CodeGenerator(NodeVisitor): if self.has_known_extends: return if self.extends_so_far > 0: - self.writeline('if parent_template is None:') + self.writeline("if parent_template is None:") self.indent() level += 1 @@ -826,16 +851,22 @@ class CodeGenerator(NodeVisitor): else: context = self.get_context_ref() - if supports_yield_from and not self.environment.is_async and \ - frame.buffer is None: - self.writeline('yield from context.blocks[%r][0](%s)' % ( - node.name, context), node) + if ( + supports_yield_from + and not self.environment.is_async + and frame.buffer is None + ): + self.writeline( + "yield from context.blocks[%r][0](%s)" % (node.name, context), node + ) else: - loop = self.environment.is_async and 'async for' or 'for' - self.writeline('%s event in context.blocks[%r][0](%s):' % ( - loop, node.name, context), node) + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in context.blocks[%r][0](%s):" % (loop, node.name, context), + node, + ) self.indent() - self.simple_write('event', frame) + self.simple_write("event", frame) self.outdent() self.outdent(level) @@ -843,8 +874,7 @@ class CodeGenerator(NodeVisitor): def visit_Extends(self, node, frame): """Calls the extender.""" if not frame.toplevel: - self.fail('cannot use extend from a non top-level scope', - node.lineno) + self.fail("cannot use extend from a non top-level scope", node.lineno) # if the number of extends statements in general is zero so # far, we don't have to add a check if something extended @@ -856,10 +886,9 @@ class CodeGenerator(NodeVisitor): # time too, but i welcome it not to confuse users by throwing the # same error at different times just "because we can". if not self.has_known_extends: - self.writeline('if parent_template is not None:') + self.writeline("if parent_template is not None:") self.indent() - self.writeline('raise TemplateRuntimeError(%r)' % - 'extended multiple times') + self.writeline("raise TemplateRuntimeError(%r)" % "extended multiple times") # if we have a known extends already we don't need that code here # as we know that the template execution will end here. @@ -868,14 +897,14 @@ class CodeGenerator(NodeVisitor): else: self.outdent() - self.writeline('parent_template = environment.get_template(', node) + self.writeline("parent_template = environment.get_template(", node) self.visit(node.template, frame) - self.write(', %r)' % self.name) - self.writeline('for name, parent_block in parent_template.' - 'blocks.%s():' % dict_item_iter) + self.write(", %r)" % self.name) + self.writeline( + "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter + ) self.indent() - self.writeline('context.blocks.setdefault(name, []).' - 'append(parent_block)') + self.writeline("context.blocks.setdefault(name, []).append(parent_block)") self.outdent() # if this extends statement was in the root level we can take @@ -890,52 +919,56 @@ class CodeGenerator(NodeVisitor): def visit_Include(self, node, frame): """Handles includes.""" if node.ignore_missing: - self.writeline('try:') + self.writeline("try:") self.indent() - func_name = 'get_or_select_template' + func_name = "get_or_select_template" if isinstance(node.template, nodes.Const): if isinstance(node.template.value, string_types): - func_name = 'get_template' + func_name = "get_template" elif isinstance(node.template.value, (tuple, list)): - func_name = 'select_template' + func_name = "select_template" elif isinstance(node.template, (nodes.Tuple, nodes.List)): - func_name = 'select_template' + func_name = "select_template" - self.writeline('template = environment.%s(' % func_name, node) + self.writeline("template = environment.%s(" % func_name, node) self.visit(node.template, frame) - self.write(', %r)' % self.name) + self.write(", %r)" % self.name) if node.ignore_missing: self.outdent() - self.writeline('except TemplateNotFound:') + self.writeline("except TemplateNotFound:") self.indent() - self.writeline('pass') + self.writeline("pass") self.outdent() - self.writeline('else:') + self.writeline("else:") self.indent() skip_event_yield = False if node.with_context: - loop = self.environment.is_async and 'async for' or 'for' - self.writeline('%s event in template.root_render_func(' - 'template.new_context(context.get_all(), True, ' - '%s)):' % (loop, self.dump_local_context(frame))) + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in template.root_render_func(" + "template.new_context(context.get_all(), True, " + "%s)):" % (loop, self.dump_local_context(frame)) + ) elif self.environment.is_async: - self.writeline('for event in (await ' - 'template._get_default_module_async())' - '._body_stream:') + self.writeline( + "for event in (await " + "template._get_default_module_async())" + "._body_stream:" + ) else: if supports_yield_from: - self.writeline('yield from template._get_default_module()' - '._body_stream') + self.writeline("yield from template._get_default_module()._body_stream") skip_event_yield = True else: - self.writeline('for event in template._get_default_module()' - '._body_stream:') + self.writeline( + "for event in template._get_default_module()._body_stream:" + ) if not skip_event_yield: self.indent() - self.simple_write('event', frame) + self.simple_write("event", frame) self.outdent() if node.ignore_missing: @@ -943,40 +976,50 @@ class CodeGenerator(NodeVisitor): def visit_Import(self, node, frame): """Visit regular imports.""" - self.writeline('%s = ' % frame.symbols.ref(node.target), node) + self.writeline("%s = " % frame.symbols.ref(node.target), node) if frame.toplevel: - self.write('context.vars[%r] = ' % node.target) + self.write("context.vars[%r] = " % node.target) if self.environment.is_async: - self.write('await ') - self.write('environment.get_template(') + self.write("await ") + self.write("environment.get_template(") self.visit(node.template, frame) - self.write(', %r).' % self.name) + self.write(", %r)." % self.name) if node.with_context: - self.write('make_module%s(context.get_all(), True, %s)' - % (self.environment.is_async and '_async' or '', - self.dump_local_context(frame))) + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) elif self.environment.is_async: - self.write('_get_default_module_async()') + self.write("_get_default_module_async()") else: - self.write('_get_default_module()') - if frame.toplevel and not node.target.startswith('_'): - self.writeline('context.exported_vars.discard(%r)' % node.target) + self.write("_get_default_module()") + if frame.toplevel and not node.target.startswith("_"): + self.writeline("context.exported_vars.discard(%r)" % node.target) def visit_FromImport(self, node, frame): """Visit named imports.""" self.newline(node) - self.write('included_template = %senvironment.get_template(' - % (self.environment.is_async and 'await ' or '')) + self.write( + "included_template = %senvironment.get_template(" + % (self.environment.is_async and "await " or "") + ) self.visit(node.template, frame) - self.write(', %r).' % self.name) + self.write(", %r)." % self.name) if node.with_context: - self.write('make_module%s(context.get_all(), True, %s)' - % (self.environment.is_async and '_async' or '', - self.dump_local_context(frame))) + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) elif self.environment.is_async: - self.write('_get_default_module_async()') + self.write("_get_default_module_async()") else: - self.write('_get_default_module()') + self.write("_get_default_module()") var_names = [] discarded_names = [] @@ -985,41 +1028,51 @@ class CodeGenerator(NodeVisitor): name, alias = name else: alias = name - self.writeline('%s = getattr(included_template, ' - '%r, missing)' % (frame.symbols.ref(alias), name)) - self.writeline('if %s is missing:' % frame.symbols.ref(alias)) + self.writeline( + "%s = getattr(included_template, " + "%r, missing)" % (frame.symbols.ref(alias), name) + ) + self.writeline("if %s is missing:" % frame.symbols.ref(alias)) self.indent() - self.writeline('%s = undefined(%r %% ' - 'included_template.__name__, ' - 'name=%r)' % - (frame.symbols.ref(alias), - 'the template %%r (imported on %s) does ' - 'not export the requested name %s' % ( - self.position(node), - repr(name) - ), name)) + self.writeline( + "%s = undefined(%r %% " + "included_template.__name__, " + "name=%r)" + % ( + frame.symbols.ref(alias), + "the template %%r (imported on %s) does " + "not export the requested name %s" + % (self.position(node), repr(name)), + name, + ) + ) self.outdent() if frame.toplevel: var_names.append(alias) - if not alias.startswith('_'): + if not alias.startswith("_"): discarded_names.append(alias) if var_names: if len(var_names) == 1: name = var_names[0] - self.writeline('context.vars[%r] = %s' % - (name, frame.symbols.ref(name))) + self.writeline( + "context.vars[%r] = %s" % (name, frame.symbols.ref(name)) + ) else: - self.writeline('context.vars.update({%s})' % ', '.join( - '%r: %s' % (name, frame.symbols.ref(name)) for name in var_names - )) + self.writeline( + "context.vars.update({%s})" + % ", ".join( + "%r: %s" % (name, frame.symbols.ref(name)) for name in var_names + ) + ) if discarded_names: if len(discarded_names) == 1: - self.writeline('context.exported_vars.discard(%r)' % - discarded_names[0]) + self.writeline("context.exported_vars.discard(%r)" % discarded_names[0]) else: - self.writeline('context.exported_vars.difference_' - 'update((%s))' % ', '.join(imap(repr, discarded_names))) + self.writeline( + "context.exported_vars.difference_" + "update((%s))" % ", ".join(imap(repr, discarded_names)) + ) def visit_For(self, node, frame): loop_frame = frame.inner() @@ -1029,35 +1082,35 @@ class CodeGenerator(NodeVisitor): # try to figure out if we have an extended loop. An extended loop # is necessary if the loop is in recursive mode if the special loop # variable is accessed in the body. - extended_loop = node.recursive or 'loop' in \ - find_undeclared(node.iter_child_nodes( - only=('body',)), ('loop',)) + extended_loop = node.recursive or "loop" in find_undeclared( + node.iter_child_nodes(only=("body",)), ("loop",) + ) loop_ref = None if extended_loop: - loop_ref = loop_frame.symbols.declare_parameter('loop') + loop_ref = loop_frame.symbols.declare_parameter("loop") - loop_frame.symbols.analyze_node(node, for_branch='body') + loop_frame.symbols.analyze_node(node, for_branch="body") if node.else_: - else_frame.symbols.analyze_node(node, for_branch='else') + else_frame.symbols.analyze_node(node, for_branch="else") if node.test: loop_filter_func = self.temporary_identifier() - test_frame.symbols.analyze_node(node, for_branch='test') - self.writeline('%s(fiter):' % self.func(loop_filter_func), node.test) + test_frame.symbols.analyze_node(node, for_branch="test") + self.writeline("%s(fiter):" % self.func(loop_filter_func), node.test) self.indent() self.enter_frame(test_frame) - self.writeline(self.environment.is_async and 'async for ' or 'for ') + self.writeline(self.environment.is_async and "async for " or "for ") self.visit(node.target, loop_frame) - self.write(' in ') - self.write(self.environment.is_async and 'auto_aiter(fiter)' or 'fiter') - self.write(':') + self.write(" in ") + self.write(self.environment.is_async and "auto_aiter(fiter)" or "fiter") + self.write(":") self.indent() - self.writeline('if ', node.test) + self.writeline("if ", node.test) self.visit(node.test, test_frame) - self.write(':') + self.write(":") self.indent() - self.writeline('yield ') + self.writeline("yield ") self.visit(node.target, loop_frame) self.outdent(3) self.leave_frame(test_frame, with_python_scope=True) @@ -1066,8 +1119,9 @@ class CodeGenerator(NodeVisitor): # variables at that point. Because loops can be nested but the loop # variable is a special one we have to enforce aliasing for it. if node.recursive: - self.writeline('%s(reciter, loop_render_func, depth=0):' % - self.func('loop'), node) + self.writeline( + "%s(reciter, loop_render_func, depth=0):" % self.func("loop"), node + ) self.indent() self.buffer(loop_frame) @@ -1077,57 +1131,60 @@ class CodeGenerator(NodeVisitor): # make sure the loop variable is a special one and raise a template # assertion error if a loop tries to write to loop if extended_loop: - self.writeline('%s = missing' % loop_ref) + self.writeline("%s = missing" % loop_ref) for name in node.find_all(nodes.Name): - if name.ctx == 'store' and name.name == 'loop': - self.fail('Can\'t assign to special loop variable ' - 'in for-loop target', name.lineno) + if name.ctx == "store" and name.name == "loop": + self.fail( + "Can't assign to special loop variable in for-loop target", + name.lineno, + ) if node.else_: iteration_indicator = self.temporary_identifier() - self.writeline('%s = 1' % iteration_indicator) + self.writeline("%s = 1" % iteration_indicator) - self.writeline(self.environment.is_async and 'async for ' or 'for ', node) + self.writeline(self.environment.is_async and "async for " or "for ", node) self.visit(node.target, loop_frame) if extended_loop: if self.environment.is_async: - self.write(', %s in await make_async_loop_context(' % loop_ref) + self.write(", %s in AsyncLoopContext(" % loop_ref) else: - self.write(', %s in LoopContext(' % loop_ref) + self.write(", %s in LoopContext(" % loop_ref) else: - self.write(' in ') + self.write(" in ") if node.test: - self.write('%s(' % loop_filter_func) + self.write("%s(" % loop_filter_func) if node.recursive: - self.write('reciter') + self.write("reciter") else: if self.environment.is_async and not extended_loop: - self.write('auto_aiter(') + self.write("auto_aiter(") self.visit(node.iter, frame) if self.environment.is_async and not extended_loop: - self.write(')') + self.write(")") if node.test: - self.write(')') + self.write(")") if node.recursive: - self.write(', undefined, loop_render_func, depth):') + self.write(", undefined, loop_render_func, depth):") else: - self.write(extended_loop and ', undefined):' or ':') + self.write(extended_loop and ", undefined):" or ":") self.indent() self.enter_frame(loop_frame) self.blockvisit(node.body, loop_frame) if node.else_: - self.writeline('%s = 0' % iteration_indicator) + self.writeline("%s = 0" % iteration_indicator) self.outdent() - self.leave_frame(loop_frame, with_python_scope=node.recursive - and not node.else_) + self.leave_frame( + loop_frame, with_python_scope=node.recursive and not node.else_ + ) if node.else_: - self.writeline('if %s:' % iteration_indicator) + self.writeline("if %s:" % iteration_indicator) self.indent() self.enter_frame(else_frame) self.blockvisit(node.else_, else_frame) @@ -1141,33 +1198,33 @@ class CodeGenerator(NodeVisitor): self.outdent() self.start_write(frame, node) if self.environment.is_async: - self.write('await ') - self.write('loop(') + self.write("await ") + self.write("loop(") if self.environment.is_async: - self.write('auto_aiter(') + self.write("auto_aiter(") self.visit(node.iter, frame) if self.environment.is_async: - self.write(')') - self.write(', loop)') + self.write(")") + self.write(", loop)") self.end_write(frame) def visit_If(self, node, frame): if_frame = frame.soft() - self.writeline('if ', node) + self.writeline("if ", node) self.visit(node.test, if_frame) - self.write(':') + self.write(":") self.indent() self.blockvisit(node.body, if_frame) self.outdent() for elif_ in node.elif_: - self.writeline('elif ', elif_) + self.writeline("elif ", elif_) self.visit(elif_.test, if_frame) - self.write(':') + self.write(":") self.indent() self.blockvisit(elif_.body, if_frame) self.outdent() if node.else_: - self.writeline('else:') + self.writeline("else:") self.indent() self.blockvisit(node.else_, if_frame) self.outdent() @@ -1176,16 +1233,15 @@ class CodeGenerator(NodeVisitor): macro_frame, macro_ref = self.macro_body(node, frame) self.newline() if frame.toplevel: - if not node.name.startswith('_'): - self.write('context.exported_vars.add(%r)' % node.name) - ref = frame.symbols.ref(node.name) - self.writeline('context.vars[%r] = ' % node.name) - self.write('%s = ' % frame.symbols.ref(node.name)) + if not node.name.startswith("_"): + self.write("context.exported_vars.add(%r)" % node.name) + self.writeline("context.vars[%r] = " % node.name) + self.write("%s = " % frame.symbols.ref(node.name)) self.macro_def(macro_ref, macro_frame) def visit_CallBlock(self, node, frame): call_frame, macro_ref = self.macro_body(node, frame) - self.writeline('caller = ') + self.writeline("caller = ") self.macro_def(macro_ref, call_frame) self.start_write(frame, node) self.visit_Call(node.call, frame, forward_caller=True) @@ -1206,10 +1262,10 @@ class CodeGenerator(NodeVisitor): with_frame = frame.inner() with_frame.symbols.analyze_node(node) self.enter_frame(with_frame) - for idx, (target, expr) in enumerate(izip(node.targets, node.values)): + for target, expr in izip(node.targets, node.values): self.newline() self.visit(target, with_frame) - self.write(' = ') + self.write(" = ") self.visit(expr, frame) self.blockvisit(node.body, with_frame) self.leave_frame(with_frame) @@ -1218,156 +1274,187 @@ class CodeGenerator(NodeVisitor): self.newline(node) self.visit(node.node, frame) - def visit_Output(self, node, frame): - # if we have a known extends statement, we don't output anything - # if we are in a require_output_check section - if self.has_known_extends and frame.require_output_check: - return + _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src")) + #: The default finalize function if the environment isn't configured + #: with one. Or if the environment has one, this is called on that + #: function's output for constants. + _default_finalize = text_type + _finalize = None + + def _make_finalize(self): + """Build the finalize function to be used on constants and at + runtime. Cached so it's only created once for all output nodes. + + Returns a ``namedtuple`` with the following attributes: + + ``const`` + A function to finalize constant data at compile time. + + ``src`` + Source code to output around nodes to be evaluated at + runtime. + """ + if self._finalize is not None: + return self._finalize + + finalize = default = self._default_finalize + src = None - allow_constant_finalize = True if self.environment.finalize: - func = self.environment.finalize - if getattr(func, 'contextfunction', False) or \ - getattr(func, 'evalcontextfunction', False): - allow_constant_finalize = False - elif getattr(func, 'environmentfunction', False): - finalize = lambda x: text_type( - self.environment.finalize(self.environment, x)) - else: - finalize = lambda x: text_type(self.environment.finalize(x)) + src = "environment.finalize(" + env_finalize = self.environment.finalize + + def finalize(value): + return default(env_finalize(value)) + + if getattr(env_finalize, "contextfunction", False) is True: + src += "context, " + finalize = None # noqa: F811 + elif getattr(env_finalize, "evalcontextfunction", False) is True: + src += "context.eval_ctx, " + finalize = None + elif getattr(env_finalize, "environmentfunction", False) is True: + src += "environment, " + + def finalize(value): + return default(env_finalize(self.environment, value)) + + self._finalize = self._FinalizeInfo(finalize, src) + return self._finalize + + def _output_const_repr(self, group): + """Given a group of constant values converted from ``Output`` + child nodes, produce a string to write to the template module + source. + """ + return repr(concat(group)) + + def _output_child_to_const(self, node, frame, finalize): + """Try to optimize a child of an ``Output`` node by trying to + convert it to constant, finalized data at compile time. + + If :exc:`Impossible` is raised, the node is not constant and + will be evaluated at runtime. Any other exception will also be + evaluated at runtime for easier debugging. + """ + const = node.as_const(frame.eval_ctx) + + if frame.eval_ctx.autoescape: + const = escape(const) + + # Template data doesn't go through finalize. + if isinstance(node, nodes.TemplateData): + return text_type(const) + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + """Output extra source code before visiting a child of an + ``Output`` node. + """ + if frame.eval_ctx.volatile: + self.write("(escape if context.eval_ctx.autoescape else to_string)(") + elif frame.eval_ctx.autoescape: + self.write("escape(") else: - finalize = text_type + self.write("to_string(") + + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + """Output extra source code after visiting a child of an + ``Output`` node. + """ + self.write(")") + + if finalize.src is not None: + self.write(")") - # if we are inside a frame that requires output checking, we do so - outdent_later = False + def visit_Output(self, node, frame): + # If an extends is active, don't render outside a block. if frame.require_output_check: - self.writeline('if parent_template is None:') + # A top-level extends is known to exist at compile time. + if self.has_known_extends: + return + + self.writeline("if parent_template is None:") self.indent() - outdent_later = True - # try to evaluate as many chunks as possible into a static - # string at compile time. + finalize = self._make_finalize() body = [] + + # Evaluate constants at compile time if possible. Each item in + # body will be either a list of static data or a node to be + # evaluated at runtime. for child in node.nodes: try: - if not allow_constant_finalize: + if not ( + # If the finalize function requires runtime context, + # constants can't be evaluated at compile time. + finalize.const + # Unless it's basic template data that won't be + # finalized anyway. + or isinstance(child, nodes.TemplateData) + ): raise nodes.Impossible() - const = child.as_const(frame.eval_ctx) - except nodes.Impossible: - body.append(child) - continue - # the frame can't be volatile here, becaus otherwise the - # as_const() function would raise an Impossible exception - # at that point. - try: - if frame.eval_ctx.autoescape: - if hasattr(const, '__html__'): - const = const.__html__() - else: - const = escape(const) - const = finalize(const) - except Exception: - # if something goes wrong here we evaluate the node - # at runtime for easier debugging + + const = self._output_child_to_const(child, frame, finalize) + except (nodes.Impossible, Exception): + # The node was not constant and needs to be evaluated at + # runtime. Or another error was raised, which is easier + # to debug at runtime. body.append(child) continue + if body and isinstance(body[-1], list): body[-1].append(const) else: body.append([const]) - # if we have less than 3 nodes or a buffer we yield or extend/append - if len(body) < 3 or frame.buffer is not None: - if frame.buffer is not None: - # for one item we append, for more we extend - if len(body) == 1: - self.writeline('%s.append(' % frame.buffer) + if frame.buffer is not None: + if len(body) == 1: + self.writeline("%s.append(" % frame.buffer) + else: + self.writeline("%s.extend((" % frame.buffer) + + self.indent() + + for item in body: + if isinstance(item, list): + # A group of constant data to join and output. + val = self._output_const_repr(item) + + if frame.buffer is None: + self.writeline("yield " + val) else: - self.writeline('%s.extend((' % frame.buffer) - self.indent() - for item in body: - if isinstance(item, list): - val = repr(concat(item)) - if frame.buffer is None: - self.writeline('yield ' + val) - else: - self.writeline(val + ',') + self.writeline(val + ",") + else: + if frame.buffer is None: + self.writeline("yield ", item) else: - if frame.buffer is None: - self.writeline('yield ', item) - else: - self.newline(item) - close = 1 - if frame.eval_ctx.volatile: - self.write('(escape if context.eval_ctx.autoescape' - ' else to_string)(') - elif frame.eval_ctx.autoescape: - self.write('escape(') - else: - self.write('to_string(') - if self.environment.finalize is not None: - self.write('environment.finalize(') - if getattr(self.environment.finalize, - "contextfunction", False): - self.write('context, ') - close += 1 - self.visit(item, frame) - self.write(')' * close) - if frame.buffer is not None: - self.write(',') - if frame.buffer is not None: - # close the open parentheses - self.outdent() - self.writeline(len(body) == 1 and ')' or '))') + self.newline(item) - # otherwise we create a format string as this is faster in that case - else: - format = [] - arguments = [] - for item in body: - if isinstance(item, list): - format.append(concat(item).replace('%', '%%')) - else: - format.append('%s') - arguments.append(item) - self.writeline('yield ') - self.write(repr(concat(format)) + ' % (') - self.indent() - for argument in arguments: - self.newline(argument) - close = 0 - if frame.eval_ctx.volatile: - self.write('(escape if context.eval_ctx.autoescape else' - ' to_string)(') - close += 1 - elif frame.eval_ctx.autoescape: - self.write('escape(') - close += 1 - if self.environment.finalize is not None: - self.write('environment.finalize(') - if getattr(self.environment.finalize, - 'contextfunction', False): - self.write('context, ') - elif getattr(self.environment.finalize, - 'evalcontextfunction', False): - self.write('context.eval_ctx, ') - elif getattr(self.environment.finalize, - 'environmentfunction', False): - self.write('environment, ') - close += 1 - self.visit(argument, frame) - self.write(')' * close + ', ') + # A node to be evaluated at runtime. + self._output_child_pre(item, frame, finalize) + self.visit(item, frame) + self._output_child_post(item, frame, finalize) + + if frame.buffer is not None: + self.write(",") + + if frame.buffer is not None: self.outdent() - self.writeline(')') + self.writeline(")" if len(body) == 1 else "))") - if outdent_later: + if frame.require_output_check: self.outdent() def visit_Assign(self, node, frame): self.push_assign_tracking() self.newline(node) self.visit(node.target, frame) - self.write(' = ') + self.write(" = ") self.visit(node.node, frame) self.pop_assign_tracking(frame) @@ -1384,20 +1471,19 @@ class CodeGenerator(NodeVisitor): self.blockvisit(node.body, block_frame) self.newline(node) self.visit(node.target, frame) - self.write(' = (Markup if context.eval_ctx.autoescape ' - 'else identity)(') + self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") if node.filter is not None: self.visit_Filter(node.filter, block_frame) else: - self.write('concat(%s)' % block_frame.buffer) - self.write(')') + self.write("concat(%s)" % block_frame.buffer) + self.write(")") self.pop_assign_tracking(frame) self.leave_frame(block_frame) # -- Expression Visitors def visit_Name(self, node, frame): - if node.ctx == 'store' and frame.toplevel: + if node.ctx == "store" and frame.toplevel: if self._assign_stack: self._assign_stack[-1].add(node.name) ref = frame.symbols.ref(node.name) @@ -1405,12 +1491,17 @@ class CodeGenerator(NodeVisitor): # If we are looking up a variable we might have to deal with the # case where it's undefined. We can skip that case if the load # instruction indicates a parameter which are always defined. - if node.ctx == 'load': + if node.ctx == "load": load = frame.symbols.find_load(ref) - if not (load is not None and load[0] == VAR_LOAD_PARAMETER and \ - not self.parameter_is_undeclared(ref)): - self.write('(undefined(name=%r) if %s is missing else %s)' % - (node.name, ref, ref)) + if not ( + load is not None + and load[0] == VAR_LOAD_PARAMETER + and not self.parameter_is_undeclared(ref) + ): + self.write( + "(undefined(name=%r) if %s is missing else %s)" + % (node.name, ref, ref) + ) return self.write(ref) @@ -1420,12 +1511,14 @@ class CodeGenerator(NodeVisitor): # `foo.bar` notation they will be parsed as a normal attribute access # when used anywhere but in a `set` context ref = frame.symbols.ref(node.name) - self.writeline('if not isinstance(%s, Namespace):' % ref) + self.writeline("if not isinstance(%s, Namespace):" % ref) self.indent() - self.writeline('raise TemplateRuntimeError(%r)' % - 'cannot assign attribute on non-namespace object') + self.writeline( + "raise TemplateRuntimeError(%r)" + % "cannot assign attribute on non-namespace object" + ) self.outdent() - self.writeline('%s[%r]' % (ref, node.attr)) + self.writeline("%s[%r]" % (ref, node.attr)) def visit_Const(self, node, frame): val = node.as_const(frame.eval_ctx) @@ -1438,230 +1531,256 @@ class CodeGenerator(NodeVisitor): try: self.write(repr(node.as_const(frame.eval_ctx))) except nodes.Impossible: - self.write('(Markup if context.eval_ctx.autoescape else identity)(%r)' - % node.data) + self.write( + "(Markup if context.eval_ctx.autoescape else identity)(%r)" % node.data + ) def visit_Tuple(self, node, frame): - self.write('(') + self.write("(") idx = -1 for idx, item in enumerate(node.items): if idx: - self.write(', ') + self.write(", ") self.visit(item, frame) - self.write(idx == 0 and ',)' or ')') + self.write(idx == 0 and ",)" or ")") def visit_List(self, node, frame): - self.write('[') + self.write("[") for idx, item in enumerate(node.items): if idx: - self.write(', ') + self.write(", ") self.visit(item, frame) - self.write(']') + self.write("]") def visit_Dict(self, node, frame): - self.write('{') + self.write("{") for idx, item in enumerate(node.items): if idx: - self.write(', ') + self.write(", ") self.visit(item.key, frame) - self.write(': ') + self.write(": ") self.visit(item.value, frame) - self.write('}') + self.write("}") - def binop(operator, interceptable=True): + def binop(operator, interceptable=True): # noqa: B902 @optimizeconst def visitor(self, node, frame): - if self.environment.sandboxed and \ - operator in self.environment.intercepted_binops: - self.write('environment.call_binop(context, %r, ' % operator) + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_binops + ): + self.write("environment.call_binop(context, %r, " % operator) self.visit(node.left, frame) - self.write(', ') + self.write(", ") self.visit(node.right, frame) else: - self.write('(') + self.write("(") self.visit(node.left, frame) - self.write(' %s ' % operator) + self.write(" %s " % operator) self.visit(node.right, frame) - self.write(')') + self.write(")") + return visitor - def uaop(operator, interceptable=True): + def uaop(operator, interceptable=True): # noqa: B902 @optimizeconst def visitor(self, node, frame): - if self.environment.sandboxed and \ - operator in self.environment.intercepted_unops: - self.write('environment.call_unop(context, %r, ' % operator) + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_unops + ): + self.write("environment.call_unop(context, %r, " % operator) self.visit(node.node, frame) else: - self.write('(' + operator) + self.write("(" + operator) self.visit(node.node, frame) - self.write(')') + self.write(")") + return visitor - visit_Add = binop('+') - visit_Sub = binop('-') - visit_Mul = binop('*') - visit_Div = binop('/') - visit_FloorDiv = binop('//') - visit_Pow = binop('**') - visit_Mod = binop('%') - visit_And = binop('and', interceptable=False) - visit_Or = binop('or', interceptable=False) - visit_Pos = uaop('+') - visit_Neg = uaop('-') - visit_Not = uaop('not ', interceptable=False) + visit_Add = binop("+") + visit_Sub = binop("-") + visit_Mul = binop("*") + visit_Div = binop("/") + visit_FloorDiv = binop("//") + visit_Pow = binop("**") + visit_Mod = binop("%") + visit_And = binop("and", interceptable=False) + visit_Or = binop("or", interceptable=False) + visit_Pos = uaop("+") + visit_Neg = uaop("-") + visit_Not = uaop("not ", interceptable=False) del binop, uaop @optimizeconst def visit_Concat(self, node, frame): if frame.eval_ctx.volatile: - func_name = '(context.eval_ctx.volatile and' \ - ' markup_join or unicode_join)' + func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)" elif frame.eval_ctx.autoescape: - func_name = 'markup_join' + func_name = "markup_join" else: - func_name = 'unicode_join' - self.write('%s((' % func_name) + func_name = "unicode_join" + self.write("%s((" % func_name) for arg in node.nodes: self.visit(arg, frame) - self.write(', ') - self.write('))') + self.write(", ") + self.write("))") @optimizeconst def visit_Compare(self, node, frame): + self.write("(") self.visit(node.expr, frame) for op in node.ops: self.visit(op, frame) + self.write(")") def visit_Operand(self, node, frame): - self.write(' %s ' % operators[node.op]) + self.write(" %s " % operators[node.op]) self.visit(node.expr, frame) @optimizeconst def visit_Getattr(self, node, frame): - self.write('environment.getattr(') + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getattr(") self.visit(node.node, frame) - self.write(', %r)' % node.attr) + self.write(", %r)" % node.attr) + + if self.environment.is_async: + self.write("))") @optimizeconst def visit_Getitem(self, node, frame): # slices bypass the environment getitem method. if isinstance(node.arg, nodes.Slice): self.visit(node.node, frame) - self.write('[') + self.write("[") self.visit(node.arg, frame) - self.write(']') + self.write("]") else: - self.write('environment.getitem(') + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getitem(") self.visit(node.node, frame) - self.write(', ') + self.write(", ") self.visit(node.arg, frame) - self.write(')') + self.write(")") + + if self.environment.is_async: + self.write("))") def visit_Slice(self, node, frame): if node.start is not None: self.visit(node.start, frame) - self.write(':') + self.write(":") if node.stop is not None: self.visit(node.stop, frame) if node.step is not None: - self.write(':') + self.write(":") self.visit(node.step, frame) @optimizeconst def visit_Filter(self, node, frame): if self.environment.is_async: - self.write('await auto_await(') - self.write(self.filters[node.name] + '(') + self.write("await auto_await(") + self.write(self.filters[node.name] + "(") func = self.environment.filters.get(node.name) if func is None: - self.fail('no filter named %r' % node.name, node.lineno) - if getattr(func, 'contextfilter', False): - self.write('context, ') - elif getattr(func, 'evalcontextfilter', False): - self.write('context.eval_ctx, ') - elif getattr(func, 'environmentfilter', False): - self.write('environment, ') + self.fail("no filter named %r" % node.name, node.lineno) + if getattr(func, "contextfilter", False) is True: + self.write("context, ") + elif getattr(func, "evalcontextfilter", False) is True: + self.write("context.eval_ctx, ") + elif getattr(func, "environmentfilter", False) is True: + self.write("environment, ") # if the filter node is None we are inside a filter block # and want to write to the current buffer if node.node is not None: self.visit(node.node, frame) elif frame.eval_ctx.volatile: - self.write('(context.eval_ctx.autoescape and' - ' Markup(concat(%s)) or concat(%s))' % - (frame.buffer, frame.buffer)) + self.write( + "(context.eval_ctx.autoescape and" + " Markup(concat(%s)) or concat(%s))" % (frame.buffer, frame.buffer) + ) elif frame.eval_ctx.autoescape: - self.write('Markup(concat(%s))' % frame.buffer) + self.write("Markup(concat(%s))" % frame.buffer) else: - self.write('concat(%s)' % frame.buffer) + self.write("concat(%s)" % frame.buffer) self.signature(node, frame) - self.write(')') + self.write(")") if self.environment.is_async: - self.write(')') + self.write(")") @optimizeconst def visit_Test(self, node, frame): - self.write(self.tests[node.name] + '(') + self.write(self.tests[node.name] + "(") if node.name not in self.environment.tests: - self.fail('no test named %r' % node.name, node.lineno) + self.fail("no test named %r" % node.name, node.lineno) self.visit(node.node, frame) self.signature(node, frame) - self.write(')') + self.write(")") @optimizeconst def visit_CondExpr(self, node, frame): def write_expr2(): if node.expr2 is not None: return self.visit(node.expr2, frame) - self.write('undefined(%r)' % ('the inline if-' - 'expression on %s evaluated to false and ' - 'no else section was defined.' % self.position(node))) - - self.write('(') + self.write( + "cond_expr_undefined(%r)" + % ( + "the inline if-" + "expression on %s evaluated to false and " + "no else section was defined." % self.position(node) + ) + ) + + self.write("(") self.visit(node.expr1, frame) - self.write(' if ') + self.write(" if ") self.visit(node.test, frame) - self.write(' else ') + self.write(" else ") write_expr2() - self.write(')') + self.write(")") @optimizeconst def visit_Call(self, node, frame, forward_caller=False): if self.environment.is_async: - self.write('await auto_await(') + self.write("await auto_await(") if self.environment.sandboxed: - self.write('environment.call(context, ') + self.write("environment.call(context, ") else: - self.write('context.call(') + self.write("context.call(") self.visit(node.node, frame) - extra_kwargs = forward_caller and {'caller': 'caller'} or None + extra_kwargs = forward_caller and {"caller": "caller"} or None self.signature(node, frame, extra_kwargs) - self.write(')') + self.write(")") if self.environment.is_async: - self.write(')') + self.write(")") def visit_Keyword(self, node, frame): - self.write(node.key + '=') + self.write(node.key + "=") self.visit(node.value, frame) # -- Unused nodes for extensions def visit_MarkSafe(self, node, frame): - self.write('Markup(') + self.write("Markup(") self.visit(node.expr, frame) - self.write(')') + self.write(")") def visit_MarkSafeIfAutoescape(self, node, frame): - self.write('(context.eval_ctx.autoescape and Markup or identity)(') + self.write("(context.eval_ctx.autoescape and Markup or identity)(") self.visit(node.expr, frame) - self.write(')') + self.write(")") def visit_EnvironmentAttribute(self, node, frame): - self.write('environment.' + node.name) + self.write("environment." + node.name) def visit_ExtensionAttribute(self, node, frame): - self.write('environment.extensions[%r].%s' % (node.identifier, node.name)) + self.write("environment.extensions[%r].%s" % (node.identifier, node.name)) def visit_ImportedName(self, node, frame): self.write(self.import_aliases[node.importname]) @@ -1670,13 +1789,16 @@ class CodeGenerator(NodeVisitor): self.write(node.name) def visit_ContextReference(self, node, frame): - self.write('context') + self.write("context") + + def visit_DerivedContextReference(self, node, frame): + self.write(self.derive_context(frame)) def visit_Continue(self, node, frame): - self.writeline('continue', node) + self.writeline("continue", node) def visit_Break(self, node, frame): - self.writeline('break', node) + self.writeline("break", node) def visit_Scope(self, node, frame): scope_frame = frame.inner() @@ -1687,8 +1809,8 @@ class CodeGenerator(NodeVisitor): def visit_OverlayScope(self, node, frame): ctx = self.temporary_identifier() - self.writeline('%s = %s' % (ctx, self.derive_context(frame))) - self.writeline('%s.vars = ' % ctx) + self.writeline("%s = %s" % (ctx, self.derive_context(frame))) + self.writeline("%s.vars = " % ctx) self.visit(node.context, frame) self.push_context_reference(ctx) @@ -1701,7 +1823,7 @@ class CodeGenerator(NodeVisitor): def visit_EvalContextModifier(self, node, frame): for keyword in node.options: - self.writeline('context.eval_ctx.%s = ' % keyword.key) + self.writeline("context.eval_ctx.%s = " % keyword.key) self.visit(keyword.value, frame) try: val = keyword.value.as_const(frame.eval_ctx) @@ -1713,9 +1835,9 @@ class CodeGenerator(NodeVisitor): def visit_ScopedEvalContextModifier(self, node, frame): old_ctx_name = self.temporary_identifier() saved_ctx = frame.eval_ctx.save() - self.writeline('%s = context.eval_ctx.save()' % old_ctx_name) + self.writeline("%s = context.eval_ctx.save()" % old_ctx_name) self.visit_EvalContextModifier(node, frame) for child in node.body: self.visit(child, frame) frame.eval_ctx.revert(saved_ctx) - self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name) + self.writeline("context.eval_ctx.revert(%s)" % old_ctx_name) diff --git a/lib/spack/external/jinja2/constants.py b/lib/spack/external/jinja2/constants.py index 11efd1ed15..bf7f2ca721 100644 --- a/lib/spack/external/jinja2/constants.py +++ b/lib/spack/external/jinja2/constants.py @@ -1,17 +1,6 @@ # -*- coding: utf-8 -*- -""" - jinja.constants - ~~~~~~~~~~~~~~~ - - Various constants. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" - - #: list of lorem ipsum words used by the lipsum() helper function -LOREM_IPSUM_WORDS = u'''\ +LOREM_IPSUM_WORDS = u"""\ a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at auctor augue bibendum blandit class commodo condimentum congue consectetuer consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus @@ -29,4 +18,4 @@ ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus -viverra volutpat vulputate''' +viverra volutpat vulputate""" diff --git a/lib/spack/external/jinja2/debug.py b/lib/spack/external/jinja2/debug.py index b61139f0cd..5d8aec31d0 100644 --- a/lib/spack/external/jinja2/debug.py +++ b/lib/spack/external/jinja2/debug.py @@ -1,372 +1,268 @@ -# -*- coding: utf-8 -*- -""" - jinja2.debug - ~~~~~~~~~~~~ - - Implements the debug interface for Jinja. This module does some pretty - ugly stuff with the Python traceback system in order to achieve tracebacks - with correct line numbers, locals and contents. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" import sys -import traceback -from types import TracebackType, CodeType -from jinja2.utils import missing, internal_code -from jinja2.exceptions import TemplateSyntaxError -from jinja2._compat import iteritems, reraise, PY2 +from types import CodeType -# on pypy we can take advantage of transparent proxies -try: - from __pypy__ import tproxy -except ImportError: - tproxy = None +from . import TemplateSyntaxError +from ._compat import PYPY +from .utils import internal_code +from .utils import missing -# how does the raise helper look like? -try: - exec("raise TypeError, 'foo'") -except SyntaxError: - raise_helper = 'raise __jinja_exception__[1]' -except TypeError: - raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]' +def rewrite_traceback_stack(source=None): + """Rewrite the current exception to replace any tracebacks from + within compiled template code with tracebacks that look like they + came from the template source. + This must be called within an ``except`` block. -class TracebackFrameProxy(object): - """Proxies a traceback frame.""" + :param exc_info: A :meth:`sys.exc_info` tuple. If not provided, + the current ``exc_info`` is used. + :param source: For ``TemplateSyntaxError``, the original source if + known. + :return: A :meth:`sys.exc_info` tuple that can be re-raised. + """ + exc_type, exc_value, tb = sys.exc_info() - def __init__(self, tb): - self.tb = tb - self._tb_next = None + if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: + exc_value.translated = True + exc_value.source = source - @property - def tb_next(self): - return self._tb_next + try: + # Remove the old traceback on Python 3, otherwise the frames + # from the compiler still show up. + exc_value.with_traceback(None) + except AttributeError: + pass - def set_next(self, next): - if tb_set_next is not None: - try: - tb_set_next(self.tb, next and next.tb or None) - except Exception: - # this function can fail due to all the hackery it does - # on various python implementations. We just catch errors - # down and ignore them if necessary. - pass - self._tb_next = next - - @property - def is_jinja_frame(self): - return '__jinja_template__' in self.tb.tb_frame.f_globals - - def __getattr__(self, name): - return getattr(self.tb, name) - - -def make_frame_proxy(frame): - proxy = TracebackFrameProxy(frame) - if tproxy is None: - return proxy - def operation_handler(operation, *args, **kwargs): - if operation in ('__getattribute__', '__getattr__'): - return getattr(proxy, args[0]) - elif operation == '__setattr__': - proxy.__setattr__(*args, **kwargs) - else: - return getattr(proxy, operation)(*args, **kwargs) - return tproxy(TracebackType, operation_handler) - - -class ProcessedTraceback(object): - """Holds a Jinja preprocessed traceback for printing or reraising.""" - - def __init__(self, exc_type, exc_value, frames): - assert frames, 'no frames for this traceback?' - self.exc_type = exc_type - self.exc_value = exc_value - self.frames = frames - - # newly concatenate the frames (which are proxies) - prev_tb = None - for tb in self.frames: - if prev_tb is not None: - prev_tb.set_next(tb) - prev_tb = tb - prev_tb.set_next(None) - - def render_as_text(self, limit=None): - """Return a string with the traceback.""" - lines = traceback.format_exception(self.exc_type, self.exc_value, - self.frames[0], limit=limit) - return ''.join(lines).rstrip() - - def render_as_html(self, full=False): - """Return a unicode string with the traceback as rendered HTML.""" - from jinja2.debugrenderer import render_traceback - return u'%s\n\n' % ( - render_traceback(self, full=full), - self.render_as_text().decode('utf-8', 'replace') + # Outside of runtime, so the frame isn't executing template + # code, but it still needs to point at the template. + tb = fake_traceback( + exc_value, None, exc_value.filename or "", exc_value.lineno ) - - @property - def is_template_syntax_error(self): - """`True` if this is a template syntax error.""" - return isinstance(self.exc_value, TemplateSyntaxError) - - @property - def exc_info(self): - """Exception info tuple with a proxy around the frame objects.""" - return self.exc_type, self.exc_value, self.frames[0] - - @property - def standard_exc_info(self): - """Standard python exc_info for re-raising""" - tb = self.frames[0] - # the frame will be an actual traceback (or transparent proxy) if - # we are on pypy or a python implementation with support for tproxy - if type(tb) is not TracebackType: - tb = tb.tb - return self.exc_type, self.exc_value, tb - - -def make_traceback(exc_info, source_hint=None): - """Creates a processed traceback object from the exc_info.""" - exc_type, exc_value, tb = exc_info - if isinstance(exc_value, TemplateSyntaxError): - exc_info = translate_syntax_error(exc_value, source_hint) - initial_skip = 0 else: - initial_skip = 1 - return translate_exception(exc_info, initial_skip) - - -def translate_syntax_error(error, source=None): - """Rewrites a syntax error to please traceback systems.""" - error.source = source - error.translated = True - exc_info = (error.__class__, error, None) - filename = error.filename - if filename is None: - filename = '' - return fake_exc_info(exc_info, filename, error.lineno) + # Skip the frame for the render function. + tb = tb.tb_next + stack = [] -def translate_exception(exc_info, initial_skip=0): - """If passed an exc_info it will automatically rewrite the exceptions - all the way down to the correct line numbers and frames. - """ - tb = exc_info[2] - frames = [] - - # skip some internal frames if wanted - for x in range(initial_skip): - if tb is not None: - tb = tb.tb_next - initial_tb = tb - + # Build the stack of traceback object, replacing any in template + # code with the source file and line information. while tb is not None: - # skip frames decorated with @internalcode. These are internal - # calls we can't avoid and that are useless in template debugging - # output. + # Skip frames decorated with @internalcode. These are internal + # calls that aren't useful in template debugging output. if tb.tb_frame.f_code in internal_code: tb = tb.tb_next continue - # save a reference to the next frame if we override the current - # one with a faked one. - next = tb.tb_next + template = tb.tb_frame.f_globals.get("__jinja_template__") - # fake template exceptions - template = tb.tb_frame.f_globals.get('__jinja_template__') if template is not None: lineno = template.get_corresponding_lineno(tb.tb_lineno) - tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, - lineno)[2] + fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) + stack.append(fake_tb) + else: + stack.append(tb) - frames.append(make_frame_proxy(tb)) - tb = next + tb = tb.tb_next - # if we don't have any exceptions in the frames left, we have to - # reraise it unchanged. - # XXX: can we backup here? when could this happen? - if not frames: - reraise(exc_info[0], exc_info[1], exc_info[2]) + tb_next = None - return ProcessedTraceback(exc_info[0], exc_info[1], frames) + # Assign tb_next in reverse to avoid circular references. + for tb in reversed(stack): + tb_next = tb_set_next(tb, tb_next) + return exc_type, exc_value, tb_next -def get_jinja_locals(real_locals): - ctx = real_locals.get('context') - if ctx: - locals = ctx.get_all().copy() + +def fake_traceback(exc_value, tb, filename, lineno): + """Produce a new traceback object that looks like it came from the + template source instead of the compiled code. The filename, line + number, and location name will point to the template, and the local + variables will be the current template context. + + :param exc_value: The original exception to be re-raised to create + the new traceback. + :param tb: The original traceback to get the local variables and + code info from. + :param filename: The template filename. + :param lineno: The line number in the template source. + """ + if tb is not None: + # Replace the real locals with the context that would be + # available at that point in the template. + locals = get_template_locals(tb.tb_frame.f_locals) + locals.pop("__jinja_exception__", None) else: locals = {} + globals = { + "__name__": filename, + "__file__": filename, + "__jinja_exception__": exc_value, + } + # Raise an exception at the correct line number. + code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec") + + # Build a new code object that points to the template file and + # replaces the location with a block name. + try: + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = 'block "%s"' % function[6:] + + # Collect arguments for the new code object. CodeType only + # accepts positional arguments, and arguments were inserted in + # new Python versions. + code_args = [] + + for attr in ( + "argcount", + "posonlyargcount", # Python 3.8 + "kwonlyargcount", # Python 3 + "nlocals", + "stacksize", + "flags", + "code", # codestring + "consts", # constants + "names", + "varnames", + ("filename", filename), + ("name", location), + "firstlineno", + "lnotab", + "freevars", + "cellvars", + ): + if isinstance(attr, tuple): + # Replace with given value. + code_args.append(attr[1]) + continue + + try: + # Copy original value if it exists. + code_args.append(getattr(code, "co_" + attr)) + except AttributeError: + # Some arguments were added later. + continue + + code = CodeType(*code_args) + except Exception: + # Some environments such as Google App Engine don't support + # modifying code objects. + pass + + # Execute the new code, which is guaranteed to raise, and return + # the new traceback without this frame. + try: + exec(code, globals, locals) + except BaseException: + return sys.exc_info()[2].tb_next + + +def get_template_locals(real_locals): + """Based on the runtime locals, get the context that would be + available at that point in the template. + """ + # Start with the current template context. + ctx = real_locals.get("context") + + if ctx: + data = ctx.get_all().copy() + else: + data = {} + + # Might be in a derived context that only sets local variables + # rather than pushing a context. Local variables follow the scheme + # l_depth_name. Find the highest-depth local that has a value for + # each name. local_overrides = {} - for name, value in iteritems(real_locals): - if not name.startswith('l_') or value is missing: + for name, value in real_locals.items(): + if not name.startswith("l_") or value is missing: + # Not a template variable, or no longer relevant. continue + try: - _, depth, name = name.split('_', 2) + _, depth, name = name.split("_", 2) depth = int(depth) except ValueError: continue + cur_depth = local_overrides.get(name, (-1,))[0] + if cur_depth < depth: local_overrides[name] = (depth, value) - for name, (_, value) in iteritems(local_overrides): + # Modify the context with any derived context. + for name, (_, value) in local_overrides.items(): if value is missing: - locals.pop(name, None) + data.pop(name, None) else: - locals[name] = value + data[name] = value - return locals + return data -def fake_exc_info(exc_info, filename, lineno): - """Helper for `translate_exception`.""" - exc_type, exc_value, tb = exc_info +if sys.version_info >= (3, 7): + # tb_next is directly assignable as of Python 3.7 + def tb_set_next(tb, tb_next): + tb.tb_next = tb_next + return tb - # figure the real context out - if tb is not None: - locals = get_jinja_locals(tb.tb_frame.f_locals) - # if there is a local called __jinja_exception__, we get - # rid of it to not break the debug functionality. - locals.pop('__jinja_exception__', None) - else: - locals = {} - - # assamble fake globals we need - globals = { - '__name__': filename, - '__file__': filename, - '__jinja_exception__': exc_info[:2], - - # we don't want to keep the reference to the template around - # to not cause circular dependencies, but we mark it as Jinja - # frame for the ProcessedTraceback - '__jinja_template__': None - } - - # and fake the exception - code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec') - - # if it's possible, change the name of the code. This won't work - # on some python environments such as google appengine +elif PYPY: + # PyPy might have special support, and won't work with ctypes. try: - if tb is None: - location = 'template' - else: - function = tb.tb_frame.f_code.co_name - if function == 'root': - location = 'top-level template code' - elif function.startswith('block_'): - location = 'block "%s"' % function[6:] - else: - location = 'template' - - if PY2: - code = CodeType(0, code.co_nlocals, code.co_stacksize, - code.co_flags, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, - location, code.co_firstlineno, - code.co_lnotab, (), ()) - else: - code = CodeType(0, code.co_kwonlyargcount, - code.co_nlocals, code.co_stacksize, - code.co_flags, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, - location, code.co_firstlineno, - code.co_lnotab, (), ()) - except Exception as e: - pass + import tputil + except ImportError: + # Without tproxy support, use the original traceback. + def tb_set_next(tb, tb_next): + return tb - # execute the code and catch the new traceback - try: - exec(code, globals, locals) - except: - exc_info = sys.exc_info() - new_tb = exc_info[2].tb_next + else: + # With tproxy support, create a proxy around the traceback that + # returns the new tb_next. + def tb_set_next(tb, tb_next): + def controller(op): + if op.opname == "__getattribute__" and op.args[0] == "tb_next": + return tb_next - # return without this frame - return exc_info[:2] + (new_tb,) + return op.delegate() + return tputil.make_proxy(controller, obj=tb) -def _init_ugly_crap(): - """This function implements a few ugly things so that we can patch the - traceback objects. The function returned allows resetting `tb_next` on - any python traceback object. Do not attempt to use this on non cpython - interpreters - """ - import ctypes - from types import TracebackType - if PY2: - # figure out size of _Py_ssize_t for Python 2: - if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): - _Py_ssize_t = ctypes.c_int64 - else: - _Py_ssize_t = ctypes.c_int - else: - # platform ssize_t on Python 3 - _Py_ssize_t = ctypes.c_ssize_t +else: + # Use ctypes to assign tb_next at the C level since it's read-only + # from Python. + import ctypes - # regular python - class _PyObject(ctypes.Structure): - pass - _PyObject._fields_ = [ - ('ob_refcnt', _Py_ssize_t), - ('ob_type', ctypes.POINTER(_PyObject)) - ] - - # python with trace - if hasattr(sys, 'getobjects'): - class _PyObject(ctypes.Structure): - pass - _PyObject._fields_ = [ - ('_ob_next', ctypes.POINTER(_PyObject)), - ('_ob_prev', ctypes.POINTER(_PyObject)), - ('ob_refcnt', _Py_ssize_t), - ('ob_type', ctypes.POINTER(_PyObject)) + class _CTraceback(ctypes.Structure): + _fields_ = [ + # Extra PyObject slots when compiled with Py_TRACE_REFS. + ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), + # Only care about tb_next as an object, not a traceback. + ("tb_next", ctypes.py_object), ] - class _Traceback(_PyObject): - pass - _Traceback._fields_ = [ - ('tb_next', ctypes.POINTER(_Traceback)), - ('tb_frame', ctypes.POINTER(_PyObject)), - ('tb_lasti', ctypes.c_int), - ('tb_lineno', ctypes.c_int) - ] - - def tb_set_next(tb, next): - """Set the tb_next attribute of a traceback object.""" - if not (isinstance(tb, TracebackType) and - (next is None or isinstance(next, TracebackType))): - raise TypeError('tb_set_next arguments must be traceback objects') - obj = _Traceback.from_address(id(tb)) - if tb.tb_next is not None: - old = _Traceback.from_address(id(tb.tb_next)) - old.ob_refcnt -= 1 - if next is None: - obj.tb_next = ctypes.POINTER(_Traceback)() - else: - next = _Traceback.from_address(id(next)) - next.ob_refcnt += 1 - obj.tb_next = ctypes.pointer(next) + def tb_set_next(tb, tb_next): + c_tb = _CTraceback.from_address(id(tb)) - return tb_set_next + # Clear out the old tb_next. + if tb.tb_next is not None: + c_tb_next = ctypes.py_object(tb.tb_next) + c_tb.tb_next = ctypes.py_object() + ctypes.pythonapi.Py_DecRef(c_tb_next) + # Assign the new tb_next. + if tb_next is not None: + c_tb_next = ctypes.py_object(tb_next) + ctypes.pythonapi.Py_IncRef(c_tb_next) + c_tb.tb_next = c_tb_next -# try to get a tb_set_next implementation if we don't have transparent -# proxies. -tb_set_next = None -if tproxy is None: - try: - tb_set_next = _init_ugly_crap() - except: - pass - del _init_ugly_crap + return tb diff --git a/lib/spack/external/jinja2/defaults.py b/lib/spack/external/jinja2/defaults.py index 7c93dec0ae..8e0e7d7710 100644 --- a/lib/spack/external/jinja2/defaults.py +++ b/lib/spack/external/jinja2/defaults.py @@ -1,56 +1,44 @@ # -*- coding: utf-8 -*- -""" - jinja2.defaults - ~~~~~~~~~~~~~~~ - - Jinja default filters and tags. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -from jinja2._compat import range_type -from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace - +from ._compat import range_type +from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 +from .tests import TESTS as DEFAULT_TESTS # noqa: F401 +from .utils import Cycler +from .utils import generate_lorem_ipsum +from .utils import Joiner +from .utils import Namespace # defaults for the parser / lexer -BLOCK_START_STRING = '{%' -BLOCK_END_STRING = '%}' -VARIABLE_START_STRING = '{{' -VARIABLE_END_STRING = '}}' -COMMENT_START_STRING = '{#' -COMMENT_END_STRING = '#}' +BLOCK_START_STRING = "{%" +BLOCK_END_STRING = "%}" +VARIABLE_START_STRING = "{{" +VARIABLE_END_STRING = "}}" +COMMENT_START_STRING = "{#" +COMMENT_END_STRING = "#}" LINE_STATEMENT_PREFIX = None LINE_COMMENT_PREFIX = None TRIM_BLOCKS = False LSTRIP_BLOCKS = False -NEWLINE_SEQUENCE = '\n' +NEWLINE_SEQUENCE = "\n" KEEP_TRAILING_NEWLINE = False - # default filters, tests and namespace -from jinja2.filters import FILTERS as DEFAULT_FILTERS -from jinja2.tests import TESTS as DEFAULT_TESTS + DEFAULT_NAMESPACE = { - 'range': range_type, - 'dict': dict, - 'lipsum': generate_lorem_ipsum, - 'cycler': Cycler, - 'joiner': Joiner, - 'namespace': Namespace + "range": range_type, + "dict": dict, + "lipsum": generate_lorem_ipsum, + "cycler": Cycler, + "joiner": Joiner, + "namespace": Namespace, } - # default policies DEFAULT_POLICIES = { - 'compiler.ascii_str': True, - 'urlize.rel': 'noopener', - 'urlize.target': None, - 'truncate.leeway': 5, - 'json.dumps_function': None, - 'json.dumps_kwargs': {'sort_keys': True}, - 'ext.i18n.trimmed': False, + "compiler.ascii_str": True, + "urlize.rel": "noopener", + "urlize.target": None, + "truncate.leeway": 5, + "json.dumps_function": None, + "json.dumps_kwargs": {"sort_keys": True}, + "ext.i18n.trimmed": False, } - - -# export all constants -__all__ = tuple(x for x in locals().keys() if x.isupper()) diff --git a/lib/spack/external/jinja2/environment.py b/lib/spack/external/jinja2/environment.py index 549d9afab4..8430390eea 100644 --- a/lib/spack/external/jinja2/environment.py +++ b/lib/spack/external/jinja2/environment.py @@ -1,60 +1,83 @@ # -*- coding: utf-8 -*- -""" - jinja2.environment - ~~~~~~~~~~~~~~~~~~ - - Provides a class that holds runtime and parsing time options. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. +"""Classes for managing templates and their runtime and compile time +options. """ import os import sys import weakref -from functools import reduce, partial -from jinja2 import nodes -from jinja2.defaults import BLOCK_START_STRING, \ - BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ - COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ - LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ - DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ - DEFAULT_POLICIES, KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS -from jinja2.lexer import get_lexer, TokenStream -from jinja2.parser import Parser -from jinja2.nodes import EvalContext -from jinja2.compiler import generate, CodeGenerator -from jinja2.runtime import Undefined, new_context, Context -from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ - TemplatesNotFound, TemplateRuntimeError -from jinja2.utils import import_string, LRUCache, Markup, missing, \ - concat, consume, internalcode, have_async_gen -from jinja2._compat import imap, ifilter, string_types, iteritems, \ - text_type, reraise, implements_iterator, implements_to_string, \ - encode_filename, PY2, PYPY - +from functools import partial +from functools import reduce + +from markupsafe import Markup + +from . import nodes +from ._compat import encode_filename +from ._compat import implements_iterator +from ._compat import implements_to_string +from ._compat import iteritems +from ._compat import PY2 +from ._compat import PYPY +from ._compat import reraise +from ._compat import string_types +from ._compat import text_type +from .compiler import CodeGenerator +from .compiler import generate +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import DEFAULT_FILTERS +from .defaults import DEFAULT_NAMESPACE +from .defaults import DEFAULT_POLICIES +from .defaults import DEFAULT_TESTS +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .lexer import get_lexer +from .lexer import TokenStream +from .nodes import EvalContext +from .parser import Parser +from .runtime import Context +from .runtime import new_context +from .runtime import Undefined +from .utils import concat +from .utils import consume +from .utils import have_async_gen +from .utils import import_string +from .utils import internalcode +from .utils import LRUCache +from .utils import missing # for direct template usage we have up to ten living environments _spontaneous_environments = LRUCache(10) -# the function to create jinja traceback objects. This is dynamically -# imported on the first exception in the exception handler. -_make_traceback = None +def get_spontaneous_environment(cls, *args): + """Return a new spontaneous environment. A spontaneous environment + is used for templates created directly rather than through an + existing environment. -def get_spontaneous_environment(*args): - """Return a new spontaneous environment. A spontaneous environment is an - unnamed and unaccessible (in theory) environment that is used for - templates generated from a string and not from the file system. + :param cls: Environment class to create. + :param args: Positional arguments passed to environment. """ + key = (cls, args) + try: - env = _spontaneous_environments.get(args) - except TypeError: - return Environment(*args) - if env is not None: + return _spontaneous_environments[key] + except KeyError: + _spontaneous_environments[key] = env = cls(*args) + env.shared = True return env - _spontaneous_environments[args] = env = Environment(*args) - env.shared = True - return env def create_cache(size): @@ -93,20 +116,25 @@ def fail_for_missing_callable(string, name): try: name._fail_with_undefined_error() except Exception as e: - msg = '%s (%s; did you forget to quote the callable name?)' % (msg, e) + msg = "%s (%s; did you forget to quote the callable name?)" % (msg, e) raise TemplateRuntimeError(msg) def _environment_sanity_check(environment): """Perform a sanity check on the environment.""" - assert issubclass(environment.undefined, Undefined), 'undefined must ' \ - 'be a subclass of undefined because filters depend on it.' - assert environment.block_start_string != \ - environment.variable_start_string != \ - environment.comment_start_string, 'block, variable and comment ' \ - 'start strings must be different' - assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ - 'newline_sequence set to unknown line ending string.' + assert issubclass( + environment.undefined, Undefined + ), "undefined must be a subclass of undefined because filters depend on it." + assert ( + environment.block_start_string + != environment.variable_start_string + != environment.comment_start_string + ), "block, variable and comment start strings must be different" + assert environment.newline_sequence in ( + "\r", + "\r\n", + "\n", + ), "newline_sequence set to unknown line ending string." return environment @@ -191,7 +219,7 @@ class Environment(object): `autoescape` If set to ``True`` the XML/HTML autoescaping feature is enabled by default. For more details about autoescaping see - :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also + :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also be a callable that is passed the template name and has to return ``True`` or ``False`` depending on autoescape should be enabled by default. @@ -249,10 +277,6 @@ class Environment(object): #: must not be modified shared = False - #: these are currently EXPERIMENTAL undocumented features. - exception_handler = None - exception_formatter = None - #: the class that is used for code generation. See #: :class:`~jinja2.compiler.CodeGenerator` for more information. code_generator_class = CodeGenerator @@ -261,29 +285,31 @@ class Environment(object): #: :class:`~jinja2.runtime.Context` for more information. context_class = Context - def __init__(self, - block_start_string=BLOCK_START_STRING, - block_end_string=BLOCK_END_STRING, - variable_start_string=VARIABLE_START_STRING, - variable_end_string=VARIABLE_END_STRING, - comment_start_string=COMMENT_START_STRING, - comment_end_string=COMMENT_END_STRING, - line_statement_prefix=LINE_STATEMENT_PREFIX, - line_comment_prefix=LINE_COMMENT_PREFIX, - trim_blocks=TRIM_BLOCKS, - lstrip_blocks=LSTRIP_BLOCKS, - newline_sequence=NEWLINE_SEQUENCE, - keep_trailing_newline=KEEP_TRAILING_NEWLINE, - extensions=(), - optimized=True, - undefined=Undefined, - finalize=None, - autoescape=False, - loader=None, - cache_size=400, - auto_reload=True, - bytecode_cache=None, - enable_async=False): + def __init__( + self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False, + ): # !!Important notice!! # The constructor accepts quite a few arguments that should be # passed by keyword rather than position. However it's important to @@ -334,6 +360,9 @@ class Environment(object): self.enable_async = enable_async self.is_async = self.enable_async and have_async_gen + if self.is_async: + # runs patch_all() to enable async support + from . import asyncsupport # noqa: F401 _environment_sanity_check(self) @@ -353,15 +382,28 @@ class Environment(object): if not hasattr(self, key): setattr(self, key, value) - def overlay(self, block_start_string=missing, block_end_string=missing, - variable_start_string=missing, variable_end_string=missing, - comment_start_string=missing, comment_end_string=missing, - line_statement_prefix=missing, line_comment_prefix=missing, - trim_blocks=missing, lstrip_blocks=missing, - extensions=missing, optimized=missing, - undefined=missing, finalize=missing, autoescape=missing, - loader=missing, cache_size=missing, auto_reload=missing, - bytecode_cache=missing): + def overlay( + self, + block_start_string=missing, + block_end_string=missing, + variable_start_string=missing, + variable_end_string=missing, + comment_start_string=missing, + comment_end_string=missing, + line_statement_prefix=missing, + line_comment_prefix=missing, + trim_blocks=missing, + lstrip_blocks=missing, + extensions=missing, + optimized=missing, + undefined=missing, + finalize=missing, + autoescape=missing, + loader=missing, + cache_size=missing, + auto_reload=missing, + bytecode_cache=missing, + ): """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. Extensions cannot be removed for an overlayed environment. An overlayed @@ -374,7 +416,7 @@ class Environment(object): through. """ args = dict(locals()) - del args['self'], args['cache_size'], args['extensions'] + del args["self"], args["cache_size"], args["extensions"] rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) @@ -402,8 +444,7 @@ class Environment(object): def iter_extensions(self): """Iterates over the extensions by priority.""" - return iter(sorted(self.extensions.values(), - key=lambda x: x.priority)) + return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) def getitem(self, obj, argument): """Get an item or attribute of an object but prefer the item.""" @@ -435,8 +476,9 @@ class Environment(object): except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) - def call_filter(self, name, value, args=None, kwargs=None, - context=None, eval_ctx=None): + def call_filter( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): """Invokes a filter on a value the same way the compiler does it. Note that on Python 3 this might return a coroutine in case the @@ -448,21 +490,22 @@ class Environment(object): """ func = self.filters.get(name) if func is None: - fail_for_missing_callable('no filter named %r', name) + fail_for_missing_callable("no filter named %r", name) args = [value] + list(args or ()) - if getattr(func, 'contextfilter', False): + if getattr(func, "contextfilter", False) is True: if context is None: - raise TemplateRuntimeError('Attempted to invoke context ' - 'filter without context') + raise TemplateRuntimeError( + "Attempted to invoke context filter without context" + ) args.insert(0, context) - elif getattr(func, 'evalcontextfilter', False): + elif getattr(func, "evalcontextfilter", False) is True: if eval_ctx is None: if context is not None: eval_ctx = context.eval_ctx else: eval_ctx = EvalContext(self) args.insert(0, eval_ctx) - elif getattr(func, 'environmentfilter', False): + elif getattr(func, "environmentfilter", False) is True: args.insert(0, self) return func(*args, **(kwargs or {})) @@ -473,7 +516,7 @@ class Environment(object): """ func = self.tests.get(name) if func is None: - fail_for_missing_callable('no test named %r', name) + fail_for_missing_callable("no test named %r", name) return func(value, *(args or ()), **(kwargs or {})) @internalcode @@ -483,14 +526,13 @@ class Environment(object): executable source- or bytecode. This is useful for debugging or to extract information from templates. - If you are :ref:`developing Jinja2 extensions ` + If you are :ref:`developing Jinja extensions ` this gives you a good overview of the node tree generated. """ try: return self._parse(source, name, filename) except TemplateSyntaxError: - exc_info = sys.exc_info() - self.handle_exception(exc_info, source_hint=source) + self.handle_exception(source=source) def _parse(self, source, name, filename): """Internal parsing function used by `parse` and `compile`.""" @@ -510,16 +552,18 @@ class Environment(object): try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: - exc_info = sys.exc_info() - self.handle_exception(exc_info, source_hint=source) + self.handle_exception(source=source) def preprocess(self, source, name=None, filename=None): """Preprocesses the source with all extensions. This is automatically called for all parsing and compiling methods but *not* for :meth:`lex` because there you usually only want the actual source tokenized. """ - return reduce(lambda s, e: e.preprocess(s, name, filename), - self.iter_extensions(), text_type(source)) + return reduce( + lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), + text_type(source), + ) def _tokenize(self, source, name, filename=None, state=None): """Called by the parser to do the preprocessing and filtering @@ -539,8 +583,14 @@ class Environment(object): .. versionadded:: 2.5 """ - return generate(source, self, name, filename, defer_init=defer_init, - optimized=self.optimized) + return generate( + source, + self, + name, + filename, + defer_init=defer_init, + optimized=self.optimized, + ) def _compile(self, source, filename): """Internal hook that can be overridden to hook a different compile @@ -548,11 +598,10 @@ class Environment(object): .. versionadded:: 2.5 """ - return compile(source, filename, 'exec') + return compile(source, filename, "exec") @internalcode - def compile(self, source, name=None, filename=None, raw=False, - defer_init=False): + def compile(self, source, name=None, filename=None, raw=False, defer_init=False): """Compile a node or template source code. The `name` parameter is the load name of the template after it was joined using :meth:`join_path` if necessary, not the filename on the file system. @@ -577,18 +626,16 @@ class Environment(object): if isinstance(source, string_types): source_hint = source source = self._parse(source, name, filename) - source = self._generate(source, name, filename, - defer_init=defer_init) + source = self._generate(source, name, filename, defer_init=defer_init) if raw: return source if filename is None: - filename = '