unparser: implement operator precedence algorithm for unparser

Backport operator precedence algorithm from here:
    https://github.com/python/cpython/commit/397b96f6d7a89f778ebc0591e32216a8183fe667

This eliminates unnecessary parentheses from our unparsed output and makes Spack's unparser
consistent with the one in upstream Python 3.9+, with one exception.

Our parser normalizes argument order when `py_ver_consistent` is set, so that star arguments
in function calls come last.  We have to do this because Python 2's AST doesn't have information
about their actual order.

If we ever support only Python 3.9 and higher, we can easily switch over to `ast.unparse`, as
the unparsing is consistent except for this detail (modulo future changes to `ast.unparse`)

4 files changed, 16 insertions, 1045 deletions

diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py
index b334e5d608..f3c077b1bb 100644
--- a/lib/spack/external/__init__.py
+++ b/lib/spack/external/__init__.py
@@ -31,6 +31,22 @@ argparse
   vendored copy ever needs to be updated again:
   https://github.com/spack/spack/pull/6786/commits/dfcef577b77249106ea4e4c69a6cd9e64fa6c418
 
+astunparse
+----------------
+
+* Homepage: https://github.com/simonpercivall/astunparse
+* Usage: Unparsing Python ASTs for package hashes in Spack
+* Version: 1.6.3 (plus modifications)
+* Note: This is in ``spack.util.unparse`` because it's very heavily
+  modified, and we want to track coverage for it.
+  Specifically, we have modified this library to generate consistent unparsed ASTs
+  regardless of the Python version. It is based on:
+    1. The original ``astunparse`` library;
+    2. Modifications for consistency;
+    3. Backports from the ``ast.unparse`` function in Python 3.9 and later
+  The unparsing is now mostly consistent with upstream ``ast.unparse``, so if
+  we ever require Python 3.9 or higher, we can drop this external package.
+
 attrs
 ----------------
 
@@ -138,21 +154,4 @@ six
 * Usage: Python 2 and 3 compatibility utilities.
 * Version: 1.16.0
 
-
-spack_astunparse
-----------------
-
-* Homepage: https://github.com/simonpercivall/astunparse
-* Usage: Unparsing Python ASTs for package hashes in Spack
-* Version: 1.6.3 (plus modifications)
-* Note: We have modified this library to generate consistent unparsed ASTs
-  regardless of the Python version. It contains the original ``astunparse``
-  library, as well as modifications for consistency. It also contains
-  backports from the ``ast.unparse`` function in Python 3.9 and later, so
-  that it will generate output consistent with the builtin ``ast.unparse``
-  function, in case we ever want to drop astunparse as an external
-  dependency.  Because we have modified the parsing (potentially at the
-  cost of round-trippability of the code), we call this ``spack_astunparse``
-  to avoid confusion with ``astunparse``.
-
 """
diff --git a/lib/spack/external/spack_astunparse/LICENSE b/lib/spack/external/spack_astunparse/LICENSE
deleted file mode 100644
index 099193e449..0000000000
--- a/lib/spack/external/spack_astunparse/LICENSE
+++ /dev/null
@@ -1,64 +0,0 @@
-LICENSE
-=======
-
-Copyright (c) 2014, Simon Percivall
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-* Neither the name of AST Unparser nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are retained
-in Python alone or in any derivative version prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
diff --git a/lib/spack/external/spack_astunparse/__init__.py b/lib/spack/external/spack_astunparse/__init__.py
deleted file mode 100644
index 33c44b7812..0000000000
--- a/lib/spack/external/spack_astunparse/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# coding: utf-8
-from __future__ import absolute_import
-from six.moves import cStringIO
-from .unparser import Unparser
-
-
-__version__ = '1.6.3'
-
-
-def unparse(tree, py_ver_consistent=False):
-    v = cStringIO()
-    Unparser(tree, file=v, py_ver_consistent=py_ver_consistent)
-    return v.getvalue()
diff --git a/lib/spack/external/spack_astunparse/unparser.py b/lib/spack/external/spack_astunparse/unparser.py
deleted file mode 100644
index 56e60c4c14..0000000000
--- a/lib/spack/external/spack_astunparse/unparser.py
+++ /dev/null
@@ -1,951 +0,0 @@
-"Usage: unparse.py <path to source file>"
-from __future__ import print_function, unicode_literals
-
-import ast
-import os
-import sys
-import tokenize
-
-from contextlib import contextmanager
-
-import six
-from six import StringIO
-
-
-# TODO: if we require Python 3.7, use its `nullcontext()`
-@contextmanager
-def nullcontext():
-    yield
-
-
-# Large float and imaginary literals get turned into infinities in the AST.
-# We unparse those infinities to INFSTR.
-INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
-
-def interleave(inter, f, seq):
-    """Call f on each item in seq, calling inter() in between.
-    """
-    seq = iter(seq)
-    try:
-        f(next(seq))
-    except StopIteration:
-        pass
-    else:
-        for x in seq:
-            inter()
-            f(x)
-
-class Unparser:
-    """Methods in this class recursively traverse an AST and
-    output source code for the abstract syntax; original formatting
-    is disregarded. """
-
-    def __init__(self, tree, file = sys.stdout, py_ver_consistent=False):
-        """Unparser(tree, file=sys.stdout) -> None.
-         Print the source for tree to file.
-
-        Arguments:
-            py_ver_consistent (bool): if True, generate unparsed code that is
-                consistent between Python 2.7 and 3.5-3.10.
-
-        Consistency is achieved by:
-            1. Ensuring there are no spaces between unary operators and
-               their operands.
-            2. Ensuring that *args and **kwargs are always the last arguments,
-               regardless of the python version.
-            3. Always unparsing print as a function.
-            4. Not putting an extra comma after Python 2 class definitions.
-
-        Without these changes, the same source can generate different code for different
-        Python versions, depending on subtle AST differences.
-
-        One place where single source will generate an inconsistent AST is with
-        multi-argument print statements, e.g.::
-
-            print("foo", "bar", "baz")
-
-        In Python 2, this prints a tuple; in Python 3, it is the print function with
-        multiple arguments.  Use ``from __future__ import print_function`` to avoid
-        this inconsistency.
-
-        """
-        self.f = file
-        self.future_imports = []
-        self._indent = 0
-        self._py_ver_consistent = py_ver_consistent
-        self.dispatch(tree)
-        print("", file=self.f)
-        self.f.flush()
-
-    def fill(self, text = ""):
-        "Indent a piece of text, according to the current indentation level"
-        self.f.write("\n"+"    "*self._indent + text)
-
-    def write(self, text):
-        "Append a piece of text to the current line."
-        self.f.write(six.text_type(text))
-
-    class _Block:
-        """A context manager for preparing the source for blocks. It adds
-        the character ':', increases the indentation on enter and decreases
-        the indentation on exit."""
-        def __init__(self, unparser):
-            self.unparser = unparser
-
-        def __enter__(self):
-            self.unparser.write(":")
-            self.unparser._indent += 1
-
-        def __exit__(self, exc_type, exc_value, traceback):
-            self.unparser._indent -= 1
-
-    def block(self):
-        return self._Block(self)
-
-    @contextmanager
-    def delimit(self, start, end):
-        """A context manager for preparing the source for expressions. It adds
-         *start* to the buffer and enters, after exit it adds *end*."""
-
-        self.write(start)
-        yield
-        self.write(end)
-
-    def delimit_if(self, start, end, condition):
-        if condition:
-            return self.delimit(start, end)
-        else:
-            return nullcontext()
-
-    def dispatch(self, tree):
-        "Dispatcher function, dispatching tree type T to method _T."
-        if isinstance(tree, list):
-            for t in tree:
-                self.dispatch(t)
-            return
-        meth = getattr(self, "_"+tree.__class__.__name__)
-        meth(tree)
-
-
-    ############### Unparsing methods ######################
-    # There should be one method per concrete grammar type #
-    # Constructors should be grouped by sum type. Ideally, #
-    # this would follow the order in the grammar, but      #
-    # currently doesn't.                                   #
-    ########################################################
-
-    def _Module(self, tree):
-        for stmt in tree.body:
-            self.dispatch(stmt)
-
-    def _Interactive(self, tree):
-        for stmt in tree.body:
-            self.dispatch(stmt)
-
-    def _Expression(self, tree):
-        self.dispatch(tree.body)
-
-    # stmt
-    def _Expr(self, tree):
-        self.fill()
-        self.dispatch(tree.value)
-
-    def _NamedExpr(self, tree):
-        with self.delimit("(", ")"):
-            self.dispatch(tree.target)
-            self.write(" := ")
-            self.dispatch(tree.value)
-
-    def _Import(self, t):
-        self.fill("import ")
-        interleave(lambda: self.write(", "), self.dispatch, t.names)
-
-    def _ImportFrom(self, t):
-        # A from __future__ import may affect unparsing, so record it.
-        if t.module and t.module == '__future__':
-            self.future_imports.extend(n.name for n in t.names)
-
-        self.fill("from ")
-        self.write("." * t.level)
-        if t.module:
-            self.write(t.module)
-        self.write(" import ")
-        interleave(lambda: self.write(", "), self.dispatch, t.names)
-
-    def _Assign(self, t):
-        self.fill()
-        for target in t.targets:
-            self.dispatch(target)
-            self.write(" = ")
-        self.dispatch(t.value)
-
-    def _AugAssign(self, t):
-        self.fill()
-        self.dispatch(t.target)
-        self.write(" "+self.binop[t.op.__class__.__name__]+"= ")
-        self.dispatch(t.value)
-
-    def _AnnAssign(self, t):
-        self.fill()
-        with self.delimit_if(
-                "(", ")", not node.simple and isinstance(t.target, ast.Name)):
-            self.dispatch(t.target)
-        self.write(": ")
-        self.dispatch(t.annotation)
-        if t.value:
-            self.write(" = ")
-            self.dispatch(t.value)
-
-    def _Return(self, t):
-        self.fill("return")
-        if t.value:
-            self.write(" ")
-            self.dispatch(t.value)
-
-    def _Pass(self, t):
-        self.fill("pass")
-
-    def _Break(self, t):
-        self.fill("break")
-
-    def _Continue(self, t):
-        self.fill("continue")
-
-    def _Delete(self, t):
-        self.fill("del ")
-        interleave(lambda: self.write(", "), self.dispatch, t.targets)
-
-    def _Assert(self, t):
-        self.fill("assert ")
-        self.dispatch(t.test)
-        if t.msg:
-            self.write(", ")
-            self.dispatch(t.msg)
-
-    def _Exec(self, t):
-        self.fill("exec ")
-        self.dispatch(t.body)
-        if t.globals:
-            self.write(" in ")
-            self.dispatch(t.globals)
-        if t.locals:
-            self.write(", ")
-            self.dispatch(t.locals)
-
-    def _Print(self, t):
-        # Use print function so that python 2 unparsing is consistent with 3
-        if self._py_ver_consistent:
-            self.fill("print(")
-        else:
-            self.fill("print ")
-
-        do_comma = False
-        if t.dest:
-            self.write(">>")
-            self.dispatch(t.dest)
-            do_comma = True
-        for e in t.values:
-            if do_comma:self.write(", ")
-            else:do_comma=True
-            self.dispatch(e)
-        if not t.nl:
-            self.write(",")
-
-        if self._py_ver_consistent:
-            self.write(")")
-
-    def _Global(self, t):
-        self.fill("global ")
-        interleave(lambda: self.write(", "), self.write, t.names)
-
-    def _Nonlocal(self, t):
-        self.fill("nonlocal ")
-        interleave(lambda: self.write(", "), self.write, t.names)
-
-    def _Await(self, t):
-        with self.delimit("(", ")"):
-            self.write("await")
-            if t.value:
-                self.write(" ")
-                self.dispatch(t.value)
-
-    def _Yield(self, t):
-        with self.delimit("(", ")"):
-            self.write("yield")
-            if t.value:
-                self.write(" ")
-                self.dispatch(t.value)
-
-    def _YieldFrom(self, t):
-        with self.delimit("(", ")"):
-            self.write("yield from")
-            if t.value:
-                self.write(" ")
-                self.dispatch(t.value)
-
-    def _Raise(self, t):
-        self.fill("raise")
-        if six.PY3:
-            if not t.exc:
-                assert not t.cause
-                return
-            self.write(" ")
-            self.dispatch(t.exc)
-            if t.cause:
-                self.write(" from ")
-                self.dispatch(t.cause)
-        else:
-            self.write(" ")
-            if t.type:
-                self.dispatch(t.type)
-            if t.inst:
-                self.write(", ")
-                self.dispatch(t.inst)
-            if t.tback:
-                self.write(", ")
-                self.dispatch(t.tback)
-
-    def _Try(self, t):
-        self.fill("try")
-        with self.block():
-            self.dispatch(t.body)
-        for ex in t.handlers:
-            self.dispatch(ex)
-        if t.orelse:
-            self.fill("else")
-            with self.block():
-                self.dispatch(t.orelse)
-        if t.finalbody:
-            self.fill("finally")
-            with self.block():
-                self.dispatch(t.finalbody)
-
-    def _TryExcept(self, t):
-        self.fill("try")
-        with self.block():
-            self.dispatch(t.body)
-
-        for ex in t.handlers:
-            self.dispatch(ex)
-        if t.orelse:
-            self.fill("else")
-            with self.block():
-                self.dispatch(t.orelse)
-
-    def _TryFinally(self, t):
-        if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
-            # try-except-finally
-            self.dispatch(t.body)
-        else:
-            self.fill("try")
-            with self.block():
-                self.dispatch(t.body)
-
-        self.fill("finally")
-        with self.block():
-            self.dispatch(t.finalbody)
-
-    def _ExceptHandler(self, t):
-        self.fill("except")
-        if t.type:
-            self.write(" ")
-            self.dispatch(t.type)
-        if t.name:
-            self.write(" as ")
-            if six.PY3:
-                self.write(t.name)
-            else:
-                self.dispatch(t.name)
-        with self.block():
-            self.dispatch(t.body)
-
-    def _ClassDef(self, t):
-        self.write("\n")
-        for deco in t.decorator_list:
-            self.fill("@")
-            self.dispatch(deco)
-        self.fill("class "+t.name)
-        if six.PY3:
-            with self.delimit("(", ")"):
-                comma = False
-                for e in t.bases:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.dispatch(e)
-                for e in t.keywords:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.dispatch(e)
-                if sys.version_info[:2] < (3, 5):
-                    if t.starargs:
-                        if comma: self.write(", ")
-                        else: comma = True
-                        self.write("*")
-                        self.dispatch(t.starargs)
-                    if t.kwargs:
-                        if comma: self.write(", ")
-                        else: comma = True
-                        self.write("**")
-                        self.dispatch(t.kwargs)
-        elif t.bases:
-            with self.delimit("(", ")"):
-                for a in t.bases[:-1]:
-                    self.dispatch(a)
-                    self.write(", ")
-                self.dispatch(t.bases[-1])
-        with self.block():
-            self.dispatch(t.body)
-
-    def _FunctionDef(self, t):
-        self.__FunctionDef_helper(t, "def")
-
-    def _AsyncFunctionDef(self, t):
-        self.__FunctionDef_helper(t, "async def")
-
-    def __FunctionDef_helper(self, t, fill_suffix):
-        self.write("\n")
-        for deco in t.decorator_list:
-            self.fill("@")
-            self.dispatch(deco)
-        def_str = fill_suffix + " " + t.name
-        self.fill(def_str)
-        with self.delimit("(", ")"):
-            self.dispatch(t.args)
-        if getattr(t, "returns", False):
-            self.write(" -> ")
-            self.dispatch(t.returns)
-        with self.block():
-            self.dispatch(t.body)
-
-    def _For(self, t):
-        self.__For_helper("for ", t)
-
-    def _AsyncFor(self, t):
-        self.__For_helper("async for ", t)
-
-    def __For_helper(self, fill, t):
-        self.fill(fill)
-        self.dispatch(t.target)
-        self.write(" in ")
-        self.dispatch(t.iter)
-        with self.block():
-            self.dispatch(t.body)
-        if t.orelse:
-            self.fill("else")
-            with self.block():
-                self.dispatch(t.orelse)
-
-    def _If(self, t):
-        self.fill("if ")
-        self.dispatch(t.test)
-        with self.block():
-            self.dispatch(t.body)
-        # collapse nested ifs into equivalent elifs.
-        while (t.orelse and len(t.orelse) == 1 and
-               isinstance(t.orelse[0], ast.If)):
-            t = t.orelse[0]
-            self.fill("elif ")
-            self.dispatch(t.test)
-            with self.block():
-                self.dispatch(t.body)
-        # final else
-        if t.orelse:
-            self.fill("else")
-            with self.block():
-                self.dispatch(t.orelse)
-
-    def _While(self, t):
-        self.fill("while ")
-        self.dispatch(t.test)
-        with self.block():
-            self.dispatch(t.body)
-        if t.orelse:
-            self.fill("else")
-            with self.block():
-                self.dispatch(t.orelse)
-
-    def _generic_With(self, t, async_=False):
-        self.fill("async with " if async_ else "with ")
-        if hasattr(t, 'items'):
-            interleave(lambda: self.write(", "), self.dispatch, t.items)
-        else:
-            self.dispatch(t.context_expr)
-            if t.optional_vars:
-                self.write(" as ")
-                self.dispatch(t.optional_vars)
-        with self.block():
-            self.dispatch(t.body)
-
-    def _With(self, t):
-        self._generic_With(t)
-
-    def _AsyncWith(self, t):
-        self._generic_With(t, async_=True)
-
-    # expr
-    def _Bytes(self, t):
-        self.write(repr(t.s))
-
-    def _Str(self, tree):
-        if six.PY3:
-            self.write(repr(tree.s))
-        else:
-            # if from __future__ import unicode_literals is in effect,
-            # then we want to output string literals using a 'b' prefix
-            # and unicode literals with no prefix.
-            if "unicode_literals" not in self.future_imports:
-                self.write(repr(tree.s))
-            elif isinstance(tree.s, str):
-                self.write("b" + repr(tree.s))
-            elif isinstance(tree.s, unicode):
-                self.write(repr(tree.s).lstrip("u"))
-            else:
-                assert False, "shouldn't get here"
-
-    def _JoinedStr(self, t):
-        # JoinedStr(expr* values)
-        self.write("f")
-        string = StringIO()
-        self._fstring_JoinedStr(t, string.write)
-        # Deviation from `unparse.py`: Try to find an unused quote.
-        # This change is made to handle _very_ complex f-strings.
-        v = string.getvalue()
-        if '\n' in v or '\r' in v:
-            quote_types = ["'''", '"""']
-        else:
-            quote_types = ["'", '"', '"""', "'''"]
-        for quote_type in quote_types:
-            if quote_type not in v:
-                v = "{quote_type}{v}{quote_type}".format(quote_type=quote_type, v=v)
-                break
-        else:
-            v = repr(v)
-        self.write(v)
-
-    def _FormattedValue(self, t):
-        # FormattedValue(expr value, int? conversion, expr? format_spec)
-        self.write("f")
-        string = StringIO()
-        self._fstring_JoinedStr(t, string.write)
-        self.write(repr(string.getvalue()))
-
-    def _fstring_JoinedStr(self, t, write):
-        for value in t.values:
-            meth = getattr(self, "_fstring_" + type(value).__name__)
-            meth(value, write)
-
-    def _fstring_Str(self, t, write):
-        value = t.s.replace("{", "{{").replace("}", "}}")
-        write(value)
-
-    def _fstring_Constant(self, t, write):
-        assert isinstance(t.value, str)
-        value = t.value.replace("{", "{{").replace("}", "}}")
-        write(value)
-
-    def _fstring_FormattedValue(self, t, write):
-        write("{")
-        expr = StringIO()
-        Unparser(t.value, expr)
-        expr = expr.getvalue().rstrip("\n")
-        if expr.startswith("{"):
-            write(" ")  # Separate pair of opening brackets as "{ {"
-        write(expr)
-        if t.conversion != -1:
-            conversion = chr(t.conversion)
-            assert conversion in "sra"
-            write("!{conversion}".format(conversion=conversion))
-        if t.format_spec:
-            write(":")
-            meth = getattr(self, "_fstring_" + type(t.format_spec).__name__)
-            meth(t.format_spec, write)
-        write("}")
-
-    def _Name(self, t):
-        self.write(t.id)
-
-    def _NameConstant(self, t):
-        self.write(repr(t.value))
-
-    def _Repr(self, t):
-        self.write("`")
-        self.dispatch(t.value)
-        self.write("`")
-
-    def _write_constant(self, value):
-        if isinstance(value, (float, complex)):
-            # Substitute overflowing decimal literal for AST infinities.
-            self.write(repr(value).replace("inf", INFSTR))
-        else:
-            self.write(repr(value))
-
-    def _Constant(self, t):
-        value = t.value
-        if isinstance(value, tuple):
-            with self.delimit("(", ")"):
-                if len(value) == 1:
-                    self._write_constant(value[0])
-                    self.write(",")
-                else:
-                    interleave(lambda: self.write(", "), self._write_constant, value)
-        elif value is Ellipsis: # instead of `...` for Py2 compatibility
-            self.write("...")
-        else:
-            if t.kind == "u":
-                self.write("u")
-            self._write_constant(t.value)
-
-    def _Num(self, t):
-        repr_n = repr(t.n)
-        if six.PY3:
-            self.write(repr_n.replace("inf", INFSTR))
-        else:
-            # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
-            with self.delimit_if("(", ")", repr_n.startswith("-")):
-                if "inf" in repr_n and repr_n.endswith("*j"):
-                    repr_n = repr_n.replace("*j", "j")
-                # Substitute overflowing decimal literal for AST infinities.
-                self.write(repr_n.replace("inf", INFSTR))
-
-    def _List(self, t):
-        with self.delimit("[", "]"):
-            interleave(lambda: self.write(", "), self.dispatch, t.elts)
-
-    def _ListComp(self, t):
-        with self.delimit("[", "]"):
-            self.dispatch(t.elt)
-            for gen in t.generators:
-                self.dispatch(gen)
-
-    def _GeneratorExp(self, t):
-        with self.delimit("(", ")"):
-            self.dispatch(t.elt)
-            for gen in t.generators:
-                self.dispatch(gen)
-
-    def _SetComp(self, t):
-        with self.delimit("{", "}"):
-            self.dispatch(t.elt)
-            for gen in t.generators:
-                self.dispatch(gen)
-
-    def _DictComp(self, t):
-        with self.delimit("{", "}"):
-            self.dispatch(t.key)
-            self.write(": ")
-            self.dispatch(t.value)
-            for gen in t.generators:
-                self.dispatch(gen)
-
-    def _comprehension(self, t):
-        if getattr(t, 'is_async', False):
-            self.write(" async for ")
-        else:
-            self.write(" for ")
-        self.dispatch(t.target)
-        self.write(" in ")
-        self.dispatch(t.iter)
-        for if_clause in t.ifs:
-            self.write(" if ")
-            self.dispatch(if_clause)
-
-    def _IfExp(self, t):
-        with self.delimit("(", ")"):
-            self.dispatch(t.body)
-            self.write(" if ")
-            self.dispatch(t.test)
-            self.write(" else ")
-            self.dispatch(t.orelse)
-
-    def _Set(self, t):
-        assert(t.elts) # should be at least one element
-        with self.delimit("{", "}"):
-            interleave(lambda: self.write(", "), self.dispatch, t.elts)
-
-    def _Dict(self, t):
-        def write_key_value_pair(k, v):
-            self.dispatch(k)
-            self.write(": ")
-            self.dispatch(v)
-
-        def write_item(item):
-            k, v = item
-            if k is None:
-                # for dictionary unpacking operator in dicts {**{'y': 2}}
-                # see PEP 448 for details
-                self.write("**")
-                self.dispatch(v)
-            else:
-                write_key_value_pair(k, v)
-
-        with self.delimit("{", "}"):
-            interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values))
-
-    def _Tuple(self, t):
-        with self.delimit("(", ")"):
-            if len(t.elts) == 1:
-                elt = t.elts[0]
-                self.dispatch(elt)
-                self.write(",")
-            else:
-                interleave(lambda: self.write(", "), self.dispatch, t.elts)
-
-    unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"}
-    def _UnaryOp(self, t):
-        with self.delimit("(", ")"):
-            self.write(self.unop[t.op.__class__.__name__])
-            if not self._py_ver_consistent:
-                self.write(" ")
-            if six.PY2 and isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
-                # If we're applying unary minus to a number, parenthesize the number.
-                # This is necessary: -2147483648 is different from -(2147483648) on
-                # a 32-bit machine (the first is an int, the second a long), and
-                # -7j is different from -(7j).  (The first has real part 0.0, the second
-                # has real part -0.0.)
-                with self.delimit("(", ")"):
-                    self.dispatch(t.operand)
-            else:
-                self.dispatch(t.operand)
-
-    binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%",
-                    "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
-                    "FloorDiv":"//", "Pow": "**"}
-    def _BinOp(self, t):
-        with self.delimit("(", ")"):
-            self.dispatch(t.left)
-            self.write(" " + self.binop[t.op.__class__.__name__] + " ")
-            self.dispatch(t.right)
-
-    cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=",
-                        "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"}
-    def _Compare(self, t):
-        with self.delimit("(", ")"):
-            self.dispatch(t.left)
-            for o, e in zip(t.ops, t.comparators):
-                self.write(" " + self.cmpops[o.__class__.__name__] + " ")
-                self.dispatch(e)
-
-    boolops = {ast.And: 'and', ast.Or: 'or'}
-    def _BoolOp(self, t):
-        with self.delimit("(", ")"):
-            s = " %s " % self.boolops[t.op.__class__]
-            interleave(lambda: self.write(s), self.dispatch, t.values)
-
-    def _Attribute(self,t):
-        self.dispatch(t.value)
-        # Special case: 3.__abs__() is a syntax error, so if t.value
-        # is an integer literal then we need to either parenthesize
-        # it or add an extra space to get 3 .__abs__().
-        if isinstance(t.value, getattr(ast, 'Constant', getattr(ast, 'Num', None))) and isinstance(t.value.n, int):
-            self.write(" ")
-        self.write(".")
-        self.write(t.attr)
-
-    def _Call(self, t):
-        self.dispatch(t.func)
-        with self.delimit("(", ")"):
-            comma = False
-
-            # starred arguments last in Python 3.5+, for consistency w/earlier versions
-            star_and_kwargs = []
-            move_stars_last = sys.version_info[:2] >= (3, 5)
-
-            for e in t.args:
-                if move_stars_last and isinstance(e, ast.Starred):
-                    star_and_kwargs.append(e)
-                else:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.dispatch(e)
-
-            for e in t.keywords:
-                # starting from Python 3.5 this denotes a kwargs part of the invocation
-                if e.arg is None and move_stars_last:
-                    star_and_kwargs.append(e)
-                else:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.dispatch(e)
-
-            if move_stars_last:
-                for e in star_and_kwargs:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.dispatch(e)
-
-            if sys.version_info[:2] < (3, 5):
-                if t.starargs:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.write("*")
-                    self.dispatch(t.starargs)
-                if t.kwargs:
-                    if comma: self.write(", ")
-                    else: comma = True
-                    self.write("**")
-                    self.dispatch(t.kwargs)
-
-    def _Subscript(self, t):
-        self.dispatch(t.value)
-        with self.delimit("[", "]"):
-            self.dispatch(t.slice)
-
-    def _Starred(self, t):
-        self.write("*")
-        self.dispatch(t.value)
-
-    # slice
-    def _Ellipsis(self, t):
-        self.write("...")
-
-    def _Index(self, t):
-        self.dispatch(t.value)
-
-    def _Slice(self, t):
-        if t.lower:
-            self.dispatch(t.lower)
-        self.write(":")
-        if t.upper:
-            self.dispatch(t.upper)
-        if t.step:
-            self.write(":")
-            self.dispatch(t.step)
-
-    def _ExtSlice(self, t):
-        interleave(lambda: self.write(', '), self.dispatch, t.dims)
-
-    # argument
-    def _arg(self, t):
-        self.write(t.arg)
-        if t.annotation:
-            self.write(": ")
-            self.dispatch(t.annotation)
-
-    # others
-    def _arguments(self, t):
-        first = True
-        # normal arguments
-        all_args = getattr(t, 'posonlyargs', []) + t.args
-        defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults
-        for index, elements in enumerate(zip(all_args, defaults), 1):
-            a, d = elements
-            if first:first = False
-            else: self.write(", ")
-            self.dispatch(a)
-            if d:
-                self.write("=")
-                self.dispatch(d)
-            if index == len(getattr(t, 'posonlyargs', ())):
-                self.write(", /")
-
-        # varargs, or bare '*' if no varargs but keyword-only arguments present
-        if t.vararg or getattr(t, "kwonlyargs", False):
-            if first:first = False
-            else: self.write(", ")
-            self.write("*")
-            if t.vararg:
-                if hasattr(t.vararg, 'arg'):
-                    self.write(t.vararg.arg)
-                    if t.vararg.annotation:
-                        self.write(": ")
-                        self.dispatch(t.vararg.annotation)
-                else:
-                    self.write(t.vararg)
-                    if getattr(t, 'varargannotation', None):
-                        self.write(": ")
-                        self.dispatch(t.varargannotation)
-
-        # keyword-only arguments
-        if getattr(t, "kwonlyargs", False):
-            for a, d in zip(t.kwonlyargs, t.kw_defaults):
-                if first:first = False
-                else: self.write(", ")
-                self.dispatch(a),
-                if d:
-                    self.write("=")
-                    self.dispatch(d)
-
-        # kwargs
-        if t.kwarg:
-            if first:first = False
-            else: self.write(", ")
-            if hasattr(t.kwarg, 'arg'):
-                self.write("**"+t.kwarg.arg)
-                if t.kwarg.annotation:
-                    self.write(": ")
-                    self.dispatch(t.kwarg.annotation)
-            else:
-                self.write("**"+t.kwarg)
-                if getattr(t, 'kwargannotation', None):
-                    self.write(": ")
-                    self.dispatch(t.kwargannotation)
-
-    def _keyword(self, t):
-        if t.arg is None:
-            # starting from Python 3.5 this denotes a kwargs part of the invocation
-            self.write("**")
-        else:
-            self.write(t.arg)
-            self.write("=")
-        self.dispatch(t.value)
-
-    def _Lambda(self, t):
-        with self.delimit("(", ")"):
-            self.write("lambda ")
-            self.dispatch(t.args)
-            self.write(": ")
-            self.dispatch(t.body)
-
-    def _alias(self, t):
-        self.write(t.name)
-        if t.asname:
-            self.write(" as "+t.asname)
-
-    def _withitem(self, t):
-        self.dispatch(t.context_expr)
-        if t.optional_vars:
-            self.write(" as ")
-            self.dispatch(t.optional_vars)
-
-def roundtrip(filename, output=sys.stdout):
-    if six.PY3:
-        with open(filename, "rb") as pyfile:
-            encoding = tokenize.detect_encoding(pyfile.readline)[0]
-        with open(filename, "r", encoding=encoding) as pyfile:
-            source = pyfile.read()
-    else:
-        with open(filename, "r") as pyfile:
-            source = pyfile.read()
-    tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST, dont_inherit=True)
-    Unparser(tree, output)
-
-
-
-def testdir(a):
-    try:
-        names = [n for n in os.listdir(a) if n.endswith('.py')]
-    except OSError:
-        print("Directory not readable: %s" % a, file=sys.stderr)
-    else:
-        for n in names:
-            fullname = os.path.join(a, n)
-            if os.path.isfile(fullname):
-                output = StringIO()
-                print('Testing %s' % fullname)
-                try:
-                    roundtrip(fullname, output)
-                except Exception as e:
-                    print('  Failed to compile, exception is %s' % repr(e))
-            elif os.path.isdir(fullname):
-                testdir(fullname)
-
-def main(args):
-    if args[0] == '--testdir':
-        for a in args[1:]:
-            testdir(a)
-    else:
-        for a in args:
-            roundtrip(a)
-
-if __name__=='__main__':
-    main(sys.argv[1:])