From 101f08013802265e9d893a482f0bd4c821957762 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 3 Jan 2022 16:46:34 -0800 Subject: unparser: add unparser unit tests These are the unit tests from astunparse, converted to pytest, with a few backports from upstream cpython. These should hopefully keep `unparser.py` well covered as we change it. --- lib/spack/spack/cmd/license.py | 2 +- lib/spack/spack/test/util/unparse/__init__.py | 3 + lib/spack/spack/test/util/unparse/unparse.py | 524 ++++++++++++++++++++++++++ 3 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/test/util/unparse/__init__.py create mode 100644 lib/spack/spack/test/util/unparse/unparse.py diff --git a/lib/spack/spack/cmd/license.py b/lib/spack/spack/cmd/license.py index dd85f2ed25..a554a7180b 100644 --- a/lib/spack/spack/cmd/license.py +++ b/lib/spack/spack/cmd/license.py @@ -35,7 +35,7 @@ licensed_files = [ r'^bin/spack-python$', # all of spack core except unparse - r'^lib/spack/spack/(?!util/unparse).*\.py$', + r'^lib/spack/spack/(?!(test/)?util/unparse).*\.py$', r'^lib/spack/spack/.*\.sh$', r'^lib/spack/spack/.*\.lp$', r'^lib/spack/llnl/.*\.py$', diff --git a/lib/spack/spack/test/util/unparse/__init__.py b/lib/spack/spack/test/util/unparse/__init__.py new file mode 100644 index 0000000000..e0aad674bc --- /dev/null +++ b/lib/spack/spack/test/util/unparse/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers. +# +# SPDX-License-Identifier: Python-2.0 diff --git a/lib/spack/spack/test/util/unparse/unparse.py b/lib/spack/spack/test/util/unparse/unparse.py new file mode 100644 index 0000000000..85ab5c6b1b --- /dev/null +++ b/lib/spack/spack/test/util/unparse/unparse.py @@ -0,0 +1,524 @@ +# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers. +# +# SPDX-License-Identifier: Python-2.0 + +import ast +import codecs +import os +import sys + +import pytest +import six + +if six.PY3: + import tokenize +else: + from lib2to3.pgen2 import tokenize + +import spack.util.unparse + + +def read_pyfile(filename): + """Read and return the contents of a Python source file (as a + string), taking into account the file encoding.""" + if six.PY3: + with open(filename, "rb") as pyfile: + encoding = tokenize.detect_encoding(pyfile.readline)[0] + with codecs.open(filename, "r", encoding=encoding) as pyfile: + source = pyfile.read() + else: + with open(filename, "r") as pyfile: + source = pyfile.read() + return source + + +code_parseable_in_all_parser_modes = """\ +(a + b + c) * (d + e + f) +""" + +for_else = """\ +def f(): + for x in range(10): + break + else: + y = 2 + z = 3 +""" + +while_else = """\ +def g(): + while True: + break + else: + y = 2 + z = 3 +""" + +relative_import = """\ +from . import fred +from .. import barney +from .australia import shrimp as prawns +""" + +import_many = """\ +import fred, barney +""" + +nonlocal_ex = """\ +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + y = 7 + def h(): + nonlocal x, y +""" + +# also acts as test for 'except ... as ...' +raise_from = """\ +try: + 1 / 0 +except ZeroDivisionError as e: + raise ArithmeticError from e +""" + +async_comprehensions_and_generators = """\ +async def async_function(): + my_set = {i async for i in aiter() if i % 2} + my_list = [i async for i in aiter() if i % 2] + my_dict = {i: -i async for i in aiter() if i % 2} + my_gen = (i ** 2 async for i in agen()) + my_other_gen = (i - 1 async for i in agen() if i % 2) +""" + +class_decorator = """\ +@f1(arg) +@f2 +class Foo: pass +""" + +elif1 = """\ +if cond1: + suite1 +elif cond2: + suite2 +else: + suite3 +""" + +elif2 = """\ +if cond1: + suite1 +elif cond2: + suite2 +""" + +try_except_finally = """\ +try: + suite1 +except ex1: + suite2 +except ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +with_simple = """\ +with f(): + suite1 +""" + +with_as = """\ +with f() as x: + suite1 +""" + +with_two_items = """\ +with f() as x, g() as y: + suite1 +""" + +a_repr = """\ +`{}` +""" + +complex_f_string = '''\ +f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\' +''' + +async_function_def = """\ +async def f(): + suite1 +""" + +async_for = """\ +async def f(): + async for _ in reader: + suite1 +""" + +async_with = """\ +async def f(): + async with g(): + suite1 +""" + +async_with_as = """\ +async def f(): + async with g() as x: + suite1 +""" + + +def assertASTEqual(ast1, ast2): + ast.dump(ast1) == ast.dump(ast2) + + +def check_ast_roundtrip(code1, filename="internal", mode="exec"): + ast1 = compile(str(code1), filename, mode, ast.PyCF_ONLY_AST) + code2 = spack.util.unparse.unparse(ast1) + + ast2 = compile(code2, filename, mode, ast.PyCF_ONLY_AST) + assertASTEqual(ast1, ast2) + + +def test_core_lib_files(): + """Roundtrip source files from the Python core libs.""" + test_directories = [ + os.path.join( + getattr(sys, 'real_prefix', sys.prefix), + 'lib', + 'python%s.%s' % sys.version_info[:2] + ) + ] + + names = [] + for test_dir in test_directories: + for n in os.listdir(test_dir): + if n.endswith('.py') and not n.startswith('bad'): + names.append(os.path.join(test_dir, n)) + + for filename in names: + print('Testing %s' % filename) + source = read_pyfile(filename) + check_ast_roundtrip(source) + + +@pytest.mark.skipif( + sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater" +) +def test_simple_fstring(): + check_ast_roundtrip("f'{x}'") + + +@pytest.mark.skipif( + sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater" +) +def test_fstrings(): + # See issue 25180 + check_ast_roundtrip(r"""f'{f"{0}"*3}'""") + check_ast_roundtrip(r"""f'{f"{y}"*3}'""") + check_ast_roundtrip("""f''""") + check_ast_roundtrip('''f"""'end' "quote\\""""''') + + +@pytest.mark.skipif( + sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater" +) +def test_fstrings_complicated(): + # See issue 28002 + check_ast_roundtrip("""f'''{"'"}'''""") + check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + check_ast_roundtrip( + '''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''') + check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""') + check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""') + check_ast_roundtrip('''f"a\\r\\nb"''') + check_ast_roundtrip('''f"\\u2028{'x'}"''') + + +def test_parser_modes(): + for mode in ['exec', 'single', 'eval']: + check_ast_roundtrip(code_parseable_in_all_parser_modes, mode=mode) + + +def test_del_statement(): + check_ast_roundtrip("del x, y, z") + + +def test_shifts(): + check_ast_roundtrip("45 << 2") + check_ast_roundtrip("13 >> 7") + + +def test_for_else(): + check_ast_roundtrip(for_else) + + +def test_while_else(): + check_ast_roundtrip(while_else) + + +def test_unary_parens(): + check_ast_roundtrip("(-1)**7") + check_ast_roundtrip("(-1.)**8") + check_ast_roundtrip("(-1j)**6") + check_ast_roundtrip("not True or False") + check_ast_roundtrip("True or not False") + + +@pytest.mark.skipif(sys.version_info >= (3, 6), reason="Only works for Python < 3.6") +def test_integer_parens(): + check_ast_roundtrip("3 .__abs__()") + + +def test_huge_float(): + check_ast_roundtrip("1e1000") + check_ast_roundtrip("-1e1000") + check_ast_roundtrip("1e1000j") + check_ast_roundtrip("-1e1000j") + + +@pytest.mark.skipif(not six.PY2, reason="Only works for Python 2") +def test_min_int27(): + check_ast_roundtrip(str(-sys.maxint - 1)) + check_ast_roundtrip("-(%s)" % (sys.maxint + 1)) + + +@pytest.mark.skipif(not six.PY3, reason="Only works for Python 3") +def test_min_int30(): + check_ast_roundtrip(str(-2**31)) + check_ast_roundtrip(str(-2**63)) + + +def test_imaginary_literals(): + check_ast_roundtrip("7j") + check_ast_roundtrip("-7j") + check_ast_roundtrip("0j") + check_ast_roundtrip("-0j") + if six.PY2: + check_ast_roundtrip("-(7j)") + check_ast_roundtrip("-(0j)") + + +def test_negative_zero(): + check_ast_roundtrip("-0") + check_ast_roundtrip("-(0)") + check_ast_roundtrip("-0b0") + check_ast_roundtrip("-(0b0)") + check_ast_roundtrip("-0o0") + check_ast_roundtrip("-(0o0)") + check_ast_roundtrip("-0x0") + check_ast_roundtrip("-(0x0)") + + +def test_lambda_parentheses(): + check_ast_roundtrip("(lambda: int)()") + + +def test_chained_comparisons(): + check_ast_roundtrip("1 < 4 <= 5") + check_ast_roundtrip("a is b is c is not d") + + +def test_function_arguments(): + check_ast_roundtrip("def f(): pass") + check_ast_roundtrip("def f(a): pass") + check_ast_roundtrip("def f(b = 2): pass") + check_ast_roundtrip("def f(a, b): pass") + check_ast_roundtrip("def f(a, b = 2): pass") + check_ast_roundtrip("def f(a = 5, b = 2): pass") + check_ast_roundtrip("def f(*args, **kwargs): pass") + if six.PY3: + check_ast_roundtrip("def f(*, a = 1, b = 2): pass") + check_ast_roundtrip("def f(*, a = 1, b): pass") + check_ast_roundtrip("def f(*, a, b = 2): pass") + check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass") + check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") + + +def test_relative_import(): + check_ast_roundtrip(relative_import) + + +def test_import_many(): + check_ast_roundtrip(import_many) + + +@pytest.mark.skipif(not six.PY3, reason="Only for Python 3") +def test_nonlocal(): + check_ast_roundtrip(nonlocal_ex) + + +@pytest.mark.skipif(not six.PY3, reason="Only for Python 3") +def test_raise_from(): + check_ast_roundtrip(raise_from) + + +def test_bytes(): + check_ast_roundtrip("b'123'") + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") +def test_formatted_value(): + check_ast_roundtrip('f"{value}"') + check_ast_roundtrip('f"{value!s}"') + check_ast_roundtrip('f"{value:4}"') + check_ast_roundtrip('f"{value!s:4}"') + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") +def test_joined_str(): + check_ast_roundtrip('f"{key}={value!s}"') + check_ast_roundtrip('f"{key}={value!r}"') + check_ast_roundtrip('f"{key}={value!a}"') + + +@pytest.mark.skipif(sys.version_info != (3, 6, 0), reason="Only supported on 3.6.0") +def test_joined_str_361(): + check_ast_roundtrip('f"{key:4}={value!s}"') + check_ast_roundtrip('f"{key:02}={value!r}"') + check_ast_roundtrip('f"{key:6}={value!a}"') + check_ast_roundtrip('f"{key:4}={value:#06x}"') + check_ast_roundtrip('f"{key:02}={value:#06x}"') + check_ast_roundtrip('f"{key:6}={value:#06x}"') + check_ast_roundtrip('f"{key:4}={value!s:#06x}"') + check_ast_roundtrip('f"{key:4}={value!r:#06x}"') + check_ast_roundtrip('f"{key:4}={value!a:#06x}"') + + +@pytest.mark.skipif(not six.PY2, reason="Only for Python 2") +def test_repr(): + check_ast_roundtrip(a_repr) + + +@pytest.mark.skipif( + sys.version_info[:2] < (3, 6), + reason="Only for Python 3.6 or greater" +) +def test_complex_f_string(): + check_ast_roundtrip(complex_f_string) + + +@pytest.mark.skipif(not six.PY3, reason="Only for Python 3") +def test_annotations(): + check_ast_roundtrip("def f(a : int): pass") + check_ast_roundtrip("def f(a: int = 5): pass") + check_ast_roundtrip("def f(*args: [int]): pass") + check_ast_roundtrip("def f(**kwargs: dict): pass") + check_ast_roundtrip("def f() -> None: pass") + + +@pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") +def test_set_literal(): + check_ast_roundtrip("{'a', 'b', 'c'}") + + +@pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") +def test_set_comprehension(): + check_ast_roundtrip("{x for x in range(5)}") + + +@pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") +def test_dict_comprehension(): + check_ast_roundtrip("{x: x*x for x in range(10)}") + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") +def test_dict_with_unpacking(): + check_ast_roundtrip("{**x}") + check_ast_roundtrip("{a: b, **x}") + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") +def test_async_comp_and_gen_in_async_function(): + check_ast_roundtrip(async_comprehensions_and_generators) + + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="Not supported < 3.7") +def test_async_comprehension(): + check_ast_roundtrip("{i async for i in aiter() if i % 2}") + check_ast_roundtrip("[i async for i in aiter() if i % 2]") + check_ast_roundtrip("{i: -i async for i in aiter() if i % 2}") + + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="Not supported < 3.7") +def test_async_generator_expression(): + check_ast_roundtrip("(i ** 2 async for i in agen())") + check_ast_roundtrip("(i - 1 async for i in agen() if i % 2)") + + +def test_class_decorators(): + check_ast_roundtrip(class_decorator) + + +@pytest.mark.skipif(not six.PY3, reason="Only for Python 3") +def test_class_definition(): + check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass") + + +def test_elifs(): + check_ast_roundtrip(elif1) + check_ast_roundtrip(elif2) + + +def test_try_except_finally(): + check_ast_roundtrip(try_except_finally) + + +@pytest.mark.skipif(not six.PY3, reason="Only for Python 3") +def test_starred_assignment(): + check_ast_roundtrip("a, *b, c = seq") + check_ast_roundtrip("a, (*b, c) = seq") + check_ast_roundtrip("a, *b[0], c = seq") + check_ast_roundtrip("a, *(b, c) = seq") + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") +def test_variable_annotation(): + check_ast_roundtrip("a: int") + check_ast_roundtrip("a: int = 0") + check_ast_roundtrip("a: int = None") + check_ast_roundtrip("some_list: List[int]") + check_ast_roundtrip("some_list: List[int] = []") + check_ast_roundtrip("t: Tuple[int, ...] = (1, 2, 3)") + check_ast_roundtrip("(a): int") + check_ast_roundtrip("(a): int = 0") + check_ast_roundtrip("(a): int = None") + + +def test_with_simple(): + check_ast_roundtrip(with_simple) + + +def test_with_as(): + check_ast_roundtrip(with_as) + + +@pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") +def test_with_two_items(): + check_ast_roundtrip(with_two_items) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") +def test_async_function_def(): + check_ast_roundtrip(async_function_def) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") +def test_async_for(): + check_ast_roundtrip(async_for) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") +def test_async_with(): + check_ast_roundtrip(async_with) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") +def test_async_with_as(): + check_ast_roundtrip(async_with_as) -- cgit v1.2.3-70-g09d2