"""Tests for the unparse.py script in the Tools/parser directory."""
import unittest
import test.support
import pathlib
import random
import tokenize
import ast
def read_pyfile(filename):
"""Read and return the contents of a Python source file (as a
string), taking into account the file encoding."""
with tokenize.open(filename) as stream:
return stream.read()
for_else = """\
def f():
for x in range(10):
break
else:
y = 2
z = 3
"""
while_else = """\
def g():
while True:
break
else:
y = 2
z = 3
"""
relative_import = """\
from . import fred
from .. import barney
from .australia import shrimp as prawns
"""
nonlocal_ex = """\
def f():
x = 1
def g():
nonlocal x
x = 2
y = 7
def h():
nonlocal x, y
"""
# also acts as test for 'except ... as ...'
raise_from = """\
try:
1 / 0
except ZeroDivisionError as e:
raise ArithmeticError from e
"""
class_decorator = """\
@f1(arg)
@f2
class Foo: pass
"""
elif1 = """\
if cond1:
suite1
elif cond2:
suite2
else:
suite3
"""
elif2 = """\
if cond1:
suite1
elif cond2:
suite2
"""
try_except_finally = """\
try:
suite1
except ex1:
suite2
except ex2:
suite3
else:
suite4
finally:
suite5
"""
with_simple = """\
with f():
suite1
"""
with_as = """\
with f() as x:
suite1
"""
with_two_items = """\
with f() as x, g() as y:
suite1
"""
docstring_prefixes = (
"",
"class foo:\n ",
"def foo():\n ",
"async def foo():\n ",
)
class ASTTestCase(unittest.TestCase):
def assertASTEqual(self, ast1, ast2):
self.assertEqual(ast.dump(ast1), ast.dump(ast2))
def check_ast_roundtrip(self, code1, **kwargs):
with self.subTest(code1=code1, ast_parse_kwargs=kwargs):
ast1 = ast.parse(code1, **kwargs)
code2 = ast.unparse(ast1)
ast2 = ast.parse(code2, **kwargs)
self.assertASTEqual(ast1, ast2)
def check_invalid(self, node, raises=ValueError):
with self.subTest(node=node):
self.assertRaises(raises, ast.unparse, node)
def get_source(self, code1, code2=None):
code2 = code2 or code1
code1 = ast.unparse(ast.parse(code1))
return code1, code2
def check_src_roundtrip(self, code1, code2=None):
code1, code2 = self.get_source(code1, code2)
with self.subTest(code1=code1, code2=code2):
self.assertEqual(code2, code1)
def check_src_dont_roundtrip(self, code1, code2=None):
code1, code2 = self.get_source(code1, code2)
with self.subTest(code1=code1, code2=code2):
self.assertNotEqual(code2, code1)
class UnparseTestCase(ASTTestCase):
# Tests for specific bugs found in earlier versions of unparse
def test_fstrings(self):
# See issue 25180
self.check_ast_roundtrip(r"""f'{f"{0}"*3}'""")
self.check_ast_roundtrip(r"""f'{f"{y}"*3}'""")
self.check_ast_roundtrip("""f''""")
self.check_ast_roundtrip('''f"""'end' "quote\\""""''')
def test_fstrings_complicated(self):
# See issue 28002
self.check_ast_roundtrip("""f'''{"'"}'''""")
self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''')
self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''')
self.check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""')
self.check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""')
self.check_ast_roundtrip('''f"a\\r\\nb"''')
self.check_ast_roundtrip('''f"\\u2028{'x'}"''')
def test_strings(self):
self.check_ast_roundtrip("u'foo'")
self.check_ast_roundtrip("r'foo'")
self.check_ast_roundtrip("b'foo'")
def test_del_statement(self):
self.check_ast_roundtrip("del x, y, z")
def test_shifts(self):
self.check_ast_roundtrip("45 << 2")
self.check_ast_roundtrip("13 >> 7")
def test_for_else(self):
self.check_ast_roundtrip(for_else)
def test_while_else(self):
self.check_ast_roundtrip(while_else)
def test_unary_parens(self):
self.check_ast_roundtrip("(-1)**7")
self.check_ast_roundtrip("(-1.)**8")
self.check_ast_roundtrip("(-1j)**6")
self.check_ast_roundtrip("not True or False")
self.check_ast_roundtrip("True or not False")
def test_integer_parens(self):
self.check_ast_roundtrip("3 .__abs__()")
def test_huge_float(self):
self.check_ast_roundtrip("1e1000")
self.check_ast_roundtrip("-1e1000")
self.check_ast_roundtrip("1e1000j")
self.check_ast_roundtrip("-1e1000j")
def test_nan(self):
self.assertASTEqual(
ast.parse(ast.unparse(ast.Constant(value=float('nan')))),
ast.parse('1e1000 - 1e1000')
)
def test_min_int(self):
self.check_ast_roundtrip(str(-(2 ** 31)))
self.check_ast_roundtrip(str(-(2 ** 63)))
def test_imaginary_literals(self):
self.check_ast_roundtrip("7j")
self.check_ast_roundtrip("-7j")
self.check_ast_roundtrip("0j")
self.check_ast_roundtrip("-0j")
def test_lambda_parentheses(self):
self.check_ast_roundtrip("(lambda: int)()")
def test_chained_comparisons(self):
self.check_ast_roundtrip("1 < 4 <= 5")
self.check_ast_roundtrip("a is b is c is not d")
def test_function_arguments(self):
self.check_ast_roundtrip("def f(): pass")
self.check_ast_roundtrip("def f(a): pass")
self.check_ast_roundtrip("def f(b = 2): pass")
self.check_ast_roundtrip("def f(a, b): pass")
self.check_ast_roundtrip("def f(a, b = 2): pass")
self.check_ast_roundtrip("def f(a = 5, b = 2): pass")
self.check_ast_roundtrip("def f(*, a = 1, b = 2): pass")
self.check_ast_roundtrip("def f(*, a = 1, b): pass")
self.check_ast_roundtrip("def f(*, a, b = 2): pass")
self.check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass")
self.check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass")
self.check_ast_roundtrip("def f(*args, **kwargs): pass")
def test_relative_import(self):
self.check_ast_roundtrip(relative_import)
def test_nonlocal(self):
self.check_ast_roundtrip(nonlocal_ex)
def test_raise_from(self):
self.check_ast_roundtrip(raise_from)
def test_bytes(self):
self.check_ast_roundtrip("b'123'")
def test_annotations(self):
self.check_ast_roundtrip("def f(a : int): pass")
self.check_ast_roundtrip("def f(a: int = 5): pass")
self.check_ast_roundtrip("def f(*args: [int]): pass")
self.check_ast_roundtrip("def f(**kwargs: dict): pass")
self.check_ast_roundtrip("def f() -> None: pass")
def test_set_literal(self):
self.check_ast_roundtrip("{'a', 'b', 'c'}")
def test_empty_set(self):
self.assertASTEqual(
ast.parse(ast.unparse(ast.Set(elts=[]))),
ast.parse('{*()}')
)
def test_set_comprehension(self):
self.check_ast_roundtrip("{x for x in range(5)}")
def test_dict_comprehension(self):
self.check_ast_roundtrip("{x: x*x for x in range(10)}")
def test_class_decorators(self):
self.check_ast_roundtrip(class_decorator)
def test_class_definition(self):
self.check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass")
def test_elifs(self):
self.check_ast_roundtrip(elif1)
self.check_ast_roundtrip(elif2)
def test_try_except_finally(self):
self.check_ast_roundtrip(try_except_finally)
def test_starred_assignment(self):
self.check_ast_roundtrip("a, *b, c = seq")
self.check_ast_roundtrip("a, (*b, c) = seq")
self.check_ast_roundtrip("a, *b[0], c = seq")
self.check_ast_roundtrip("a, *(b, c) = seq")
def test_with_simple(self):
self.check_ast_roundtrip(with_simple)
def test_with_as(self):
self.check_ast_roundtrip(with_as)
def test_with_two_items(self):
self.check_ast_roundtrip(with_two_items)
def test_dict_unpacking_in_dict(self):
# See issue 26489
self.check_ast_roundtrip(r"""{**{'y': 2}, 'x': 1}""")
self.check_ast_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""")
def test_slices(self):
self.check_ast_roundtrip("a[i]")
self.check_ast_roundtrip("a[i,]")
self.check_ast_roundtrip("a[i, j]")
self.check_ast_roundtrip("a[(*a,)]")
self.check_ast_roundtrip("a[(a:=b)]")
self.check_ast_roundtrip("a[(a:=b,c)]")
self.check_ast_roundtrip("a[()]")
self.check_ast_roundtrip("a[i:j]")
self.check_ast_roundtrip("a[:j]")
self.check_ast_roundtrip("a[i:]")
self.check_ast_roundtrip("a[i:j:k]")
self.check_ast_roundtrip("a[:j:k]")
self.check_ast_roundtrip("a[i::k]")
self.check_ast_roundtrip("a[i:j,]")
self.check_ast_roundtrip("a[i:j, k]")
def test_invalid_raise(self):
self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X")))
def test_invalid_fstring_constant(self):
self.check_invalid(ast.JoinedStr(values=[ast.Constant(value=100)]))
def test_invalid_fstring_conversion(self):
self.check_invalid(
ast.FormattedValue(
value=ast.Constant(value="a", kind=None),
conversion=ord("Y"), # random character
format_spec=None,
)
)
def test_invalid_fstring_backslash(self):
self.check_invalid(ast.FormattedValue(value=ast.Constant(value="\\\\")))
def test_invalid_yield_from(self):
self.check_invalid(ast.YieldFrom(value=None))
def test_docstrings(self):
docstrings = (
'this ends with double quote"',
'this includes a """triple quote"""',
'\r',
'\\r',
'\t',
'\\t',
'\n',
'\\n',
'\r\\r\t\\t\n\\n',
'""">>> content = \"\"\"blabla\"\"\" <<<"""',
r'foo\n\x00',
"' \\'\\'\\'\"\"\" \"\"\\'\\' \\'",
'🐍⛎𩸽üéş^\\\\X\\\\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}'
)
for docstring in docstrings:
# check as Module docstrings for easy testing
self.check_ast_roundtrip(f"'''{docstring}'''")
def test_constant_tuples(self):
self.check_src_roundtrip(ast.Constant(value=(1,), kind=None), "(1,)")
self.check_src_roundtrip(
ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)"
)
def test_function_type(self):
for function_type in (
"() -> int",
"(int, int) -> int",
"(Callable[complex], More[Complex(call.to_typevar())]) -> None"
):
self.check_ast_roundtrip(function_type, mode="func_type")
def test_type_comments(self):
for statement in (
"a = 5 # type:",
"a = 5 # type: int",
"a = 5 # type: int and more",
"def x(): # type: () -> None\n\tpass",
"def x(y): # type: (int) -> None and more\n\tpass",
"async def x(): # type: () -> None\n\tpass",
"async def x(y): # type: (int) -> None and more\n\tpass",
"for x in y: # type: int\n\tpass",
"async for x in y: # type: int\n\tpass",
"with x(): # type: int\n\tpass",
"async with x(): # type: int\n\tpass"
):
self.check_ast_roundtrip(statement, type_comments=True)
def test_type_ignore(self):
for statement in (
"a = 5 # type: ignore",
"a = 5 # type: ignore and more",
"def x(): # type: ignore\n\tpass",
"def x(y): # type: ignore and more\n\tpass",
"async def x(): # type: ignore\n\tpass",
"async def x(y): # type: ignore and more\n\tpass",
"for x in y: # type: ignore\n\tpass",
"async for x in y: # type: ignore\n\tpass",
"with x(): # type: ignore\n\tpass",
"async with x(): # type: ignore\n\tpass"
):
self.check_ast_roundtrip(statement, type_comments=True)
class CosmeticTestCase(ASTTestCase):
"""Test if there are cosmetic issues caused by unnecessary additions"""
def test_simple_expressions_parens(self):
self.check_src_roundtrip("(a := b)")
self.check_src_roundtrip("await x")
self.check_src_roundtrip("x if x else y")
self.check_src_roundtrip("lambda x: x")
self.check_src_roundtrip("1 + 1")
self.check_src_roundtrip("1 + 2 / 3")
self.check_src_roundtrip("(1 + 2) / 3")
self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2)")
self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2) ** 2")
self.check_src_roundtrip("~x")
self.check_src_roundtrip("x and y")
self.check_src_roundtrip("x and y and z")
self.check_src_roundtrip("x and (y and x)")
self.check_src_roundtrip("(x and y) and z")
self.check_src_roundtrip("(x ** y) ** z ** q")
self.check_src_roundtrip("x >> y")
self.check_src_roundtrip("x << y")
self.check_src_roundtrip("x >> y and x >> z")
self.check_src_roundtrip("x + y - z * q ^ t ** k")
self.check_src_roundtrip("P * V if P and V else n * R * T")
self.check_src_roundtrip("lambda P, V, n: P * V == n * R * T")
self.check_src_roundtrip("flag & (other | foo)")
self.check_src_roundtrip("not x == y")
self.check_src_roundtrip("x == (not y)")
self.check_src_roundtrip("yield x")
self.check_src_roundtrip("yield from x")
self.check_src_roundtrip("call((yield x))")
self.check_src_roundtrip("return x + (yield x)")
def test_class_bases_and_keywords(self):
self.check_src_roundtrip("class X:\n pass")
self.check_src_roundtrip("class X(A):\n pass")
self.check_src_roundtrip("class X(A, B, C, D):\n pass")
self.check_src_roundtrip("class X(x=y):\n pass")
self.check_src_roundtrip("class X(metaclass=z):\n pass")
self.check_src_roundtrip("class X(x=y, z=d):\n pass")
self.check_src_roundtrip("class X(A, x=y):\n pass")
self.check_src_roundtrip("class X(A, **kw):\n pass")
self.check_src_roundtrip("class X(*args):\n pass")
self.check_src_roundtrip("class X(*args, **kwargs):\n pass")
def test_fstrings(self):
self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''')
self.check_src_roundtrip('''f"\\u2028{'x'}"''')
self.check_src_roundtrip(r"f'{x}\n'")
self.check_src_roundtrip('''f''\'{"""\n"""}\\n''\'''')
self.check_src_roundtrip('''f''\'{f"""{x}\n"""}\\n''\'''')
def test_docstrings(self):
docstrings = (
'"""simple doc string"""',
'''"""A more complex one
with some newlines"""''',
'''"""Foo bar baz
empty newline"""''',
'"""With some \t"""',
'"""Foo "bar" baz """',
'"""\\r"""',
'""""""',
'"""\'\'\'"""',
'"""\'\'\'\'\'\'"""',
'"""🐍⛎𩸽üéş^\\\\X\\\\BB⟿"""',
'"""end in single \'quote\'"""',
"'''end in double \"quote\"'''",
'"""almost end in double "quote"."""',
)
for prefix in docstring_prefixes:
for docstring in docstrings:
self.check_src_roundtrip(f"{prefix}{docstring}")
def test_docstrings_negative_cases(self):
# Test some cases that involve strings in the children of the
# first node but aren't docstrings to make sure we don't have
# False positives.
docstrings_negative = (
'a = """false"""',
'"""false""" + """unless its optimized"""',
'1 + 1\n"""false"""',
'f"""no, top level but f-fstring"""'
)
for prefix in docstring_prefixes:
for negative in docstrings_negative:
# this cases should be result with single quote
# rather then triple quoted docstring
src = f"{prefix}{negative}"
self.check_ast_roundtrip(src)
self.check_src_dont_roundtrip(src)
def test_unary_op_factor(self):
for prefix in ("+", "-", "~"):
self.check_src_roundtrip(f"{prefix}1")
for prefix in ("not",):
self.check_src_roundtrip(f"{prefix} 1")
def test_slices(self):
self.check_src_roundtrip("a[1]")
self.check_src_roundtrip("a[1, 2]")
self.check_src_roundtrip("a[(1, *a)]")
class DirectoryTestCase(ASTTestCase):
"""Test roundtrip behaviour on all files in Lib and Lib/test."""
lib_dir = pathlib.Path(__file__).parent / ".."
test_directories = (lib_dir, lib_dir / "test")
run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py",
"test_ast.py", "test_asdl_parser.py", "test_fstring.py"}
_files_to_test = None
@classmethod
def files_to_test(cls):
if cls._files_to_test is not None:
return cls._files_to_test
items = [
item.resolve()
for directory in cls.test_directories
for item in directory.glob("*.py")
if not item.name.startswith("bad")
]
# Test limited subset of files unless the 'cpu' resource is specified.
if not test.support.is_resource_enabled("cpu"):
tests_to_run_always = {item for item in items if
item.name in cls.run_always_files}
items = set(random.sample(items, 10))
# Make sure that at least tests that heavily use grammar features are
# always considered in order to reduce the chance of missing something.
items = list(items | tests_to_run_always)
# bpo-31174: Store the names sample to always test the same files.
# It prevents false alarms when hunting reference leaks.
cls._files_to_test = items
return items
def test_files(self):
for item in self.files_to_test():
if test.support.verbose:
print(f"Testing {item.absolute()}")
with self.subTest(filename=item):
source = read_pyfile(item)
self.check_ast_roundtrip(source)
if __name__ == "__main__":
unittest.main()