[WIP] Make native parser the default

mypy/main.py

@@ -1303,10 +1303,10 @@ def add_invertible_flag(
         dest="local_partial_types",
         help=argparse.SUPPRESS,
     )
-    # --native-parser enables the native parser (experimental)
+    # --native-parser enables the native parser.
     add_invertible_flag(
         "--native-parser",
-        default=False,
+        default=True,
         help="Enable faster parser that parses directly to mypy AST",
     )
     # --logical-deps adds some more dependencies that are not semantically needed, but

mypy/options.py

@@ -389,8 +389,8 @@ def __init__(self) -> None:
         self.logical_deps = False
         # If True, partial types can't span a module top level and a function
         self.local_partial_types = True
-        # If True, use the native parser (experimental)
-        self.native_parser = False
+        # If True, use the native parser
+        self.native_parser = True
         # Some behaviors are changed when using Bazel (https://bazel.build).
         self.bazel = False
         # If True, export inferred types for all expressions as BuildResult.types

mypy/test/testcheck.py

@@ -137,7 +137,6 @@ def run_case_once(
         options = parse_options(original_program_text, testcase, incremental_step)
         options.use_builtins_fixtures = True
         options.show_traceback = True
-        options.native_parser = bool(os.environ.get("TEST_NATIVE_PARSER"))
         options.reveal_verbose_types = not testcase.name.endswith("_no_verbose_reveal")
 
         if options.num_workers:
@@ -152,8 +151,9 @@ def run_case_once(
             if testcase.name.endswith("_parallel_only"):
                 raise pytest.skip("Test is only for parallel mode")
 
-        if options.native_parser and testcase.name.endswith("_no_native_parse"):
-            raise pytest.skip("Test not supported by native parser yet")
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
 
         # Enable some options automatically based on test file name.
         if "columns" in testcase.file:

mypy/test/testdeps.py

@@ -39,6 +39,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         options.export_types = True
         options.preserve_asts = True
         options.allow_empty_bodies = True
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
         messages, files, type_map = self.build(src, options)
         a = messages
         if files is None or type_map is None:

mypy/test/testfinegrained.py

@@ -44,6 +44,8 @@
 )
 
 # Set to True to perform (somewhat expensive) checks for duplicate AST nodes after merge
+from mypy.test.update_data import update_testcase_output
+
 CHECK_CONSISTENCY = False
 
 
@@ -130,6 +132,10 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         # Normalize paths in test output (for Windows).
         a = [line.replace("\\", "/") for line in a]
 
+        # This may not work perfectly, since it was designed for testcheck.py, use with care.
+        if testcase.output != a and testcase.config.getoption("--update-data", False):
+            update_testcase_output(testcase, a, incremental_step=1)
+
         assert_string_arrays_equal(
             testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})"
         )
@@ -155,6 +161,11 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
         options.export_types = "inspect" in testcase.file
         # Treat empty bodies safely for these test cases.
         options.allow_empty_bodies = not testcase.name.endswith("_no_empty")
+
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
+
         options.reveal_verbose_types = True
         if re.search("flags:.*--follow-imports", source) is None:
             # Override the default for follow_imports

mypy/test/testparse.py

@@ -13,6 +13,7 @@
 from mypy.parse import parse
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options
+from mypy.test.update_data import update_testcase_output
 from mypy.util import get_mypy_comments
 
 
@@ -115,6 +116,11 @@ def test_parse_error(testcase: DataDrivenTestCase) -> None:
     except CompileError as e:
         if e.module_with_blocker is not None:
             assert e.module_with_blocker == "__main__"
+
+        # This may not work perfectly, since it was designed for testcheck.py, use with care.
+        if testcase.output != e.messages and testcase.config.getoption("--update-data", False):
+            update_testcase_output(testcase, e.messages, incremental_step=1)
+
         # Verify that there was a compile error and that the error messages
         # are equivalent.
         assert_string_arrays_equal(

mypy/test/testsemanal.py

@@ -21,6 +21,7 @@
 
 # Semantic analyzer test cases: dump parse tree
 # Semantic analysis test case description files.
+from mypy.test.update_data import update_testcase_output
 from mypy.types import TypeStrVisitor
 
 semanal_files = find_test_files(
@@ -63,6 +64,9 @@ def test_semanal(testcase: DataDrivenTestCase) -> None:
         src = "\n".join(testcase.input)
         options = get_semanal_options(src, testcase)
         options.python_version = testfile_pyversion(testcase.file)
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
         result = build.build(
             sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir
         )
@@ -112,6 +116,11 @@ def test_semanal_error(testcase: DataDrivenTestCase) -> None:
         a = e.messages
     if testcase.normalize_output:
         a = normalize_error_messages(a)
+
+    # This may not work perfectly, since it was designed for testcheck.py, use with care.
+    if testcase.output != a and testcase.config.getoption("--update-data", False):
+        update_testcase_output(testcase, a, incremental_step=1)
+
     assert_string_arrays_equal(
         testcase.output, a, f"Invalid compiler output ({testcase.file}, line {testcase.line})"
     )

mypy/test/teststubtest.py

@@ -3207,7 +3207,7 @@ def test_mypy_build(self) -> None:
         output = run_stubtest(stub="+", runtime="", options=[])
         assert output == (
             "error: not checking stubs due to failed mypy compile:\n{}.pyi:1: "
-            "error: Invalid syntax  [syntax]\n".format(TEST_MODULE_NAME)
+            "error: Expected an expression  [syntax]\n".format(TEST_MODULE_NAME)
         )
 
         output = run_stubtest(stub="def f(): ...\ndef f(): ...", runtime="", options=[])

test-data/unit/check-async-await.test

@@ -192,8 +192,8 @@ async def f() -> None:
 [builtins fixtures/async_await.pyi]
 [typing fixtures/typing-async.pyi]
 
-[case testAsyncForTypeComments_no_native_parse]
-
+-- Native parser does not support type comments in `for` and `with` statements.
+[case testAsyncForTypeComments_old_parser]
 from typing import AsyncIterator, Union
 class C(AsyncIterator[int]):
     async def __anext__(self) -> int: return 0
@@ -342,8 +342,8 @@ async def f() -> None:
 [builtins fixtures/async_await.pyi]
 [typing fixtures/typing-async.pyi]
 
-[case testAsyncWithTypeComments_no_native_parse]
-
+-- Native parser does not support type comments in `for` and `with` statements.
+[case testAsyncWithTypeComments_old_parser]
 class C:
     async def __aenter__(self) -> int: pass
     async def __aexit__(self, x, y, z) -> None: pass

test-data/unit/check-basic.test

@@ -286,10 +286,11 @@ x = 1
 x in 1,  # E: Unsupported right operand type for in ("int")
 [builtins fixtures/tuple.pyi]
 
-[case testTrailingCommaInIfParsing_no_native_parse]
+[case testTrailingCommaInIfParsing]
 if x in 1, : pass
 [out]
-main:1: error: Invalid syntax
+main:1: error: Expected `:`, found `,`
+main:1: error: Expected a statement
 
 [case testInitReturnTypeError]
 class C:

test-data/unit/check-columns.test

@@ -1,10 +1,10 @@
 # Test column numbers in messages. --show-column-numbers is enabled implicitly by test runner.
 
-[case testColumnsSyntaxError_no_native_parse]
+[case testColumnsSyntaxError]
 f()
 1 +
 [out]
-main:2:5: error: Invalid syntax
+main:2:5: error: Expected an expression
 
 [case testColumnsNestedFunctions]
 import typing
@@ -146,8 +146,7 @@ if int():
     def f(a: 'A') -> None: pass
     (f(b=object())) # E:6: Unexpected keyword argument "b" for "f"
 
-[case testColumnInvalidType_no_native_parse]
-
+[case testColumnInvalidType]
 from typing import Iterable
 
 bad = 0
@@ -158,8 +157,8 @@ def f(x: bad): # E:10: Variable "__main__.bad" is not valid as a type \
            # N:8: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
 
 if int():
-    def g(x): # E:5: Variable "__main__.bad" is not valid as a type \
-              # N:5: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
+    def g(x): # E:11: Variable "__main__.bad" is not valid as a type \
+              # N:11: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
         # type: (bad) -> None
         y = 0  # type: bad  # E:9: Variable "__main__.bad" is not valid as a type \
                             # N:9: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
@@ -342,7 +341,7 @@ if int():
 main:2:11: error: Syntax error in type annotation
 main:2:11: note: Suggestion: Is there a spurious trailing comma?
 
-[case testColumnSyntaxErrorInTypeAnnotation2_no_native_parse]
+[case testColumnSyntaxErrorInTypeAnnotation2]
 if int():
     # TODO: It would be better to point to the type comment
     xyz = 0  # type: blurbnard blarb

test-data/unit/check-errorcodes.test

@@ -101,21 +101,19 @@ class A:
 [case testErrorCodeNoteHasNoCode]
 reveal_type(1) # N: Revealed type is "Literal[1]?"
 
-[case testErrorCodeSyntaxError_no_native_parse]
+[case testErrorCodeSyntaxError]
 1 ''
 [out]
-main:1: error: Invalid syntax  [syntax]
-[out version==3.10.0]
-main:1: error: Invalid syntax. Perhaps you forgot a comma?  [syntax]
+main:1: error: Simple statements must be separated by newlines or semicolons  [syntax]
 
-[case testErrorCodeSyntaxError2_no_native_parse]
+[case testErrorCodeSyntaxError2]
 def f(): # E: Type signature has too many parameters  [syntax]
     # type: (int) -> None
     1
 
 x = 0  # type: x y  # E: Syntax error in type comment "x y"  [syntax]
 
-[case testErrorCodeSyntaxError3_no_native_parse]
+[case testErrorCodeSyntaxError3]
 # This is a bit inconsistent -- syntax error would be more logical?
 x: 'a b'  # E: Invalid type comment or annotation  [valid-type]
 for v in x:  # type: int, int  # E: Syntax error in type annotation  [syntax] \
@@ -281,7 +279,7 @@ def h(x  # type: xyz  # type: ignore[foo]  # E: Name "xyz" is not defined  [name
 import nostub  # type: ignore[import]
 from defusedxml import xyz  # type: ignore[import]
 
-[case testErrorCodeBadIgnore_no_native_parse]
+[case testErrorCodeBadIgnore]
 import nostub # type: ignore xyz  # E: Invalid "type: ignore" comment  [syntax] \
                                   # E: Cannot find implementation or library stub for module named "nostub"  [import-not-found] \
                                   # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
@@ -299,7 +297,7 @@ def f(x,  # type: int  # type: ignore[  # E: Invalid "type: ignore" comment  [sy
     # type: (...) -> None
     pass
 
-[case testErrorCodeBadIgnoreNoExtraComment_no_native_parse]
+[case testErrorCodeBadIgnoreNoExtraComment]
 # Omit the E: ... comments, as they affect parsing
 import nostub # type: ignore xyz
 import nostub # type: ignore[xyz
@@ -846,7 +844,7 @@ main:1: error: Name "y" is not defined  [name-defined]
 main:2: error: Name "ignored" is not defined  [name-defined]
 main:2: error: Name "y" is not defined  [name-defined]
 
-[case testErrorCodeTypeIgnoreMisspelled2_no_native_parse]
+[case testErrorCodeTypeIgnoreMisspelled2]
 x = y  # type: int  # type: ignored[foo]
 x = y  # type: int  # type: ignored [foo]
 [out]
@@ -1098,7 +1096,7 @@ def f(arg: int) -> int:
 def f(arg: str) -> str:
     ...
 
-[case testSliceInDictBuiltin_no_native_parse]
+[case testSliceInDictBuiltin]
 # flags: --show-column-numbers
 b: dict[int, x:y]
 c: dict[x:y]
@@ -1111,7 +1109,7 @@ main:3:4: error: "dict" expects 2 type arguments, but 1 given  [type-arg]
 main:3:9: error: Invalid type comment or annotation  [valid-type]
 main:3:9: note: did you mean to use ',' instead of ':' ?
 
-[case testSliceInDictTyping_no_native_parse]
+[case testSliceInDictTyping]
 # flags: --show-column-numbers
 from typing import Dict
 b: Dict[int, x:y]

test-data/unit/check-expressions.test

@@ -1478,7 +1478,7 @@ if int():
     b = (x for x in a)  # E: Generator has incompatible item type "Callable[[], str]"; expected "Callable[[], int]"
 [builtins fixtures/list.pyi]
 
-[case testGeneratorNoSpuriousError_no_native_parse]
+[case testGeneratorNoSpuriousError]
 from typing import Iterable, overload
 
 @overload
@@ -1493,8 +1493,8 @@ take_iterable(reveal_type(1 for _ in [])) # N: Revealed type is "typing.Generato
 # NOTE: Type is revealed for every overload tried
 # TODO: Overload shouldn't fail if expression contains an error that shouldn't affect the inferred type.
 take_iterable(reveal_type(1 if (-"") else 1 for _ in [])) # N: Revealed type is "typing.Generator[builtins.int, None, None]" \
-                                                          # N: Revealed type is "typing.Generator[builtins.bool, None, None]" \
                                                           # E: Generator has incompatible item type "int"; expected "bool" \
+                                                          # N: Revealed type is "typing.Generator[builtins.bool, None, None]" \
                                                           # E: Unsupported operand type for unary - ("str")
 
 [builtins fixtures/for.pyi]
@@ -1937,13 +1937,12 @@ None == None
 None < None  # E: Unsupported left operand type for < ("None")
 [builtins fixtures/ops.pyi]
 
-[case testDictWithStarExpr_no_native_parse]
-
-b = {'z': 26, *a}  # E: Invalid syntax
+[case testDictWithStarExpr]
+b = {'z': 26, *a}  # E: Starred expression cannot be used here \
+                   # E: Expected `:`, found `}`
 [builtins fixtures/dict.pyi]
 
 [case testDictWithStarStarExpr]
-
 from typing import Dict, Iterable
 
 class Thing:

test-data/unit/check-fastparse.test

@@ -1,31 +1,25 @@
-[case testFastParseSyntaxError_no_native_parse]
-
-1 +  # E: Invalid syntax
-
-[case testFastParseTypeCommentSyntaxError_no_native_parse]
+[case testFastParseSyntaxError]
+1 +  # E: Expected an expression
 
+[case testFastParseTypeCommentSyntaxError]
 x = None # type: a : b  # E: Syntax error in type comment "a : b"
 
 [case testFastParseInvalidTypeComment]
-
 x = None # type: a + b  # E: Invalid type comment or annotation
 
 -- Function type comments are attributed to the function def line.
 -- This happens in both parsers.
-[case testFastParseFunctionAnnotationSyntaxError_no_native_parse]
-
+[case testFastParseFunctionAnnotationSyntaxError]
 def f():  # E: Syntax error in type comment "None -> None" # N: Suggestion: wrap argument types in parentheses
   # type: None -> None
   pass
 
-[case testFastParseFunctionAnnotationSyntaxErrorSpaces_no_native_parse]
-
+[case testFastParseFunctionAnnotationSyntaxErrorSpaces]
 def f():  # E: Syntax error in type comment "None -> None" # N: Suggestion: wrap argument types in parentheses
   # type:             None -> None
   pass
 
 [case testFastParseInvalidFunctionAnnotation]
-
 def f(x):  # E: Invalid type comment or annotation
   # type: (a + b) -> None
   pass
@@ -156,8 +150,7 @@ def f(a,        # type: A
 [builtins fixtures/dict.pyi]
 [out]
 
-[case testFastParsePerArgumentAnnotationsWithAnnotatedBareStar_no_native_parse]
-
+[case testFastParsePerArgumentAnnotationsWithAnnotatedBareStar]
 def f(*, # type: int  # E: Bare * has associated type comment
       x  # type: str
       ):
@@ -286,8 +279,7 @@ def f7(x: int):  # E: Function has duplicate type signatures
   # type: (int) -> int
   pass
 
-[case testFastParserDuplicateNames_no_native_parse]
-
+[case testFastParserDuplicateNames]
 def f(x, y, z):
   pass