Merge pull request #1374 from PyCQA/bug/1372

Handle new SyntaxError tuple on 3.10

src/flake8/checker.py

@@ -443,14 +443,26 @@ class FileChecker:
             token = ()
             row, column = (1, 0)
 
-        if column > 0 and token and isinstance(exception, SyntaxError):
+        if (
+            column > 0
+            and token
+            and isinstance(exception, SyntaxError)
+            and len(token) == 4  # Python 3.9 or earlier
+        ):
             # NOTE(sigmavirus24): SyntaxErrors report 1-indexed column
             # numbers. We need to decrement the column number by 1 at
             # least.
             column_offset = 1
             row_offset = 0
-            # See also: https://github.com/pycqa/flake8/issues/169
-            physical_line = token[-1]
+            # See also: https://github.com/pycqa/flake8/issues/169,
+            # https://github.com/PyCQA/flake8/issues/1372
+            # On Python 3.9 and earlier, token will be a 4-item tuple with the
+            # last item being the string. Starting with 3.10, they added to
+            # the tuple so now instead of it ending with the code that failed
+            # to parse, it ends with the end of the section of code that
+            # failed to parse. Luckily the absolute position in the tuple is
+            # stable across versions so we can use that here
+            physical_line = token[3]
 
             # NOTE(sigmavirus24): Not all "tokens" have a string as the last
             # argument. In this event, let's skip trying to find the correct

tests/integration/test_checker.py

@@ -1,4 +1,5 @@
 """Integration tests for the checker submodule."""
+import sys
 from unittest import mock
 
 import pytest
@@ -336,3 +337,26 @@ def test_acquire_when_multiprocessing_pool_can_not_initialize():
 
     pool.assert_called_once_with(2, checker._pool_init)
     assert result is None
+
+
+def test_handling_syntaxerrors_across_pythons():
+    """Verify we properly handle exception argument tuples.
+
+    Python 3.10 added more information to the SyntaxError parse token tuple.
+    We need to handle that correctly to avoid crashing.
+    https://github.com/PyCQA/flake8/issues/1372
+    """
+    if sys.version_info < (3, 10):  # pragma: no cover (<3.10)
+        # Python 3.9 or older
+        err = SyntaxError(
+            "invalid syntax", ("<unknown>", 2, 5, "bad python:\n")
+        )
+        expected = (2, 4)
+    else:  # pragma: no cover (3.10+)
+        err = SyntaxError(
+            "invalid syntax", ("<unknown>", 2, 1, "bad python:\n", 2, 11)
+        )
+        expected = (2, 1)
+    file_checker = checker.FileChecker("-", {}, mock.MagicMock())
+    actual = file_checker._extract_syntax_information(err)
+    assert actual == expected