From 4e56fc0f6a8d1d9dde4de14abcba38f54ea6edbe Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 25 Jan 2022 13:30:36 -0500 Subject: [PATCH] pregenerate the pycodestyle plugin to avoid call overhead --- bin/gen-pycodestyle-plugin | 96 +++++++++++++++++++ setup.cfg | 38 +------- src/flake8/plugins/pycodestyle.py | 123 +++++++++++++++++++++++++ tests/integration/test_plugins.py | 2 +- tests/unit/plugins/pycodestyle_test.py | 33 +++++++ 5 files changed, 255 insertions(+), 37 deletions(-) create mode 100755 bin/gen-pycodestyle-plugin create mode 100644 src/flake8/plugins/pycodestyle.py create mode 100644 tests/unit/plugins/pycodestyle_test.py diff --git a/bin/gen-pycodestyle-plugin b/bin/gen-pycodestyle-plugin new file mode 100755 index 0000000..3540a9a --- /dev/null +++ b/bin/gen-pycodestyle-plugin @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import inspect +import os.path +from typing import Any +from typing import Callable +from typing import Generator +from typing import NamedTuple +from typing import Tuple + +import pycodestyle + + +def _too_long(s: str) -> str: + if len(s) >= 80: + return f"{s} # noqa: E501" + else: + return s + + +class Call(NamedTuple): + name: str + is_generator: bool + params: Tuple[str, ...] + + def to_src(self) -> str: + params_s = ", ".join(self.params) + if self.is_generator: + return _too_long(f" yield from _{self.name}({params_s})") + else: + lines = ( + _too_long(f" ret = _{self.name}({params_s})"), + " if ret is not None:", + " yield ret", + ) + return "\n".join(lines) + + @classmethod + def from_func(cls, func: Callable[..., Any]) -> "Call": + spec = inspect.getfullargspec(func) + params = tuple(spec.args) + return cls(func.__name__, inspect.isgeneratorfunction(func), params) + + +def lines() -> Generator[str, None, None]: + logical = [] + physical = [] + + logical = [ + Call.from_func(check) for check in pycodestyle._checks["logical_line"] + ] + physical = [ + Call.from_func(check) for check in pycodestyle._checks["physical_line"] + ] + assert not pycodestyle._checks["tree"] + + yield f'"""Generated using ./bin/{os.path.basename(__file__)}."""' + yield "# fmt: off" + yield "from typing import Any" + yield "from typing import Generator" + yield "from typing import Tuple" + yield "" + imports = sorted(call.name for call in logical + physical) + for name in imports: + yield _too_long(f"from pycodestyle import {name} as _{name}") + yield "" + yield "" + + yield "def pycodestyle_logical(" + logical_params = {param for call in logical for param in call.params} + for param in sorted(logical_params): + yield f" {param}: Any," + yield ") -> Generator[Tuple[int, str], None, None]:" + yield ' """Run pycodestyle logical checks."""' + for call in sorted(logical): + yield call.to_src() + yield "" + yield "" + + yield "def pycodestyle_physical(" + physical_params = {param for call in physical for param in call.params} + for param in sorted(physical_params): + yield f" {param}: Any," + yield ") -> Generator[Tuple[int, str], None, None]:" + yield ' """Run pycodestyle physical checks."""' + for call in sorted(physical): + yield call.to_src() + + +def main() -> int: + for line in lines(): + print(line) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.cfg b/setup.cfg index f230cfe..9e199c4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,42 +53,8 @@ console_scripts = flake8 = flake8.main.cli:main flake8.extension = F = flake8.plugins.pyflakes:FlakesChecker - pycodestyle.ambiguous_identifier = pycodestyle:ambiguous_identifier - pycodestyle.bare_except = pycodestyle:bare_except - pycodestyle.blank_lines = pycodestyle:blank_lines - pycodestyle.break_after_binary_operator = pycodestyle:break_after_binary_operator - pycodestyle.break_before_binary_operator = pycodestyle:break_before_binary_operator - pycodestyle.comparison_negative = pycodestyle:comparison_negative - pycodestyle.comparison_to_singleton = pycodestyle:comparison_to_singleton - pycodestyle.comparison_type = pycodestyle:comparison_type - pycodestyle.compound_statements = pycodestyle:compound_statements - pycodestyle.continued_indentation = pycodestyle:continued_indentation - pycodestyle.explicit_line_join = pycodestyle:explicit_line_join - pycodestyle.extraneous_whitespace = pycodestyle:extraneous_whitespace - pycodestyle.imports_on_separate_lines = pycodestyle:imports_on_separate_lines - pycodestyle.indentation = pycodestyle:indentation - pycodestyle.maximum_doc_length = pycodestyle:maximum_doc_length - pycodestyle.maximum_line_length = pycodestyle:maximum_line_length - pycodestyle.missing_whitespace = pycodestyle:missing_whitespace - pycodestyle.missing_whitespace_after_import_keyword = pycodestyle:missing_whitespace_after_import_keyword - pycodestyle.missing_whitespace_around_operator = pycodestyle:missing_whitespace_around_operator - pycodestyle.module_imports_on_top_of_file = pycodestyle:module_imports_on_top_of_file - pycodestyle.python_3000_async_await_keywords = pycodestyle:python_3000_async_await_keywords - pycodestyle.python_3000_backticks = pycodestyle:python_3000_backticks - pycodestyle.python_3000_has_key = pycodestyle:python_3000_has_key - pycodestyle.python_3000_invalid_escape_sequence = pycodestyle:python_3000_invalid_escape_sequence - pycodestyle.python_3000_not_equal = pycodestyle:python_3000_not_equal - pycodestyle.python_3000_raise_comma = pycodestyle:python_3000_raise_comma - pycodestyle.tabs_obsolete = pycodestyle:tabs_obsolete - pycodestyle.tabs_or_spaces = pycodestyle:tabs_or_spaces - pycodestyle.trailing_blank_lines = pycodestyle:trailing_blank_lines - pycodestyle.trailing_whitespace = pycodestyle:trailing_whitespace - pycodestyle.whitespace_around_comma = pycodestyle:whitespace_around_comma - pycodestyle.whitespace_around_keywords = pycodestyle:whitespace_around_keywords - pycodestyle.whitespace_around_named_parameter_equals = pycodestyle:whitespace_around_named_parameter_equals - pycodestyle.whitespace_around_operator = pycodestyle:whitespace_around_operator - pycodestyle.whitespace_before_comment = pycodestyle:whitespace_before_comment - pycodestyle.whitespace_before_parameters = pycodestyle:whitespace_before_parameters + E = flake8.plugins.pycodestyle:pycodestyle_logical + W = flake8.plugins.pycodestyle:pycodestyle_physical flake8.report = default = flake8.formatting.default:Default pylint = flake8.formatting.default:Pylint diff --git a/src/flake8/plugins/pycodestyle.py b/src/flake8/plugins/pycodestyle.py new file mode 100644 index 0000000..e8011b2 --- /dev/null +++ b/src/flake8/plugins/pycodestyle.py @@ -0,0 +1,123 @@ +"""Generated using ./bin/gen-pycodestyle-plugin.""" +# fmt: off +from typing import Any +from typing import Generator +from typing import Tuple + +from pycodestyle import ambiguous_identifier as _ambiguous_identifier +from pycodestyle import bare_except as _bare_except +from pycodestyle import blank_lines as _blank_lines +from pycodestyle import break_after_binary_operator as _break_after_binary_operator # noqa: E501 +from pycodestyle import break_before_binary_operator as _break_before_binary_operator # noqa: E501 +from pycodestyle import comparison_negative as _comparison_negative +from pycodestyle import comparison_to_singleton as _comparison_to_singleton +from pycodestyle import comparison_type as _comparison_type +from pycodestyle import compound_statements as _compound_statements +from pycodestyle import continued_indentation as _continued_indentation +from pycodestyle import explicit_line_join as _explicit_line_join +from pycodestyle import extraneous_whitespace as _extraneous_whitespace +from pycodestyle import imports_on_separate_lines as _imports_on_separate_lines +from pycodestyle import indentation as _indentation +from pycodestyle import maximum_doc_length as _maximum_doc_length +from pycodestyle import maximum_line_length as _maximum_line_length +from pycodestyle import missing_whitespace as _missing_whitespace +from pycodestyle import missing_whitespace_after_import_keyword as _missing_whitespace_after_import_keyword # noqa: E501 +from pycodestyle import missing_whitespace_around_operator as _missing_whitespace_around_operator # noqa: E501 +from pycodestyle import module_imports_on_top_of_file as _module_imports_on_top_of_file # noqa: E501 +from pycodestyle import python_3000_async_await_keywords as _python_3000_async_await_keywords # noqa: E501 +from pycodestyle import python_3000_backticks as _python_3000_backticks +from pycodestyle import python_3000_has_key as _python_3000_has_key +from pycodestyle import python_3000_invalid_escape_sequence as _python_3000_invalid_escape_sequence # noqa: E501 +from pycodestyle import python_3000_not_equal as _python_3000_not_equal +from pycodestyle import python_3000_raise_comma as _python_3000_raise_comma +from pycodestyle import tabs_obsolete as _tabs_obsolete +from pycodestyle import tabs_or_spaces as _tabs_or_spaces +from pycodestyle import trailing_blank_lines as _trailing_blank_lines +from pycodestyle import trailing_whitespace as _trailing_whitespace +from pycodestyle import whitespace_around_comma as _whitespace_around_comma +from pycodestyle import whitespace_around_keywords as _whitespace_around_keywords # noqa: E501 +from pycodestyle import whitespace_around_named_parameter_equals as _whitespace_around_named_parameter_equals # noqa: E501 +from pycodestyle import whitespace_around_operator as _whitespace_around_operator # noqa: E501 +from pycodestyle import whitespace_before_comment as _whitespace_before_comment +from pycodestyle import whitespace_before_parameters as _whitespace_before_parameters # noqa: E501 + + +def pycodestyle_logical( + blank_before: Any, + blank_lines: Any, + checker_state: Any, + hang_closing: Any, + indent_char: Any, + indent_level: Any, + indent_size: Any, + line_number: Any, + lines: Any, + logical_line: Any, + max_doc_length: Any, + noqa: Any, + previous_indent_level: Any, + previous_logical: Any, + previous_unindented_logical_line: Any, + tokens: Any, + verbose: Any, +) -> Generator[Tuple[int, str], None, None]: + """Run pycodestyle logical checks.""" + yield from _ambiguous_identifier(logical_line, tokens) + yield from _bare_except(logical_line, noqa) + yield from _blank_lines(logical_line, blank_lines, indent_level, line_number, blank_before, previous_logical, previous_unindented_logical_line, previous_indent_level, lines) # noqa: E501 + yield from _break_after_binary_operator(logical_line, tokens) + yield from _break_before_binary_operator(logical_line, tokens) + yield from _comparison_negative(logical_line) + yield from _comparison_to_singleton(logical_line, noqa) + yield from _comparison_type(logical_line, noqa) + yield from _compound_statements(logical_line) + yield from _continued_indentation(logical_line, tokens, indent_level, hang_closing, indent_char, indent_size, noqa, verbose) # noqa: E501 + yield from _explicit_line_join(logical_line, tokens) + yield from _extraneous_whitespace(logical_line) + yield from _imports_on_separate_lines(logical_line) + yield from _indentation(logical_line, previous_logical, indent_char, indent_level, previous_indent_level, indent_size) # noqa: E501 + yield from _maximum_doc_length(logical_line, max_doc_length, noqa, tokens) + yield from _missing_whitespace(logical_line) + yield from _missing_whitespace_after_import_keyword(logical_line) + yield from _missing_whitespace_around_operator(logical_line, tokens) + yield from _module_imports_on_top_of_file(logical_line, indent_level, checker_state, noqa) # noqa: E501 + yield from _python_3000_async_await_keywords(logical_line, tokens) + yield from _python_3000_backticks(logical_line) + yield from _python_3000_has_key(logical_line, noqa) + yield from _python_3000_invalid_escape_sequence(logical_line, tokens, noqa) + yield from _python_3000_not_equal(logical_line) + yield from _python_3000_raise_comma(logical_line) + yield from _whitespace_around_comma(logical_line) + yield from _whitespace_around_keywords(logical_line) + yield from _whitespace_around_named_parameter_equals(logical_line, tokens) + yield from _whitespace_around_operator(logical_line) + yield from _whitespace_before_comment(logical_line, tokens) + yield from _whitespace_before_parameters(logical_line, tokens) + + +def pycodestyle_physical( + indent_char: Any, + line_number: Any, + lines: Any, + max_line_length: Any, + multiline: Any, + noqa: Any, + physical_line: Any, + total_lines: Any, +) -> Generator[Tuple[int, str], None, None]: + """Run pycodestyle physical checks.""" + ret = _maximum_line_length(physical_line, max_line_length, multiline, line_number, noqa) # noqa: E501 + if ret is not None: + yield ret + ret = _tabs_obsolete(physical_line) + if ret is not None: + yield ret + ret = _tabs_or_spaces(physical_line, indent_char) + if ret is not None: + yield ret + ret = _trailing_blank_lines(physical_line, lines, line_number, total_lines) + if ret is not None: + yield ret + ret = _trailing_whitespace(physical_line) + if ret is not None: + yield ret diff --git a/tests/integration/test_plugins.py b/tests/integration/test_plugins.py index 03f0caf..bcf4d61 100644 --- a/tests/integration/test_plugins.py +++ b/tests/integration/test_plugins.py @@ -104,7 +104,7 @@ def test_local_plugin_can_add_option(local_config): args = aggregator.aggregate_options(option_manager, cfg, cfg_dir, argv) - assert args.extended_default_select == {"XE", "F", "E", "C90"} + assert args.extended_default_select == {"XE", "F", "E", "W", "C90"} assert args.anopt == "foo" diff --git a/tests/unit/plugins/pycodestyle_test.py b/tests/unit/plugins/pycodestyle_test.py new file mode 100644 index 0000000..703970f --- /dev/null +++ b/tests/unit/plugins/pycodestyle_test.py @@ -0,0 +1,33 @@ +import importlib.machinery +import importlib.util +import os.path + +import flake8.plugins.pycodestyle + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def test_up_to_date(): + """Validate that the generated pycodestyle plugin is up to date. + + We generate two "meta" plugins for pycodestyle to avoid calling overhead. + + To regenerate run: + + ./bin/gen-pycodestyle-plugin > src/flake8/plugins/pycodestyle.py + """ + + path = os.path.join(HERE, "../../../bin/gen-pycodestyle-plugin") + name = os.path.basename(path) + loader = importlib.machinery.SourceFileLoader(name, path) + spec = importlib.util.spec_from_loader(loader.name, loader) + assert spec is not None + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + + expected = "".join(f"{line}\n" for line in mod.lines()) + + with open(flake8.plugins.pycodestyle.__file__) as f: + contents = f.read() + + assert contents == expected