From 2529983e5c10961c7ba4d70a9e7d6c9c3db56b8c Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 13 May 2020 00:37:38 -0700 Subject: [PATCH 1/2] FileChecker.run_check: Delete an extremely expensive LOG.debug call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This single statement accounts for about 11% of flake8’s running time when debugging is *disabled*. Signed-off-by: Anders Kaseorg --- src/flake8/checker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index 30d44b8..c8a53ea 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -424,7 +424,6 @@ class FileChecker(object): def run_check(self, plugin, **arguments): """Run the check in a single plugin.""" - LOG.debug("Running %r with %r", plugin, arguments) try: self.processor.keyword_arguments_for( plugin["parameters"], arguments From 0bb9e453c7f68ac64cdb7f425da5e5be7302f7b1 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 13 May 2020 02:20:22 -0700 Subject: [PATCH 2/2] FileProcessor.keyword_arguments_for: Optimize by removing arguments. Attaching the extra arguments directly to the FileProcessor object lets us streamline this code for a 6% speedup. Signed-off-by: Anders Kaseorg --- src/flake8/checker.py | 20 ++++++++++++++------ src/flake8/processor.py | 13 +++++-------- tests/unit/test_file_processor.py | 10 ++++++---- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index c8a53ea..61dfb33 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -422,11 +422,11 @@ class FileChecker(object): self.results.append((error_code, line_number, column, text, line)) return error_code - def run_check(self, plugin, **arguments): + def run_check(self, plugin): """Run the check in a single plugin.""" try: - self.processor.keyword_arguments_for( - plugin["parameters"], arguments + arguments = self.processor.keyword_arguments_for( + plugin["parameters"] ) except AttributeError as ae: LOG.error("Plugin requested unknown parameters.") @@ -493,8 +493,10 @@ class FileChecker(object): ) return + self.processor.tree = ast + for plugin in self.checks["ast_plugins"]: - checker = self.run_check(plugin, tree=ast) + checker = self.run_check(plugin) # If the plugin uses a class, call the run method of it, otherwise # the call should return something iterable itself try: @@ -509,6 +511,8 @@ class FileChecker(object): text=text, ) + del self.processor.tree + def run_logical_checks(self): """Run all checks expecting a logical line.""" comments, logical_line, mapping = self.processor.build_logical_line() @@ -520,7 +524,7 @@ class FileChecker(object): for plugin in self.checks["logical_line_plugins"]: self.processor.update_checker_state_for(plugin) - results = self.run_check(plugin, logical_line=logical_line) or () + results = self.run_check(plugin) or () for offset, text in results: line_number, column_offset = find_offset(offset, mapping) if line_number == column_offset == 0: @@ -539,9 +543,11 @@ class FileChecker(object): A single physical check may return multiple errors. """ + self.processor.physical_line = physical_line + for plugin in self.checks["physical_line_plugins"]: self.processor.update_checker_state_for(plugin) - result = self.run_check(plugin, physical_line=physical_line) + result = self.run_check(plugin) if result is not None: # This is a single result if first element is an int @@ -564,6 +570,8 @@ class FileChecker(object): text=text, ) + del self.processor.physical_line + def process_tokens(self): """Process tokens and trigger checks. diff --git a/src/flake8/processor.py b/src/flake8/processor.py index 0375ed9..ee8694c 100644 --- a/src/flake8/processor.py +++ b/src/flake8/processor.py @@ -239,18 +239,15 @@ class FileProcessor(object): yield line self.line_number += 1 - def keyword_arguments_for(self, parameters, arguments=None): - # type: (Dict[str, bool], Optional[Dict[str, Any]]) -> Dict[str, Any] + def keyword_arguments_for(self, parameters): + # type: (Dict[str, bool]) -> Dict[str, Any] """Generate the keyword arguments for a list of parameters.""" - if arguments is None: - arguments = {} - for param, required in parameters.items(): - if param in arguments: - continue + arguments = {} + for param in parameters: try: arguments[param] = getattr(self, param) except AttributeError as exc: - if required: + if parameters[param]: LOG.exception(exc) raise else: diff --git a/tests/unit/test_file_processor.py b/tests/unit/test_file_processor.py index af7717e..552280b 100644 --- a/tests/unit/test_file_processor.py +++ b/tests/unit/test_file_processor.py @@ -188,7 +188,7 @@ def test_next_line(default_options): @pytest.mark.parametrize('params, args, expected_kwargs', [ ({'blank_before': True, 'blank_lines': True}, - None, + {}, {'blank_before': 0, 'blank_lines': 0}), ({'noqa': True, 'fake': True}, {'fake': 'foo'}, @@ -196,8 +196,8 @@ def test_next_line(default_options): ({'blank_before': True, 'blank_lines': True, 'noqa': True}, {'blank_before': 10, 'blank_lines': 5, 'noqa': True}, {'blank_before': 10, 'blank_lines': 5, 'noqa': True}), - ({}, {'fake': 'foo'}, {'fake': 'foo'}), - ({'non-existent': False}, {'fake': 'foo'}, {'fake': 'foo'}), + ({'fake': False}, {'fake': 'foo'}, {'fake': 'foo'}), + ({'fake': False, 'non-existent': False}, {'fake': 'foo'}, {'fake': 'foo'}), ]) def test_keyword_arguments_for(params, args, expected_kwargs, default_options): """Verify the keyword args are generated properly.""" @@ -206,7 +206,9 @@ def test_keyword_arguments_for(params, args, expected_kwargs, default_options): ]) kwargs_for = file_processor.keyword_arguments_for - assert kwargs_for(params, args) == expected_kwargs + for k, v in args.items(): + setattr(file_processor, k, v) + assert kwargs_for(params) == expected_kwargs def test_keyword_arguments_for_does_not_handle_attribute_errors(