diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 78ea654..59c227d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -25,3 +25,4 @@ Contributors (by order of appearance) : - Tyrel Souza - Corey Farwell - Michael Penkov +- Anthony Sottile diff --git a/docs/source/release-notes/3.3.0.rst b/docs/source/release-notes/3.3.0.rst index fc7c2f4..0b0feae 100644 --- a/docs/source/release-notes/3.3.0.rst +++ b/docs/source/release-notes/3.3.0.rst @@ -3,6 +3,8 @@ You can view the `3.3.0 milestone`_ on GitLab for more details. +- Dramatically improve the performance of Flake8 (`GitLab!156`_) + - Fix problem where hooks should only check \*.py files. (See also `GitLab#268`_) @@ -14,3 +16,5 @@ You can view the `3.3.0 milestone`_ on GitLab for more details. https://gitlab.com/pycqa/flake8/milestones/16 .. _GitLab#268: https://gitlab.com/pycqa/flake8/issues/268 +.. _GitLab!156: + https://gitlab.com/pycqa/flake8/merge_requests/156 diff --git a/src/flake8/checker.py b/src/flake8/checker.py index e0fa832..4cd049d 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -92,11 +92,6 @@ class Manager(object): raise self.using_multiprocessing = False - @staticmethod - def _cleanup_queue(q): - while not q.empty(): - q.get_nowait() - def _process_statistics(self): for checker in self.checkers: for statistic in defaults.STATISTIC_NAMES: @@ -279,10 +274,15 @@ class Manager(object): """Run the checkers in parallel.""" final_results = collections.defaultdict(list) final_statistics = collections.defaultdict(dict) - for ret in self.pool.imap_unordered( - _run_checks, self.checkers, - chunksize=_pool_chunksize(len(self.checkers), self.jobs), - ): + pool_map = self.pool.imap_unordered( + _run_checks, + self.checkers, + chunksize=calculate_pool_chunksize( + len(self.checkers), + self.jobs, + ), + ) + for ret in pool_map: filename, results, statistics = ret final_results[filename] = results final_statistics[filename] = statistics @@ -620,7 +620,7 @@ def _pool_init(): signal.signal(signal.SIGINT, signal.SIG_IGN) -def _pool_chunksize(num_checkers, num_jobs): +def calculate_pool_chunksize(num_checkers, num_jobs): """Determine the chunksize for the multiprocessing Pool. - For chunksize, see: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap # noqa