Speed up flake8 when only 1 filename is passed

~40% improvement over status quo (perf measurements are best-of-5)

### before

```console
$ time flake8 /dev/null

real    0m0.337s
user    0m0.212s
sys     0m0.028s
```

### after

```console
$ time flake8 /dev/null

real    0m0.197s
user    0m0.182s
sys     0m0.012s
```
This commit is contained in:
Anthony Sottile 2019-02-16 17:28:31 -08:00
parent 684ffb3306
commit e8f43e1243
2 changed files with 14 additions and 16 deletions

View file

@ -73,7 +73,6 @@ class Manager(object):
self.options = style_guide.options self.options = style_guide.options
self.checks = checker_plugins self.checks = checker_plugins
self.jobs = self._job_count() self.jobs = self._job_count()
self.using_multiprocessing = self.jobs > 1
self.processes = [] self.processes = []
self.checkers = [] self.checkers = []
self.statistics = { self.statistics = {
@ -279,7 +278,6 @@ class Manager(object):
except OSError as oserr: except OSError as oserr:
if oserr.errno not in SERIAL_RETRY_ERRNOS: if oserr.errno not in SERIAL_RETRY_ERRNOS:
raise raise
self.using_multiprocessing = False
self.run_serial() self.run_serial()
return return
@ -326,16 +324,10 @@ class Manager(object):
fallback to serial processing. fallback to serial processing.
""" """
try: try:
if self.using_multiprocessing: if self.jobs > 1 and len(self.checkers) > 1:
self.run_parallel() self.run_parallel()
else: else:
self.run_serial() self.run_serial()
except OSError as oserr:
if oserr.errno not in SERIAL_RETRY_ERRNOS:
LOG.exception(oserr)
raise
LOG.warning("Running in serial after OS exception, %r", oserr)
self.run_serial()
except KeyboardInterrupt: except KeyboardInterrupt:
LOG.warning("Flake8 was interrupted by the user") LOG.warning("Flake8 was interrupted by the user")
raise exceptions.EarlyQuit("Early quit while running checks") raise exceptions.EarlyQuit("Early quit while running checks")

View file

@ -16,27 +16,33 @@ def style_guide_mock(**kwargs):
return style_guide return style_guide
def _parallel_checker_manager():
"""Call Manager.run() and return the number of calls to `run_serial`."""
style_guide = style_guide_mock()
manager = checker.Manager(style_guide, [], [])
# multiple checkers is needed for parallel mode
manager.checkers = [mock.Mock(), mock.Mock()]
return manager
def test_oserrors_cause_serial_fall_back(): def test_oserrors_cause_serial_fall_back():
"""Verify that OSErrors will cause the Manager to fallback to serial.""" """Verify that OSErrors will cause the Manager to fallback to serial."""
err = OSError(errno.ENOSPC, 'Ominous message about spaceeeeee') err = OSError(errno.ENOSPC, 'Ominous message about spaceeeeee')
style_guide = style_guide_mock()
with mock.patch('_multiprocessing.SemLock', side_effect=err): with mock.patch('_multiprocessing.SemLock', side_effect=err):
manager = checker.Manager(style_guide, [], []) manager = _parallel_checker_manager()
with mock.patch.object(manager, 'run_serial') as serial: with mock.patch.object(manager, 'run_serial') as serial:
manager.run() manager.run()
assert serial.call_count == 1 assert serial.call_count == 1
assert manager.using_multiprocessing is False
@mock.patch('flake8.utils.is_windows', return_value=False) @mock.patch('flake8.utils.is_windows', return_value=False)
def test_oserrors_are_reraised(is_windows): def test_oserrors_are_reraised(is_windows):
"""Verify that unexpected OSErrors will cause the Manager to reraise.""" """Verify that unexpected OSErrors will cause the Manager to reraise."""
err = OSError(errno.EAGAIN, 'Ominous message') err = OSError(errno.EAGAIN, 'Ominous message')
style_guide = style_guide_mock()
with mock.patch('_multiprocessing.SemLock', side_effect=err): with mock.patch('_multiprocessing.SemLock', side_effect=err):
with pytest.raises(OSError): manager = _parallel_checker_manager()
manager = checker.Manager(style_guide, [], []) with mock.patch.object(manager, 'run_serial') as serial:
with mock.patch.object(manager, 'run_serial') as serial: with pytest.raises(OSError):
manager.run() manager.run()
assert serial.call_count == 0 assert serial.call_count == 0