Speed up flake8 when only 1 filename is passed

~40% improvement over status quo (perf measurements are best-of-5)

### before

```console
$ time flake8 /dev/null

real    0m0.337s
user    0m0.212s
sys     0m0.028s
```

### after

```console
$ time flake8 /dev/null

real    0m0.197s
user    0m0.182s
sys     0m0.012s
```
This commit is contained in:
Anthony Sottile 2019-02-16 17:28:31 -08:00
parent 684ffb3306
commit e8f43e1243
2 changed files with 14 additions and 16 deletions

View file

@ -73,7 +73,6 @@ class Manager(object):
self.options = style_guide.options
self.checks = checker_plugins
self.jobs = self._job_count()
self.using_multiprocessing = self.jobs > 1
self.processes = []
self.checkers = []
self.statistics = {
@ -279,7 +278,6 @@ class Manager(object):
except OSError as oserr:
if oserr.errno not in SERIAL_RETRY_ERRNOS:
raise
self.using_multiprocessing = False
self.run_serial()
return
@ -326,16 +324,10 @@ class Manager(object):
fallback to serial processing.
"""
try:
if self.using_multiprocessing:
if self.jobs > 1 and len(self.checkers) > 1:
self.run_parallel()
else:
self.run_serial()
except OSError as oserr:
if oserr.errno not in SERIAL_RETRY_ERRNOS:
LOG.exception(oserr)
raise
LOG.warning("Running in serial after OS exception, %r", oserr)
self.run_serial()
except KeyboardInterrupt:
LOG.warning("Flake8 was interrupted by the user")
raise exceptions.EarlyQuit("Early quit while running checks")

View file

@ -16,27 +16,33 @@ def style_guide_mock(**kwargs):
return style_guide
def _parallel_checker_manager():
"""Call Manager.run() and return the number of calls to `run_serial`."""
style_guide = style_guide_mock()
manager = checker.Manager(style_guide, [], [])
# multiple checkers is needed for parallel mode
manager.checkers = [mock.Mock(), mock.Mock()]
return manager
def test_oserrors_cause_serial_fall_back():
"""Verify that OSErrors will cause the Manager to fallback to serial."""
err = OSError(errno.ENOSPC, 'Ominous message about spaceeeeee')
style_guide = style_guide_mock()
with mock.patch('_multiprocessing.SemLock', side_effect=err):
manager = checker.Manager(style_guide, [], [])
manager = _parallel_checker_manager()
with mock.patch.object(manager, 'run_serial') as serial:
manager.run()
assert serial.call_count == 1
assert manager.using_multiprocessing is False
@mock.patch('flake8.utils.is_windows', return_value=False)
def test_oserrors_are_reraised(is_windows):
"""Verify that unexpected OSErrors will cause the Manager to reraise."""
err = OSError(errno.EAGAIN, 'Ominous message')
style_guide = style_guide_mock()
with mock.patch('_multiprocessing.SemLock', side_effect=err):
with pytest.raises(OSError):
manager = checker.Manager(style_guide, [], [])
with mock.patch.object(manager, 'run_serial') as serial:
manager = _parallel_checker_manager()
with mock.patch.object(manager, 'run_serial') as serial:
with pytest.raises(OSError):
manager.run()
assert serial.call_count == 0