pre-commit/flake8
1
0
Fork 0
mirror of https://github.com/PyCQA/flake8.git synced 2026-03-30 10:46:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'fix-mp-win-3.0' into 'master'

Browse source 
Fix multiprocessing on Windows

*Related to:*  #184 

I'm still running into #179 with this fix but all processes start now at least.

BTW the current design is pretty bad regarding memory footprint and bootstrap time as you have to pickle the manager object for *every* subprocess. So this does not only fix multiprocessing on Windows. It improves the multiprocessing design on other platforms, too.

See merge request !97
This commit is contained in:
Ian Cordasco 2016-07-27 11:43:51 +00:00
commit 846bfafe6c
1 changed files with 11 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/flake8/checker.py
View file

@ -38,6 +38,14 @@ SERIAL_RETRY_ERRNOS = set([
])
def _run_checks_from_queue(process_queue, results_queue, statistics_queue):
LOG.info('Running checks in parallel')
for checker in iter(process_queue.get, 'DONE'):
LOG.info('Checking "%s"', checker.filename)
checker.run_checks(results_queue, statistics_queue)
results_queue.put('DONE')
class Manager(object):
"""Manage the parallelism and checker instances for each plugin and file.
@ -215,13 +223,6 @@ class Manager(object):
)
return reported_results_count
def _run_checks_from_queue(self):
LOG.info('Running checks in parallel')
for checker in iter(self.process_queue.get, 'DONE'):
LOG.info('Checking "%s"', checker.filename)
checker.run_checks(self.results_queue, self.statistics_queue)
self.results_queue.put('DONE')
def is_path_excluded(self, path):
# type: (str) -> bool
"""Check if a path is excluded.
@ -309,7 +310,9 @@ class Manager(object):
LOG.info('Starting %d process workers', self.jobs)
for i in range(self.jobs):
proc = multiprocessing.Process(
target=self._run_checks_from_queue
target=_run_checks_from_queue,
args=(self.process_queue, self.results_queue,
self.statistics_queue)
)
proc.daemon = True
proc.start()