mirror of
https://github.com/PyCQA/flake8.git
synced 2026-04-14 16:34:46 +00:00
Simplify result handling
If we start collecting the results in run_parallel *immediately* after starting the worker processes, we do not need another process to handle the results. This also allows us to store all of the results on a the FileChecker class and process results uniformly. This also means we can count the number of errors and warnings in a run and use that to exit appropriately (using SystemExit).
This commit is contained in:
parent
e32476b671
commit
d8635bb92a
2 changed files with 26 additions and 27 deletions
|
|
@ -173,22 +173,6 @@ class Manager(object):
|
||||||
physical_line=physical_line,
|
physical_line=physical_line,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _report_after_parallel(self):
|
|
||||||
final_results = {}
|
|
||||||
for (filename, results) in self._results():
|
|
||||||
final_results[filename] = results
|
|
||||||
|
|
||||||
for checker in self.checkers:
|
|
||||||
filename = checker.filename
|
|
||||||
results = sorted(final_results.get(filename, []),
|
|
||||||
key=lambda tup: (tup[1], tup[2]))
|
|
||||||
self._handle_results(filename, results)
|
|
||||||
|
|
||||||
def _report_after_serial(self):
|
|
||||||
for checker in self.checkers:
|
|
||||||
results = sorted(checker.results, key=lambda tup: (tup[2], tup[3]))
|
|
||||||
self._handle_results(checker.filename, results)
|
|
||||||
|
|
||||||
def _run_checks_from_queue(self):
|
def _run_checks_from_queue(self):
|
||||||
LOG.info('Running checks in parallel')
|
LOG.info('Running checks in parallel')
|
||||||
for checker in iter(self.process_queue.get, 'DONE'):
|
for checker in iter(self.process_queue.get, 'DONE'):
|
||||||
|
|
@ -242,12 +226,16 @@ class Manager(object):
|
||||||
This iterates over each of the checkers and reports the errors sorted
|
This iterates over each of the checkers and reports the errors sorted
|
||||||
by line number.
|
by line number.
|
||||||
"""
|
"""
|
||||||
if not self.using_multiprocessing:
|
results_found = 0
|
||||||
self._report_after_serial()
|
for checker in self.checkers:
|
||||||
|
results = sorted(checker.results, key=lambda tup: (tup[2], tup[3]))
|
||||||
|
self._handle_results(checker.filename, results)
|
||||||
|
results_found += len(results)
|
||||||
|
return results_found
|
||||||
|
|
||||||
def run_parallel(self):
|
def run_parallel(self):
|
||||||
"""Run the checkers in parallel."""
|
"""Run the checkers in parallel."""
|
||||||
LOG.info('Starting %d process workers', self.jobs - 1)
|
LOG.info('Starting %d process workers', self.jobs)
|
||||||
for i in range(self.jobs):
|
for i in range(self.jobs):
|
||||||
proc = multiprocessing.Process(
|
proc = multiprocessing.Process(
|
||||||
target=self._run_checks_from_queue
|
target=self._run_checks_from_queue
|
||||||
|
|
@ -256,10 +244,14 @@ class Manager(object):
|
||||||
proc.start()
|
proc.start()
|
||||||
self.processes.append(proc)
|
self.processes.append(proc)
|
||||||
|
|
||||||
proc = multiprocessing.Process(target=self._report_after_parallel)
|
final_results = {}
|
||||||
proc.start()
|
for (filename, results) in self._results():
|
||||||
LOG.info('Started process to report errors')
|
final_results[filename] = results
|
||||||
self.processes.append(proc)
|
|
||||||
|
for checker in self.checkers:
|
||||||
|
filename = checker.filename
|
||||||
|
checker.results = sorted(final_results.get(filename, []),
|
||||||
|
key=lambda tup: (tup[1], tup[2]))
|
||||||
|
|
||||||
def run_serial(self):
|
def run_serial(self):
|
||||||
"""Run the checkers in serial."""
|
"""Run the checkers in serial."""
|
||||||
|
|
@ -299,11 +291,11 @@ class Manager(object):
|
||||||
for checker in self.checkers:
|
for checker in self.checkers:
|
||||||
self.process_queue.put(checker)
|
self.process_queue.put(checker)
|
||||||
|
|
||||||
def stop(self):
|
|
||||||
"""Stop checking files."""
|
|
||||||
for i in range(self.jobs):
|
for i in range(self.jobs):
|
||||||
self.process_queue.put('DONE')
|
self.process_queue.put('DONE')
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""Stop checking files."""
|
||||||
for proc in self.processes:
|
for proc in self.processes:
|
||||||
LOG.info('Joining %s to the main process', proc.name)
|
LOG.info('Joining %s to the main process', proc.name)
|
||||||
proc.join()
|
proc.join()
|
||||||
|
|
|
||||||
|
|
@ -195,6 +195,7 @@ class Application(object):
|
||||||
|
|
||||||
self.options = None
|
self.options = None
|
||||||
self.args = None
|
self.args = None
|
||||||
|
self.result_count = 0
|
||||||
|
|
||||||
def find_plugins(self):
|
def find_plugins(self):
|
||||||
# type: () -> NoneType
|
# type: () -> NoneType
|
||||||
|
|
@ -274,7 +275,7 @@ class Application(object):
|
||||||
# type: () -> NoneType
|
# type: () -> NoneType
|
||||||
"""Report all the errors found by flake8 3.0."""
|
"""Report all the errors found by flake8 3.0."""
|
||||||
LOG.info('Reporting errors')
|
LOG.info('Reporting errors')
|
||||||
self.file_checker_manager.report()
|
self.result_count = self.file_checker_manager.report()
|
||||||
|
|
||||||
def _run(self, argv):
|
def _run(self, argv):
|
||||||
self.find_plugins()
|
self.find_plugins()
|
||||||
|
|
@ -289,7 +290,12 @@ class Application(object):
|
||||||
|
|
||||||
def run(self, argv=None):
|
def run(self, argv=None):
|
||||||
# type: (Union[NoneType, List[str]]) -> NoneType
|
# type: (Union[NoneType, List[str]]) -> NoneType
|
||||||
"""Run our application."""
|
"""Run our application.
|
||||||
|
|
||||||
|
This method will also handle KeyboardInterrupt exceptions for the
|
||||||
|
entirety of the flake8 application. If it sees a KeyboardInterrupt it
|
||||||
|
will forcibly clean up the :class:`~flake8.checker.Manager`.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
self._run(argv)
|
self._run(argv)
|
||||||
except KeyboardInterrupt as exc:
|
except KeyboardInterrupt as exc:
|
||||||
|
|
@ -303,3 +309,4 @@ def main(argv=None):
|
||||||
"""Main entry-point for the flake8 command-line tool."""
|
"""Main entry-point for the flake8 command-line tool."""
|
||||||
app = Application()
|
app = Application()
|
||||||
app.run(argv)
|
app.run(argv)
|
||||||
|
raise SystemExit(app.result_count > 0)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue