Merge branch 'bug/179' into 'master'

Serialize Checkers PluginTypeManager to a dict

*Description of changes*

Try to side-step issues with attributes not being set/updated on plugins when used with multiprocessing and Queues.

*Related to:*  #179, #164 

See merge request !98

.appveyor.yml

@@ -5,4 +5,4 @@ install:
 build: off
 
 test_script:
-  - python -m tox -e py27,py33,py34,py35
+  - python -m tox -e py27,py33,py34,py35,dogfood

src/flake8/checker.py

@@ -38,21 +38,6 @@ SERIAL_RETRY_ERRNOS = set([
 ])
 
 
-def _run_checks_from_queue(process_queue, results_queue, statistics_queue):
-    LOG.info('Running checks in parallel')
-    try:
-        for checker in iter(process_queue.get, 'DONE'):
-            LOG.info('Checking "%s"', checker.filename)
-            checker.run_checks(results_queue, statistics_queue)
-    except exceptions.PluginRequestedUnknownParameters as exc:
-        print(str(exc))
-    except Exception as exc:
-        LOG.error('Unhandled exception occurred')
-        raise
-    finally:
-        results_queue.put('DONE')
-
-
 class Manager(object):
     """Manage the parallelism and checker instances for each plugin and file.
 
@@ -284,8 +269,9 @@ class Manager(object):
             file_exists = os.path.exists(filename)
             return (file_exists and matches_filename_patterns) or is_stdin
 
+        checks = self.checks.to_dictionary()
         self.checkers = [
-            FileChecker(filename, self.checks, self.options)
+            FileChecker(filename, checks, self.options)
             for argument in paths
             for filename in utils.filenames_from(argument,
                                                  self.is_path_excluded)
@@ -405,7 +391,7 @@ class FileChecker(object):
         :param checks:
             The plugins registered to check the file.
         :type checks:
-            flake8.plugins.manager.Checkers
+            dict
         :param options:
             Parsed option values from config and command-line.
         :type options:
@@ -458,14 +444,17 @@ class FileChecker(object):
         """Run the check in a single plugin."""
         LOG.debug('Running %r with %r', plugin, arguments)
         try:
-            self.processor.keyword_arguments_for(plugin.parameters, arguments)
+            self.processor.keyword_arguments_for(
+                plugin['parameters'],
+                arguments,
+            )
         except AttributeError as ae:
             LOG.error('Plugin requested unknown parameters.')
             raise exceptions.PluginRequestedUnknownParameters(
                 plugin=plugin,
                 exception=ae,
             )
-        return plugin.execute(**arguments)
+        return plugin['plugin'](**arguments)
 
     def run_ast_checks(self):
         """Run all checks expecting an abstract syntax tree."""
@@ -484,7 +473,7 @@ class FileChecker(object):
                         (exc_type.__name__, exception.args[0]))
             return
 
-        for plugin in self.checks.ast_plugins:
+        for plugin in self.checks['ast_plugins']:
             checker = self.run_check(plugin, tree=ast)
             # If the plugin uses a class, call the run method of it, otherwise
             # the call should return something iterable itself
@@ -509,7 +498,7 @@ class FileChecker(object):
 
         LOG.debug('Logical line: "%s"', logical_line.rstrip())
 
-        for plugin in self.checks.logical_line_plugins:
+        for plugin in self.checks['logical_line_plugins']:
             self.processor.update_checker_state_for(plugin)
             results = self.run_check(plugin, logical_line=logical_line) or ()
             for offset, text in results:
@@ -526,7 +515,7 @@ class FileChecker(object):
 
     def run_physical_checks(self, physical_line, override_error_line=None):
         """Run all checks for a given physical line."""
-        for plugin in self.checks.physical_line_plugins:
+        for plugin in self.checks['physical_line_plugins']:
             self.processor.update_checker_state_for(plugin)
             result = self.run_check(plugin, physical_line=physical_line)
             if result is not None:
@@ -636,6 +625,21 @@ class FileChecker(object):
                                              override_error_line=token[4])
 
 
+def _run_checks_from_queue(process_queue, results_queue, statistics_queue):
+    LOG.info('Running checks in parallel')
+    try:
+        for checker in iter(process_queue.get, 'DONE'):
+            LOG.info('Checking "%s"', checker.filename)
+            checker.run_checks(results_queue, statistics_queue)
+    except exceptions.PluginRequestedUnknownParameters as exc:
+        print(str(exc))
+    except Exception as exc:
+        LOG.error('Unhandled exception occurred')
+        raise
+    finally:
+        results_queue.put('DONE')
+
+
 def find_offset(offset, mapping):
     """Find the offset tuple for a single offset."""
     if isinstance(offset, tuple):

src/flake8/exceptions.py

@@ -65,7 +65,7 @@ class PluginRequestedUnknownParameters(Flake8Exception):
 
     def __str__(self):
         """Format our exception message."""
-        return self.FORMAT % {'name': self.plugin.plugin_name,
+        return self.FORMAT % {'name': self.plugin['plugin_name'],
                               'exc': self.original_exception}
 
 

src/flake8/plugins/manager.py

@@ -49,6 +49,16 @@ class Plugin(object):
             self.name, self.entry_point
         )
 
+    def to_dictionary(self):
+        """Convert this plugin to a dictionary."""
+        return {
+            'name': self.name,
+            'parameters': self.parameters,
+            'parameter_names': self.parameter_names,
+            'plugin': self.plugin,
+            'plugin_name': self.plugin_name,
+        }
+
     def is_in_a_group(self):
         """Determine if this plugin is in a group.
 
@@ -433,6 +443,20 @@ class Checkers(PluginTypeManager):
             if argument_name == plugin.parameter_names[0]:
                 yield plugin
 
+    def to_dictionary(self):
+        """Return a dictionary of AST and line-based plugins."""
+        return {
+            'ast_plugins': [
+                plugin.to_dictionary() for plugin in self.ast_plugins
+            ],
+            'logical_line_plugins': [
+                plugin.to_dictionary() for plugin in self.logical_line_plugins
+            ],
+            'physical_line_plugins': [
+                plugin.to_dictionary() for plugin in self.physical_line_plugins
+            ],
+        }
+
     def register_options(self, optmanager):
         """Register all of the checkers' options to the OptionManager.
 

src/flake8/plugins/pyflakes.py

@@ -46,6 +46,9 @@ class FlakesChecker(pyflakes.checker.Checker):
 
     name = 'pyflakes'
     version = pyflakes.__version__
+    with_doctest = False
+    include_in_doctest = []
+    exclude_from_doctest = []
 
     def __init__(self, tree, filename):
         """Initialize the PyFlakes plugin with an AST tree and filename."""

src/flake8/processor.py

@@ -129,9 +129,9 @@ class FileProcessor(object):
 
     def update_checker_state_for(self, plugin):
         """Update the checker_state attribute for the plugin."""
-        if 'checker_state' in plugin.parameters:
+        if 'checker_state' in plugin['parameters']:
             self.checker_state = self._checker_states.setdefault(
-                plugin.name, {}
+                plugin['name'], {}
             )
 
     def next_logical_line(self):

tests/integration/test_checker.py

@@ -63,7 +63,11 @@ def test_handle_file_plugins(plugin_target):
     # Prevent it from reading lines from stdin or somewhere else
     with mock.patch('flake8.processor.FileProcessor.read_lines',
                     return_value=['Line 1']):
-        file_checker = checker.FileChecker('-', checks, mock.MagicMock())
+        file_checker = checker.FileChecker(
+            '-',
+            checks.to_dictionary(),
+            mock.MagicMock()
+        )
 
     # Do not actually build an AST
     file_checker.processor.build_ast = lambda: True

tests/unit/test_checker_manager.py

@@ -47,8 +47,14 @@ def test_make_checkers():
     """Verify that we create a list of FileChecker instances."""
     style_guide = style_guide_mock()
     files = ['file1', 'file2']
+    checkplugins = mock.Mock()
+    checkplugins.to_dictionary.return_value = {
+        'ast_plugins': [],
+        'logical_line_plugins': [],
+        'physical_line_plugins': [],
+    }
     with mock.patch('flake8.checker.multiprocessing', None):
-        manager = checker.Manager(style_guide, files, [])
+        manager = checker.Manager(style_guide, files, checkplugins)
 
     with mock.patch('flake8.utils.filenames_from') as filenames_from:
         filenames_from.side_effect = [['file1'], ['file2']]

tox.ini

@@ -16,7 +16,7 @@ commands = {posargs}
 
 # Dogfood our current mastera version
 [testenv:dogfood]
-basepython = python3
+basepython = python3.5
 skip_install = true
 deps =
     wheel