From 89762b9dd17b15e60595b843d0f90dfc0d68f75b Mon Sep 17 00:00:00 2001
From: Ian Cordasco <graffatcolmingov@gmail.com>
Date: Sat, 14 May 2016 19:35:58 -0500
Subject: [PATCH] Document flake8.checker and flake8.processor

---
 docs/source/index.rst            |  1 +
 docs/source/internal/checker.rst | 70 ++++++++++++++++++++++++++++++++
 flake8/processor.py              | 28 ++++++-------
 3 files changed, 85 insertions(+), 14 deletions(-)
 create mode 100644 docs/source/internal/checker.rst

diff --git a/docs/source/index.rst b/docs/source/index.rst
index 4d35951..74810ed 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -27,6 +27,7 @@ Developer Guide
 .. toctree::
     :maxdepth: 2
 
+    internal/checker
     internal/cli
     internal/formatters
     internal/option_handling
diff --git a/docs/source/internal/checker.rst b/docs/source/internal/checker.rst
new file mode 100644
index 0000000..fb5380e
--- /dev/null
+++ b/docs/source/internal/checker.rst
@@ -0,0 +1,70 @@
+===============
+ Check Running
+===============
+
+In Flake8 2.x, Flake8 delegated check running to pep8. In 3.0 we have taken
+that responsibility upon ourselves. This has allowed us to simplify our
+handling of the ``--jobs`` parameter (using :mod:`multiprocessing`) as well as
+simplifying our fallback in the event something goes awry with concurency.
+At the lowest level we have a |FileChecker|. Instances of |FileChecker| are
+created for *each* file to be analyzed by Flake8. Each instance, has a copy of
+all of the plugins registered with setuptools in the ``flake8.extension``
+entry-point group.
+
+The |FileChecker| instances are managed by an instance of |Manager|. The
+|Manager| instance is what handles creating sub-processes with
+:mod:`multiprocessing` module and falling back to running checks in serial if
+an operating system level error arises. When creating |FileChecker| instances,
+the |Manager| is responsible for determining if a particular file has been
+excluded.
+
+
+Processing Files
+----------------
+
+Unfortunately, since Flake8 took over check running from pep8/pycodestyle, it
+also was required to take over parsing and processing files for the checkers
+to use. Since we couldn't reuse pycodestyle's functionality (since it did not
+separate cleanly the processing from check running) we isolated that function
+into the :class:`~flake8.processor.FileProcessor` class. Further, we moved
+several helper functions into the :mod:`flake8.processor` module (see also 
+:ref:`Processor Utility Functions <processor_utility_functions>`).
+
+
+API Reference
+-------------
+
+.. autoclass:: flake8.checker.FileChecker
+    :members:
+
+.. autoclass:: flake8.checker.Manager
+    :members:
+
+.. autoclass:: flake8.processor.FileProcessor
+    :members:
+
+
+.. _processor_utility_functions:
+
+Utility Functions
+`````````````````
+
+.. autofunction:: flake8.processor.count_parentheses
+
+.. autofunction:: flake8.processor.expand_indent
+
+.. autofunction:: flake8.processor.is_eol_token
+
+.. autofunction:: flake8.processor.is_multiline_string
+
+.. autofunction:: flake8.processor.log_token
+
+.. autofunction:: flake8.processor.mutate_string
+
+.. autofunction:: flake8.processor.token_is_comment
+
+.. autofunction:: flake8.processor.token_is_newline
+
+.. Substitutions
+.. |FileChecker| replace:: :class:`~flake8.checker.FileChecker`
+.. |Manager| replace:: :class:`~flake8.checker.Manager`
diff --git a/flake8/processor.py b/flake8/processor.py
index e71784c..ab99089 100644
--- a/flake8/processor.py
+++ b/flake8/processor.py
@@ -30,20 +30,20 @@ class FileProcessor(object):
     to checks expecting that state. Any public attribute on this object can
     be requested by a plugin. The known public attributes are:
 
-    - blank_before
-    - blank_lines
-    - indect_char
-    - indent_level
-    - line_number
-    - logical_line
-    - max_line_length
-    - multiline
-    - noqa
-    - previous_indent_level
-    - previous_logical
-    - tokens
-    - total_lines
-    - verbose
+    - :attr:`blank_before`
+    - :attr:`blank_lines`
+    - :attr:`indect_char`
+    - :attr:`indent_level`
+    - :attr:`line_number`
+    - :attr:`logical_line`
+    - :attr:`max_line_length`
+    - :attr:`multiline`
+    - :attr:`noqa`
+    - :attr:`previous_indent_level`
+    - :attr:`previous_logical`
+    - :attr:`tokens`
+    - :attr:`total_lines`
+    - :attr:`verbose`
     """
 
     NOQA_FILE = re.compile(r'\s*# flake8[:=]\s*noqa', re.I)