Merge remote-tracking branch 'rewrite/master' into proposed/3.0

docs/source/conf.py

@@ -0,0 +1,295 @@
+# -*- coding: utf-8 -*-
+#
+# flake8 documentation build configuration file, created by
+# sphinx-quickstart on Tue Jan 19 07:14:10 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+needs_sphinx = '1.3'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'flake8'
+copyright = u'2016, Ian Cordasco'
+author = u'Ian Cordasco'
+
+import flake8
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = flake8.__version__
+# The full version, including alpha/beta/rc tags.
+release = flake8.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
+#html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# Now only 'ja' uses this config value
+#html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'flake8doc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+
+# Latex figure (float) alignment
+#'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'flake8.tex', u'flake8 Documentation',
+     u'Ian Cordasco', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'flake8', u'flake8 Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'flake8', u'flake8 Documentation',
+     author, 'flake8', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'python': ('https://docs.python.org/3.4', None)}

docs/source/dev/formatters.rst

@@ -0,0 +1,51 @@
+===========================================
+ Developing a Formatting Plugin for Flake8
+===========================================
+
+Flake8 added the ability to develop custom formatting plugins in version
+3.0.0. Let's write a plugin together:
+
+.. code-block:: python
+
+    from flake8.formatting import base
+
+
+    class Example(base.BaseFormatter):
+        """Flake8's example formatter."""
+
+        pass
+
+We notice, as soon as we start, that we inherit from Flake8's
+:class:`~flake8.formatting.base.BaseFormatter` class. If we follow the
+:ref:`instructions to register a plugin <register-a-plugin>` and try to use
+our example formatter, e.g., ``flake8 --format=example`` then Flake8 will fail
+because we did not implement the ``format`` method. Let's do that next.
+
+.. code-block:: python
+
+    class Example(base.BaseFormatter):
+        """Flake8's example formatter."""
+
+        def format(self, error):
+            return 'Example formatter: {0!r}'.format(error)
+
+With that we're done. Obviously this isn't a very useful formatter, but it
+should highlight the simplicitly of creating a formatter with Flake8. If we
+wanted to instead create a formatter that aggregated the results and returned
+XML, JSON, or subunit we could also do that. Flake8 interacts with the
+formatter in two ways:
+
+#. It creates the formatter and provides it the options parsed from the
+   configuration files and command-line
+
+#. It uses the instance of the formatter and calls ``handle`` with the error.
+
+By default :meth:`flake8.formatting.base.BaseFormatter.handle` simply calls
+the ``format`` method and then ``write``. Any extra handling you wish to do
+for formatting purposes should override the ``handle`` method.
+
+API Documentation
+=================
+
+.. autoclass:: flake8.formatting.base.BaseFormatter
+    :members:

docs/source/dev/registering_plugins.rst

@@ -0,0 +1,115 @@
+.. _register-a-plugin:
+
+==================================
+ Registering a Plugin with Flake8
+==================================
+
+To register any kind of plugin with Flake8, you need a few things:
+
+#. You need a way to install the plugin (whether it is packaged on its own or
+   as part of something else). In this section, we will use a ``setup.py``
+   written for an example plugin.
+
+#. A name for your plugin that will (ideally) be unique.
+
+#. A somewhat recent version of setuptools (newer than 0.7.0 but preferably as
+   recent as you can attain).
+
+Flake8 presently relies on a functionality provided by setuptools called
+`Entry Points`_. These allow any package to register a plugin with Flake8 via
+that package's ``setup.py`` file.
+
+Let's presume that we already have our plugin written and it's in a module
+called ``flake8_example``. We might have a ``setup.py`` that looks something
+like:
+
+.. code-block:: python
+
+    from __future__ import with_statement
+    import setuptools
+
+    requires = [
+        "flake8 > 3.0.0",
+    ]
+
+    flake8_entry_point = # ...
+
+    setuptools.setup(
+        name="flake8_example",
+        license="MIT",
+        version="0.1.0",
+        description="our extension to flake8",
+        author="Me",
+        author_email="example@example.com",
+        url="https://gitlab.com/me/flake8_example",
+        packages=[
+            "flake8_example",
+        ],
+        install_requires=requires,
+        entry_points={
+            flake8_entry_point: [
+                'X = flake8_example:ExamplePlugin',
+            ],
+        },
+        classifiers=[
+            "Environment :: Console",
+            "Intended Audience :: Developers",
+            "License :: OSI Approved :: MIT License",
+            "Programming Language :: Python",
+            "Programming Language :: Python :: 2",
+            "Programming Language :: Python :: 3",
+            "Topic :: Software Development :: Libraries :: Python Modules",
+            "Topic :: Software Development :: Quality Assurance",
+        ],
+    )
+
+Note specifically these lines:
+
+.. code-block:: python
+
+    flake8_entry_point = # ...
+
+    setuptools.setup(
+        # snip ...
+        entry_points={
+            flake8_entry_point: [
+                'X = flake8_example:ExamplePlugin',
+            ],
+        },
+        # snip ...
+    )
+
+We tell setuptools to register our entry point "X" inside the specific
+grouping of entry-points that flake8 should look in.
+
+Flake8 presently looks at three groups:
+
+- ``flake8.extension``
+
+- ``flake8.listen``
+
+- ``flake8.report``
+
+If your plugin is one that adds checks to Flake8, you will use
+``flake8.extension``. If your plugin automatically fixes errors in code, you
+will use ``flake8.listen``.  Finally, if your plugin performs extra report
+handling (formatting, filtering, etc.) it will use ``flake8.report``.
+
+If our ``ExamplePlugin`` is something that adds checks, our code would look
+like:
+
+.. code-block:: python
+
+    setuptools.setup(
+        # snip ...
+        entry_points={
+            'flake8.extension': [
+                'X = flake8_example:ExamplePlugin',
+            ],
+        },
+        # snip ...
+    )
+
+
+.. _Entry Points:
+    https://pythonhosted.org/setuptools/pkg_resources.html#entry-points

docs/source/index.rst

@@ -0,0 +1,40 @@
+.. flake8 documentation master file, created by
+   sphinx-quickstart on Tue Jan 19 07:14:10 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Flake8: Your Tool For Style Guide Enforcement
+=============================================
+
+User Guide
+----------
+
+.. toctree::
+    :maxdepth: 2
+
+Plugin Developer Guide
+----------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    dev/formatters
+    dev/registering_plugins
+
+Developer Guide
+---------------
+
+.. toctree::
+    :maxdepth: 2
+
+    internal/formatters
+    internal/option_handling
+    internal/plugin_handling
+    internal/utils
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/internal/formatters.rst

@@ -0,0 +1,47 @@
+=====================
+ Built-in Formatters
+=====================
+
+By default Flake8 has two formatters built-in, ``default`` and ``pylint``.
+These correspond to two classes |DefaultFormatter| and |PylintFormatter|.
+
+In Flake8 2.0, pep8 handled formatting of errors and also allowed users to
+specify an arbitrary format string as a parameter to ``--format``. In order
+to allow for this backwards compatibility, Flake8 3.0 made two choices:
+
+#. To not limit a user's choices for ``--format`` to the format class names
+
+#. To make the default formatter attempt to use the string provided by the
+   user if it cannot find a formatter with that name.
+
+Default Formatter
+=================
+
+The |DefaultFormatter| continues to use the same default format string as
+pep8: ``'%(path)s:%(row)d:%(col)d: %(code)s %(text)s'``.
+
+In order to provide the default functionality it overrides two methods:
+
+#. ``after_init``
+
+#. ``format``
+
+The former allows us to inspect the value provided to ``--format`` by the
+user and alter our own format based on that value. The second simply uses
+that format string to format the error.
+
+.. autoclass:: flake8.formatting.default.Default
+    :members:
+
+Pylint Formatter
+================
+
+The |PylintFormatter| simply defines the default Pylint format string from
+pep8: ``'%(path)s:%(row)d: [%(code)s] %(text)s'``.
+
+.. autoclass:: flake8.formatting.default.Pylint
+    :members:
+
+
+.. |DefaultFormatter| replace:: :class:`~flake8.formatting.default.Default`
+.. |PylintFormatter| replace:: :class:`~flake8.formatting.default.Pylint`

docs/source/internal/option_handling.rst

@@ -0,0 +1,225 @@
+Option and Configuration Handling
+=================================
+
+Option Management
+-----------------
+
+Command-line options are often also set in configuration files for Flake8.
+While not all options are meant to be parsed from configuration files, many
+default options are also parsed from configuration files as are most plugin
+options.
+
+In Flake8 2, plugins received a :class:`optparse.OptionParser` instance and
+called :meth:`optparse.OptionParser.add_option` to register options. If the
+plugin author also wanted to have that option parsed from config files they
+also had to do something like:
+
+.. code-block:: python
+
+    parser.config_options.append('my_config_option')
+    parser.config_options.extend(['config_opt1', 'config_opt2'])
+
+This was previously undocumented and led to a lot of confusion as to why
+registered options were not automatically parsed from configuration files.
+
+Since Flake8 3 was rewritten from scratch, we decided to take a different
+approach to configuration file parsing. Instead of needing to know about an
+undocumented attribute that pep8 looks for, Flake8 3 now accepts a parameter
+to ``add_option``, specifically ``parse_from_config`` which is a boolean
+value.
+
+Flake8 does this by creating its own abstractions on top of :mod:`optparse`.
+The first abstraction is the :class:`flake8.options.manager.Option` class. The
+second is the :class:`flake8.options.manager.OptionManager`. In fact, we add
+three new parameters:
+
+- ``parse_from_config``
+
+- ``comma_separated_list``
+
+- ``normalize_paths``
+
+The last two are not specifically for configuration file handling, but they
+do improve that dramatically. We found that there were options that when
+specified in a configuration file, lended themselves to being split across
+multiple lines and those options were almost always comma-separated. For
+example, let's consider a user's list of ignored error codes for a project:
+
+.. code-block:: ini
+
+    [flake8]
+    ignore =
+        E111,  # Reasoning
+        E711,  # Reasoning
+        E712,  # Reasoning
+        E121,  # Reasoning
+        E122,  # Reasoning
+        E123,  # Reasoning
+        E131,  # Reasoning
+        E251   # Reasoning
+
+It makes sense here to allow users to specify the value this way, but, the
+standard libary's :class:`configparser.RawConfigParser` class does returns a
+string that looks like
+
+.. code-block:: python
+
+    "\nE111,  \nE711,  \nE712,  \nE121,  \nE122,  \nE123,  \nE131,  \nE251  "
+
+This means that a typical call to :meth:`str.split` with ``','`` will not be
+sufficient here. Telling Flake8 that something is a comma-separated list
+(e.g., ``comma_separated_list=True``) will handle this for you. Flake8 will
+return:
+
+.. code-block:: python
+
+    ["E111", "E711", "E712", "E121", "E122", "E123", "E131", "E251"]
+
+Next let's look at how users might like to specify their ``exclude`` list.
+Presently OpenStack's Nova project has this line in their `tox.ini`_:
+
+.. code-block:: ini
+
+    exclude = .venv,.git,.tox,dist,doc,*openstack/common/*,*lib/python*,*egg,build,tools/xenserver*,releasenotes
+
+I think we can all agree that this would be easier to read like this:
+
+.. code-block:: ini
+
+    exclude =
+        .venv,
+        .git,
+        .tox,
+        dist,
+        doc,
+        *openstack/common/*,
+        *lib/python*,
+        *egg,
+        build,
+        tools/xenserver*,
+        releasenotes
+
+In this case, since these are actually intended to be paths, we would specify
+both ``comma_separated_list=True`` and ``normalize_paths=True`` because we
+want the paths to be provided to us with some consistency (either all absolute
+paths or not).
+
+Now let's look at how this would actually be utilized. Most plugin developers
+will receive an instance of :class:`~flake8.options.manager.OptionManager` so
+to ease the transition we kept the same API as the
+:class:`optparse.OptionParser` object. The only difference is that
+:meth:`~flake8.options.manager.OptionManager.add_option` accepts the three
+extra arguments we highlighted above.
+
+.. _tox.ini:
+    https://github.com/openstack/nova/blob/3eb190c4cfc0eefddac6c2cc1b94a699fb1687f8/tox.ini#L155
+
+Configuration File Management
+-----------------------------
+
+In Flake8 2, configuration file discovery and management was handled by pep8.
+In pep8's 1.6 release series, it drastically broke how discovery and merging
+worked (as a result of trying to improve it). To avoid a dependency breaking
+Flake8 again in the future, we have created our own discovery and management.
+As part of managing this ourselves, we decided to change management/discovery
+for 3.0.0. We have done the following:
+
+- User files (files stored in a user's home directory or in the XDG directory
+  inside their home directory) are the first files read. For example, if the
+  user has a ``~/.flake8`` file, we will read that first.
+
+- Project files (files stored in the current directory) are read next and
+  merged on top of the user file. In other words, configuration in project
+  files takes precedence over configuration in user files.
+
+- **New in 3.0.0** The user can specify ``--append-config <path-to-file>``
+  repeatedly to include extra configuration files that should be read and
+  take precedence over user and project files.
+
+- **New in 3.0.0** The user can specify ``--config <path-to-file>`` to so this
+  file is the only configuration file used. This is a change from Flake8 2
+  where pep8 would simply merge this configuration file into the configuration
+  generated by user and project files (where this takes precedence).
+
+- **New in 3.0.0** The user can specify ``--isolated`` to disable
+  configuration via discovered configuration files.
+
+To facilitate the configuration file management, we've taken a different
+approach to discovery and management of files than pep8. In pep8 1.5, 1.6, and
+1.7 configuration discovery and management was centralized in `66 lines of
+very terse python`_ which was confusing and not very explicit. The terseness
+of this function (Flake8's authors believe) caused the confusion and problems
+with pep8's 1.6 series. As such, Flake8 has separated out discovery,
+management, and merging into a module to make reasoning about each of these
+pieces easier and more explicit (as well as easier to test).
+
+Configuration file discovery is managed by the
+:class:`~flake8.options.config.ConfigFileFinder` object. This object needs to
+know information about the program's name, any extra arguments passed to it,
+and any configuration files that should be appended to the list of discovered
+files. It provides methods for finding the files and similiar methods for
+parsing those fles. For example, it provides
+:meth:`~flake8.options.config.ConfigFileFinder.local_config_files` to find
+known local config files (and append the extra configuration files) and it
+also provides :meth:`~flake8.options.config.ConfigFileFinder.local_configs`
+to parse those configuration files.
+
+.. note:: ``local_config_files`` also filters out non-existent files.
+
+Configuration file merging and managemnt is controlled by the
+:class:`~flake8.options.config.MergedConfigParser`. This requires the instance
+of :class:`~flake8.options.manager.OptionManager` that the program is using,
+the list of appended config files, and the list of extra arguments. This
+object is currently the sole user of the
+:class:`~flake8.options.config.ConfigFileFinder` object. It appropriately
+initializes the object and uses it in each of
+
+- :meth:`~flake8.options.config.MergedConfigParser.parse_cli_config`
+- :meth:`~flake8.options.config.MergedConfigParser.parse_local_config`
+- :meth:`~flake8.options.config.MergedConfigParser.parse_user_config`
+
+Finally,
+:meth:`~flake8.options.config.MergedConfigParser.merge_user_and_local_config`
+takes the user and local configuration files that are parsed by
+:meth:`~flake8.options.config.MergedConfigParser.parse_local_config` and
+:meth:`~flake8.options.config.MergedConfigParser.parse_user_config`. The
+main usage of the ``MergedConfigParser`` is in
+:func:`~flake8.options.aggregator.aggregate_options`.
+
+Aggregating Configuration File and Command Line Arguments
+---------------------------------------------------------
+
+:func:`~flake8.options.aggregator.aggregate_options` accepts an instance of
+:class:`~flake8.options.maanger.OptionManager` and does the work to parse the
+command-line arguments passed by the user necessary for creating an instance
+of :class:`~flake8.options.config.MergedConfigParser`.
+
+After parsing the configuration file, we determine the default ignore list. We
+use the defaults from the OptionManager and update those with the parsed
+configuration files. Finally we parse the user-provided options one last time
+using the option defaults and configuration file values as defaults. The
+parser merges on the command-line specified arguments for us so we have our
+final, definitive, aggregated options.
+
+.. _66 lines of very terse python:
+    https://github.com/PyCQA/pep8/blob/b8088a2b6bc5b76bece174efad877f764529bc74/pep8.py#L1981..L2047
+
+API Documentation
+-----------------
+
+.. autofunction:: flake8.options.aggregator.aggregate_options
+
+.. autoclass:: flake8.options.manager.Option
+    :members: __init__, normalize, to_optparse
+
+.. autoclass:: flake8.options.manager.OptionManager
+    :members:
+    :special-members:
+
+.. autoclass:: flake8.options.config.ConfigFileFinder
+    :members:
+    :special-members:
+
+.. autoclass:: flake8.options.config.MergedConfigParser
+    :members:
+    :special-members:

docs/source/internal/plugin_handling.rst

@@ -0,0 +1,114 @@
+Plugin Handling
+===============
+
+Plugin Management
+-----------------
+
+Flake8 3.0 added support for two other plugins besides those which define new
+checks. It now supports:
+
+- extra checks
+
+- alternative report formatters
+
+- listeners to auto-correct violations of checks
+
+To facilitate this, Flake8 needed a more mature way of managing plugins. As
+such, we developed the |PluginManager| which accepts a namespace and will load
+the plugins for that namespace. A |PluginManager| creates and manages many
+|Plugin| instances.
+
+A |Plugin| lazily loads the underlying entry-point provided by setuptools.
+The entry-point will be loaded either by calling
+:meth:`~flake8.plugins.manager.Plugin.load_plugin` or accessing the ``plugin``
+attribute. We also use this abstraction to retrieve options that the plugin
+wishes to register and parse.
+
+The only public method that the |PluginManager| provides is
+:meth:`~flake8.plugins.manager.PluginManager.map`. This will accept a function
+(or other callable) and call it with each plugin as the first parameter.
+
+We build atop the |PluginManager| with the |PTM|. It is expected that users of
+the |PTM| will subclass it and specify the ``namespace``, e.g.,
+
+.. code-block:: python
+
+    class ExamplePluginType(flake8.plugin.manager.PluginTypeManager):
+        namespace = 'example-plugins'
+
+This provides a few extra methods via the |PluginManager|'s ``map`` method.
+
+Finally, we create three classes of plugins:
+
+- :class:`~flake8.plugins.manager.Checkers`
+
+- :class:`~flake8.plugins.manager.Listeners`
+
+- :class:`~flake8.plugins.manager.ReportFormatters`
+
+These are used to interact with each of the types of plugins individually.
+
+.. note::
+
+    Our inspiration for our plugin handling comes from the author's extensive
+    experience with ``stevedore``.
+
+Notifying Listener Plugins
+--------------------------
+
+One of the interesting challenges with allowing plugins to be notified each
+time an error or warning is emitted by a checker is finding listeners quickly
+and efficiently. It makes sense to allow a listener to listen for a certain
+class of warnings or just a specific warning. As such, we need to allow all
+plugins that listen to a specific warning or class to be notified. For
+example, someone might register a listener for ``E1`` and another for ``E111``
+if ``E111`` is triggered by the code, both listeners should be notified.
+If ``E112`` is returned, then only ``E1`` (and any other listeners) would be
+notified.
+
+To implement this goal, we needed an object to store listeners in that would
+allow for efficient look up - a Trie (or Prefix Tree). Given that none of the
+existing packages on PyPI allowed for storing data on each node of the trie,
+it was left up to write our own as :class:`~flake8.plugins._trie.Trie`. On
+top of that we layer our :class:`~flake8.plugins.notifier.Notifier` class.
+
+Now when Flake8 receives an error or warning, we can easily call the
+:meth:`~flake8.plugins.notifier.Notifier.notify` method and let plugins act on
+that knowledge.
+
+Default Plugins
+---------------
+
+Finally, Flake8 has always provided its own plugin shim for Pyflakes. As part
+of that we carry our own shim in-tree and now store that in
+:mod:`flake8.plugins.pyflakes`.
+
+API Documentation
+-----------------
+
+.. autoclass:: flake8.plugins.manager.PluginManager
+    :members:
+    :special-members: __init__, __contains__, __getitem__
+
+.. autoclass:: flake8.plugins.manager.Plugin
+    :members:
+    :special-members: __init__
+
+.. autoclass:: flake8.plugins.manager.PluginTypeManager
+    :members:
+
+.. autoclass:: flake8.plugins.manager.Checkers
+    :members:
+
+.. autoclass:: flake8.plugins.manager.Listeners
+    :members: build_notifier
+
+.. autoclass:: flake8.plugins.manager.ReportFormatters
+
+.. autoclass:: flake8.plugins.notifier.Notifier
+
+.. autoclass:: flake8.plugins._trie.Trie
+
+.. |PluginManager| replace:: :class:`~flake8.plugins.manager.PluginManager`
+.. |Plugin| replace:: :class:`~flake8.plugins.manager.Plugin`
+.. |PTM| replace:: :class:`~flake8.plugins.manager.PluginTypeManager`

docs/source/internal/utils.rst

@@ -0,0 +1,100 @@
+===================
+ Utility Functions
+===================
+
+Flake8 has a few utility functions that it uses and provides to plugins.
+
+.. autofunction:: flake8.utils.parse_comma_separated_list
+
+:func:`~flake8.utils.parse_comma_separated_list` takes either a string like
+
+.. code-block:: python
+
+    "E121,W123,F904"
+    "E121,\nW123,\nF804"
+    "E121,\n\tW123,\n\tF804"
+
+Or it will take a list of strings (potentially with whitespace) such as
+
+.. code-block:: python
+
+    ["   E121\n", "\t\nW123   ", "\n\tF904\n    "]
+
+And converts it to a list that looks as follows
+
+.. code-block:: python
+
+    ["E121", "W123", "F904"]
+
+This function helps normalize any kind of comma-separated input you or Flake8
+might receive. This is most helpful when taking advantage of Flake8's
+additional parameters to :class:`~flake8.options.manager.Option`.
+
+.. autofunction:: flake8.utils.normalize_path
+
+This utility takes a string that represents a path and returns the absolute
+path if the string has a ``/`` in it. It also removes trailing ``/``\ s.
+
+.. autofunction:: flake8.utils.normalize_paths
+
+This function utilizes :func:`~flake8.utils.parse_comma_separated_list` and
+:func:`~flake8.utils.normalize_path` to normalize it's input to a list of
+strings that should be paths.
+
+.. autofunction:: flake8.utils.stdin_get_value
+
+This function retrieves and caches the value provided on ``sys.stdin``. This
+allows plugins to use this to retrieve ``stdin`` if necessary.
+
+.. autofunction:: flake8.utils.is_windows
+
+This provides a convenient and explicitly named function that checks if we are
+currently running on a Windows (or ``nt``) operating system.
+
+.. autofunction:: flake8.utils.is_using_stdin
+
+Another helpful function that is named only to be explicit given it is a very
+trivial check, this checks if the user specified ``-`` in their arguments to
+Flake8 to indicate we should read from stdin.
+
+.. autofunction:: flake8.utils.filenames_from
+
+When provided an argument to Flake8, we need to be able to traverse
+directories in a convenient manner. For example, if someone runs
+
+.. code::
+
+    $ flake8 flake8/
+
+Then they want us to check all of the files in the directory ``flake8/``. This
+function will handle that while also handling the case where they specify a
+file like:
+
+.. code::
+
+    $ flake8 flake8/__init__.py
+
+
+.. autofunction:: flake8.utils.fnmatch
+
+The standard library's :func:`fnmatch.fnmatch` is excellent at deciding if a
+filename matches a single pattern. In our use case, however, we typically have
+a list of patterns and want to know if the filename matches any of them. This
+function abstracts that logic away with a little extra logic.
+
+.. autofunction:: flake8.utils.parameters_for
+
+Flake8 analyzes the parameters to plugins to determine what input they are
+expecting. Plugins may expect one of the following:
+
+- ``physical_line`` to receive the line as it appears in the file
+
+- ``logical_line`` to receive the logical line (not as it appears in the file)
+
+- ``tree`` to receive the abstract syntax tree (AST) for the file
+
+We also analyze the rest of the parameters to provide more detail to the
+plugin. This function will return the parameters in a consistent way across
+versions of Python and will handle both classes and functions that are used as
+plugins. Further, if the plugin is a class, it will strip the ``self``
+argument so we can check the parameters of the plugin consistently.