diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 0000000..ea868e2 --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,84 @@ +tests: +skips: +- B404 # Ignore warnings about importing subprocess +- B603 # Ignore warnings about calling subprocess.Popen without shell=True +- B607 # Ignore warnings about calling subprocess.Popen without a full path to executable + +### (optional) plugin settings - some test plugins require configuration data +### that may be given here, per-plugin. All bandit test plugins have a built in +### set of sensible defaults and these will be used if no configuration is +### provided. It is not necessary to provide settings for every (or any) plugin +### if the defaults are acceptable. + +any_other_function_with_shell_equals_true: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +execute_with_run_as_root_equals_true: + function_names: [ceilometer.utils.execute, cinder.utils.execute, neutron.agent.linux.utils.execute, + nova.utils.execute, nova.utils.trycmd] +hardcoded_tmp_directory: + tmp_dirs: [/tmp, /var/tmp, /dev/shm] +linux_commands_wildcard_injection: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +password_config_option_not_marked_secret: + function_names: [oslo.config.cfg.StrOpt, oslo_config.cfg.StrOpt] +ssl_with_bad_defaults: + bad_protocol_versions: [PROTOCOL_SSLv2, SSLv2_METHOD, SSLv23_METHOD, PROTOCOL_SSLv3, + PROTOCOL_TLSv1, SSLv3_METHOD, TLSv1_METHOD] +ssl_with_bad_version: + bad_protocol_versions: [PROTOCOL_SSLv2, SSLv2_METHOD, SSLv23_METHOD, PROTOCOL_SSLv3, + PROTOCOL_TLSv1, SSLv3_METHOD, TLSv1_METHOD] +start_process_with_a_shell: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +start_process_with_no_shell: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +start_process_with_partial_path: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +subprocess_popen_with_shell_equals_true: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +subprocess_without_shell_equals_true: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +try_except_continue: {check_typed_exception: false} +try_except_pass: {check_typed_exception: false} + diff --git a/.gitignore b/.gitignore index c968761..baf560d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,12 @@ *.pyc .tox +.eggs *.egg *.egg-info build dist *.zip +.cache +*.sw* +*.log +docs/build/html/* diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..f778dd4 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,378 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS,.git,flake8.egg-info + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence=INFERENCE_FAILURE + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=intern-builtin,nonzero-method,parameter-unpacking,backtick,raw_input-builtin,dict-view-method,filter-builtin-not-iterating,long-builtin,unichr-builtin,input-builtin,unicode-builtin,file-builtin,map-builtin-not-iterating,delslice-method,apply-builtin,cmp-method,setslice-method,coerce-method,long-suffix,raising-string,import-star-module-level,buffer-builtin,reload-builtin,unpacking-in-except,print-statement,hex-method,old-octal-literal,metaclass-assignment,dict-iter-method,range-builtin-not-iterating,using-cmp-argument,indexing-exception,no-absolute-import,coerce-builtin,getslice-method,suppressed-message,execfile-builtin,round-builtin,useless-suppression,reduce-builtin,old-raise-syntax,zip-builtin-not-iterating,cmp-builtin,xrange-builtin,standarderror-builtin,old-division,oct-method,next-method-called,old-ne-operator,basestring-builtin + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=yes + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes= + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=20 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=10 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml index 78a0db1..20c7c29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,6 @@ notifications: matrix: include: - - python: 2.6 - env: TOXENV=py26 - python: 2.7 env: TOXENV=py27 - python: 3.3 @@ -24,6 +22,14 @@ matrix: - python: pypy env: TOXENV=pypy - python: 2.7 - env: TOXENV=py27-flake8 + env: TOXENV=readme - python: 3.4 - env: TOXENV=py34-flake8 + env: TOXENV=flake8 + - python: 3.4 + env: TOXENV=pylint + - python: 3.4 + env: TOXENV=doc8 + - python: 3.4 + env: TOXENV=bandit + - python: 3.4 + env: TOXENV=docs diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index ad3d2c3..0000000 --- a/CHANGES.rst +++ /dev/null @@ -1,362 +0,0 @@ -CHANGES -======= - -2.6.1 - 2016-06-25 ------------------- - -- **Bug** Update the config files to search for to include ``setup.cfg`` and - ``tox.ini``. This was broken in 2.5.5 when we stopped passing - ``config_file`` to our Style Guide - -2.6.0 - 2016-06-15 ------------------- - -- **Requirements Change** Switch to pycodestyle as all future pep8 releases - will use that package name - -- **Improvement** Allow for Windows users on *select* versions of Python to - use ``--jobs`` and multiprocessing - -- **Improvement** Update bounds on McCabe - -- **Improvement** Update bounds on PyFlakes and blacklist known broken - versions - -- **Improvement** Handle new PyFlakes warning with a new error code: F405 - -2.5.5 - 2016-06-14 ------------------- - -- **Bug** Fix setuptools integration when parsing config files - -- **Bug** Don't pass the user's config path as the config_file when creating a - StyleGuide - -2.5.4 - 2016-02-11 ------------------- - -- **Bug** Missed an attribute rename during the v2.5.3 release. - -2.5.3 - 2016-02-11 ------------------- - -- **Bug** Actually parse ``output_file`` and ``enable_extensions`` from config - files - -2.5.2 - 2016-01-30 ------------------- - -- **Bug** Parse ``output_file`` and ``enable_extensions`` from config files - -- **Improvement** Raise upper bound on mccabe plugin to allow for version - 0.4.0 - -2.5.1 - 2015-12-08 ------------------- - -- **Bug** Properly look for ``.flake8`` in current working directory - (`GitLab#103`_) - -- **Bug** Monkey-patch ``pep8.stdin_get_value`` to cache the actual value in - stdin. This helps plugins relying on the function when run with - multiprocessing. (`GitLab#105`_, `GitLab#107`_) - -.. _GitLab#103: https://gitlab.com/pycqa/flake8/issues/103 -.. _GitLab#105: https://gitlab.com/pycqa/flake8/issues/105 -.. _GitLab#107: https://gitlab.com/pycqa/flake8/issues/107 - -2.5.0 - 2015-10-26 ------------------- - -- **Improvement** Raise cap on PyFlakes for Python 3.5 support - -- **Improvement** Avoid deprecation warnings when loading extensions - (`GitLab#59`_, `GitLab#90`_) - -- **Improvement** Separate logic to enable "off-by-default" extensions - (`GitLab#67`_) - -- **Bug** Properly parse options to setuptools Flake8 command (`GitLab!41`_) - -- **Bug** Fix exceptions when output on stdout is truncated before Flake8 - finishes writing the output (`GitLab#69`_) - -- **Bug** Fix error on OS X where Flake8 can no longer acquire or create new - semaphores (`GitLab#74`_) - -.. _GitLab!41: https://gitlab.com/pycqa/flake8/merge_requests/41 -.. _GitLab#59: https://gitlab.com/pycqa/flake8/issues/59 -.. _GitLab#67: https://gitlab.com/pycqa/flake8/issues/67 -.. _GitLab#69: https://gitlab.com/pycqa/flake8/issues/69 -.. _GitLab#74: https://gitlab.com/pycqa/flake8/issues/74 -.. _GitLab#90: https://gitlab.com/pycqa/flake8/issues/90 - -2.4.1 - 2015-05-18 ------------------- - -- **Bug** Do not raise a ``SystemError`` unless there were errors in the - setuptools command. (`GitLab#39`_, `GitLab!23`_) - -- **Bug** Do not verify dependencies of extensions loaded via entry-points. - -- **Improvement** Blacklist versions of pep8 we know are broken - -.. _GitLab#39: https://gitlab.com/pycqa/flake8/issues/39 -.. _GitLab!23: https://gitlab.com/pycqa/flake8/merge_requests/23 - -2.4.0 - 2015-03-07 ------------------- - -- **Bug** Print filenames when using multiprocessing and ``-q`` option. - (`GitLab#31`_) - -- **Bug** Put upper cap on dependencies. The caps for 2.4.0 are: - - - ``pep8 < 1.6`` (Related to `GitLab#35`_) - - - ``mccabe < 0.4`` - - - ``pyflakes < 0.9`` - - See also `GitLab#32`_ - -- **Bug** Files excluded in a config file were not being excluded when flake8 - was run from a git hook. (`GitHub#2`_) - -- **Improvement** Print warnings for users who are providing mutually - exclusive options to flake8. (`GitLab#8`_, `GitLab!18`_) - -- **Feature** Allow git hook configuration to live in ``.git/config``. - See the updated `VCS hooks docs`_ for more details. (`GitLab!20`_) - -.. _GitHub#2: https://github.com/pycqa/flake8/pull/2 -.. _GitLab#8: https://gitlab.com/pycqa/flake8/issues/8 -.. _GitLab#31: https://gitlab.com/pycqa/flake8/issues/31 -.. _GitLab#32: https://gitlab.com/pycqa/flake8/issues/32 -.. _GitLab#35: https://gitlab.com/pycqa/flake8/issues/35 -.. _GitLab!18: https://gitlab.com/pycqa/flake8/merge_requests/18 -.. _GitLab!20: https://gitlab.com/pycqa/flake8/merge_requests/20 -.. _VCS hooks docs: https://flake8.readthedocs.io/en/latest/vcs.html - -2.3.0 - 2015-01-04 ------------------- - -- **Feature**: Add ``--output-file`` option to specify a file to write to - instead of ``stdout``. - -- **Bug** Fix interleaving of output while using multiprocessing - (`GitLab#17`_) - -.. _GitLab#17: https://gitlab.com/pycqa/flake8/issues/17 - -2.2.5 - 2014-10-19 ------------------- - -- Flush standard out when using multiprocessing - -- Make the check for "# flake8: noqa" more strict - -2.2.4 - 2014-10-09 ------------------- - -- Fix bugs triggered by turning multiprocessing on by default (again) - - Multiprocessing is forcibly disabled in the following cases: - - - Passing something in via stdin - - - Analyzing a diff - - - Using windows - -- Fix --install-hook when there are no config files present for pep8 or - flake8. - -- Fix how the setuptools command parses excludes in config files - -- Fix how the git hook determines which files to analyze (Thanks Chris - Buccella!) - -2.2.3 - 2014-08-25 ------------------- - -- Actually turn multiprocessing on by default - -2.2.2 - 2014-07-04 ------------------- - -- Re-enable multiprocessing by default while fixing the issue Windows users - were seeing. - -2.2.1 - 2014-06-30 ------------------- - -- Turn off multiple jobs by default. To enable automatic use of all CPUs, use - ``--jobs=auto``. Fixes #155 and #154. - -2.2.0 - 2014-06-22 ------------------- - -- New option ``doctests`` to run Pyflakes checks on doctests too -- New option ``jobs`` to launch multiple jobs in parallel -- Turn on using multiple jobs by default using the CPU count -- Add support for ``python -m flake8`` on Python 2.7 and Python 3 -- Fix Git and Mercurial hooks: issues #88, #133, #148 and #149 -- Fix crashes with Python 3.4 by upgrading dependencies -- Fix traceback when running tests with Python 2.6 -- Fix the setuptools command ``python setup.py flake8`` to read - the project configuration - - -2.1.0 - 2013-10-26 ------------------- - -- Add FLAKE8_LAZY and FLAKE8_IGNORE environment variable support to git and - mercurial hooks -- Force git and mercurial hooks to repsect configuration in setup.cfg -- Only check staged files if that is specified -- Fix hook file permissions -- Fix the git hook on python 3 -- Ignore non-python files when running the git hook -- Ignore .tox directories by default -- Flake8 now reports the column number for PyFlakes messages - - -2.0.0 - 2013-02-23 ------------------- - -- Pyflakes errors are prefixed by an ``F`` instead of an ``E`` -- McCabe complexity warnings are prefixed by a ``C`` instead of a ``W`` -- Flake8 supports extensions through entry points -- Due to the above support, we **require** setuptools -- We publish the `documentation `_ -- Fixes #13: pep8, pyflakes and mccabe become external dependencies -- Split run.py into main.py, engine.py and hooks.py for better logic -- Expose our parser for our users -- New feature: Install git and hg hooks automagically -- By relying on pyflakes (0.6.1), we also fixed #45 and #35 - - -1.7.0 - 2012-12-21 ------------------- - -- Fixes part of #35: Exception for no WITHITEM being an attribute of Checker - for Python 3.3 -- Support stdin -- Incorporate @phd's builtins pull request -- Fix the git hook -- Update pep8.py to the latest version - - -1.6.2 - 2012-11-25 ------------------- - -- fixed the NameError: global name 'message' is not defined (#46) - - -1.6.1 - 2012-11-24 ------------------- - -- fixed the mercurial hook, a change from a previous patch was not properly - applied -- fixed an assumption about warnings/error messages that caused an exception - to be thrown when McCabe is used - - -1.6 - 2012-11-16 ----------------- - -- changed the signatures of the ``check_file`` function in flake8/run.py, - ``skip_warning`` in flake8/util.py and the ``check``, ``checkPath`` - functions in flake8/pyflakes.py. -- fix ``--exclude`` and ``--ignore`` command flags (#14, #19) -- fix the git hook that wasn't catching files not already added to the index - (#29) -- pre-emptively includes the addition to pep8 to ignore certain lines. - Add ``# nopep8`` to the end of a line to ignore it. (#37) -- ``check_file`` can now be used without any special prior setup (#21) -- unpacking exceptions will no longer cause an exception (#20) -- fixed crash on non-existent file (#38) - - -1.5 - 2012-10-13 ----------------- - -- fixed the stdin -- make sure mccabe catches the syntax errors as warnings -- pep8 upgrade -- added max_line_length default value -- added Flake8Command and entry points if setuptools is around -- using the setuptools console wrapper when available - - -1.4 - 2012-07-12 ----------------- - -- git_hook: Only check staged changes for compliance -- use pep8 1.2 - - -1.3.1 - 2012-05-19 ------------------- - -- fixed support for Python 2.5 - - -1.3 - 2012-03-12 ----------------- - -- fixed false W402 warning on exception blocks. - - -1.2 - 2012-02-12 ----------------- - -- added a git hook -- now Python 3 compatible -- mccabe and pyflakes have warning codes like pep8 now - - -1.1 - 2012-02-14 ----------------- - -- fixed the value returned by --version -- allow the flake8: header to be more generic -- fixed the "hg hook raises 'physical lines'" bug -- allow three argument form of raise -- now uses setuptools if available, for 'develop' command - - -1.0 - 2011-11-29 ----------------- - -- Deactivates by default the complexity checker -- Introduces the complexity option in the HG hook and the command line. - - -0.9 - 2011-11-09 ----------------- - -- update pep8 version to 0.6.1 -- mccabe check: gracefully handle compile failure - - -0.8 - 2011-02-27 ----------------- - -- fixed hg hook -- discard unexisting files on hook check - - -0.7 - 2010-02-18 ----------------- - -- Fix pep8 initialization when run through Hg -- Make pep8 short options work when run through the command line -- Skip duplicates when controlling files via Hg - - -0.6 - 2010-02-15 ----------------- - -- Fix the McCabe metric on some loops diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 120000 index 0000000..8deb188 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1 @@ +docs/source/internal/contributing.rst \ No newline at end of file diff --git a/LICENSE b/LICENSE index 32b274a..e5e3d6f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ == Flake8 License (MIT) == Copyright (C) 2011-2013 Tarek Ziade -Copyright (C) 2012-2013 Ian Cordasco +Copyright (C) 2012-2016 Ian Cordasco Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/MANIFEST.in b/MANIFEST.in index ab3d229..b87aff1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ include *.rst include CONTRIBUTORS.txt include LICENSE -recursive-include flake8 * +recursive-include docs * +recursive-include tests * +recursive-include src * diff --git a/bin/git-patch-to-hg-export.py b/bin/git-patch-to-hg-export.py deleted file mode 100755 index 118e94e..0000000 --- a/bin/git-patch-to-hg-export.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -"""\ -Git patch to HG changeset patch converter. - -USAGE: git-patch-to-hg-export.py < git.patch > hg.patch -""" - -from email.utils import parsedate_tz, mktime_tz -import re -import os -import sys - - -def git_patch_to_hg(fin, fout): - fout.write('# HG changeset patch\n') - - subject_re = re.compile(r'^(RE:)?\s*(\[[^]]*\])?\s*', re.I) - - # headers - for line in fin: - if line.startswith('From: '): - fout.write('# User %s' % line[6:]) - elif line.startswith('Date: '): - t = parsedate_tz(line[6:]) - timestamp = mktime_tz(t) - timezone = -t[-1] - fout.write('# Date %d %d\n' % (timestamp, timezone)) - elif line.startswith('Subject: '): - subject = subject_re.sub('', line[9:]) - fout.write(subject + '\n') - elif line == '\n' or line == '\r\n': - break - - # commit message - for line in fin: - if line == '---\n': - break - fout.write(line) - - # skip over the diffstat - for line in fin: - if line.startswith('diff --git'): - fout.write('\n' + line) - break - - # diff - # NOTE: there will still be an index line after each diff --git, but it - # will be ignored - for line in fin: - fout.write(line.encode('utf-8')) - - # NOTE: the --/version will still be at the end, but it will be ignored - - -def open_file(): - if len(sys.argv) > 1: - if re.match('https?://', sys.argv[1]): - import requests - import io - resp = requests.get(sys.argv[1]) - if resp.ok: - return io.StringIO(resp.content.decode('utf-8')) - else: - return io.StringIO('') - elif os.path.exists(sys.argv[1]): - return open(sys.argv[1]) - return sys.stdin - - -if __name__ == "__main__": - patch_fd = open_file() - git_patch_to_hg(patch_fd, sys.stdout) - - -__author__ = "Mark Lodato " - -__license__ = """ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php - -Copyright (c) 2009 Mark Lodato - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. -""" diff --git a/dev-requirements.txt b/dev-requirements.txt index 99a2340..053148f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1 @@ -pycodestyle -pyflakes -mccabe -mock -nose +tox diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index bf49b54..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,130 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Raclette.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Raclette.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/Raclette" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Raclette" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index fd746f7..0000000 --- a/docs/api.rst +++ /dev/null @@ -1,38 +0,0 @@ -========== -Flake8 API -========== - -.. module:: flake8 - -flake8.engine -============= - -.. autofunction:: flake8.engine.get_parser - -.. autofunction:: flake8.engine.get_style_guide - -flake8.hooks -============ - -.. autofunction:: flake8.hooks.git_hook - -.. autofunction:: flake8.hooks.hg_hook - -flake8.main -=========== - -.. autofunction:: flake8.main.main - -.. autofunction:: flake8.main.check_file - -.. autofunction:: flake8.main.check_code - -.. autoclass:: flake8.main.Flake8Command - -flake8.util -=========== - -For AST checkers, this module has the ``iter_child_nodes`` function and -handles compatibility for all versions of Python between 2.5 and 3.3. The -function was added to the ``ast`` module in Python 2.6 but is redefined in the -case where the user is running Python 2.5 diff --git a/docs/build/.keep b/docs/build/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/buildout.rst b/docs/buildout.rst deleted file mode 100644 index da9c58a..0000000 --- a/docs/buildout.rst +++ /dev/null @@ -1,17 +0,0 @@ -Buildout integration -===================== - -In order to use Flake8 inside a buildout, edit your buildout.cfg and add this:: - - [buildout] - - parts += - ... - flake8 - - [flake8] - recipe = zc.recipe.egg - eggs = flake8 - ${buildout:eggs} - entry-points = - flake8=flake8.main:main diff --git a/docs/changes.rst b/docs/changes.rst deleted file mode 100644 index 43731a7..0000000 --- a/docs/changes.rst +++ /dev/null @@ -1,5 +0,0 @@ -Changes -======= - -.. include:: ../CHANGES.rst - :start-line: 3 diff --git a/docs/config.rst b/docs/config.rst deleted file mode 100644 index c4d94d7..0000000 --- a/docs/config.rst +++ /dev/null @@ -1,68 +0,0 @@ -Configuration -============= - -Configuration settings are applied in three ways: user, project, and the -``--config`` CLI argument. The user (global) configuration is read first. Next -the project configuration is loaded, and overrides any settings found in both -the user (global) and project configurations. Finally, if the ``--config`` -argument is used on the command line, the specified file is loaded and -overrides any settings that overlap with the user (global) and project -configurations. - - -User (Global) -------------- - -The user settings are read from the ``~/.config/flake8`` file (or the -``~/.flake8`` file on Windows). -Example:: - - [flake8] - ignore = E226,E302,E41 - max-line-length = 160 - exclude = tests/* - max-complexity = 10 - -Per-Project ------------ - -At the project level, the ``tox.ini``, ``setup.cfg``, ``.pep8`` or ``.flake8`` -files are read if present. Only the first file is considered. If this file -does not have a ``[flake8]`` section, no project specific configuration is -loaded. - -Per Code Line -------- - -To ignore one line of code add ``# NOQA`` as a line comment. - -Default -------- - -If the ``ignore`` option is not in the configuration and not in the arguments, -only the error codes ``E123/E133``, ``E226`` and ``E241/E242`` are ignored -(see the :ref:`warning and error codes `). - -Settings --------- - -This is a (likely incomplete) list of settings that can be used in your config -file. In general, any settings that ``pycodestyle`` supports we also support and -we add the ability to set ``max-complexity`` as well. - -- ``exclude``: comma-separated filename and glob patterns - default: ``.svn,CVS,.bzr,.hg,.git,__pycache__`` - -- ``filename``: comma-separated filename and glob patterns - default: ``*.py`` - -- ``select``: select errors and warnings to enable which are off by default - -- ``ignore``: skip errors or warnings - -- ``max-line-length``: set maximum allowed line length - default: 79 - -- ``format``: set the error format - -- ``max-complexity``: McCabe complexity threshold diff --git a/docs/extensions.rst b/docs/extensions.rst deleted file mode 100644 index 9e89fb1..0000000 --- a/docs/extensions.rst +++ /dev/null @@ -1,149 +0,0 @@ -Writing an Extension for Flake8 -=============================== - -Since Flake8 is now adding support for extensions, we require ``setuptools`` -so we can manage extensions through entry points. If you are making an -existing tool compatible with Flake8 but do not already require -``setuptools``, you should probably add it to your list of requirements. Next, -you'll need to edit your ``setup.py`` file so that upon installation, your -extension is registered. If you define a class called ``PackageEntryClass`` -then this would look something like the following:: - - - setup( - # ... - entry_points={ - 'flake8.extension': ['P10 = package.PackageEntryClass'], - } - # ... - ) - - -If you intend to publish your extension, choose a unique code prefix -following the convention for :ref:`error codes `. -In addition, you can open a request in the `issue tracker -`_ to register the prefix in the -documentation. - -.. TODO: describe the API required for the 3 kind of extensions: - * physical line checkers - * logical line checkers - * AST checkers - - -A real example: McCabe ----------------------- - -Below is an example from mccabe_ for how to write your ``setup.py`` file for -your Flake8 extension. - -.. code-block:: python - - # https://github.com/flintwork/mccabe/blob/0.2/setup.py#L38:L42 - # -*- coding: utf-8 -*- - from setuptools import setup - - # ... - - setup( - name='mccabe', - - # ... - - install_requires=[ - 'setuptools', - ], - entry_points={ - 'flake8.extension': [ - 'C90 = mccabe:McCabeChecker', - ], - }, - - # ... - - ) - -In ``mccabe.py`` you can see that extra options are added to the parser when -flake8 registers the extension: - -.. code-block:: python - - # https://github.com/flintwork/mccabe/blob/0.2/mccabe.py#L225:L254 - class McCabeChecker(object): - """McCabe cyclomatic complexity checker.""" - name = 'mccabe' - version = __version__ - _code = 'C901' - _error_tmpl = "C901 %r is too complex (%d)" - max_complexity = 0 - - def __init__(self, tree, filename): - self.tree = tree - - @classmethod - def add_options(cls, parser): - parser.add_option('--max-complexity', default=-1, action='store', - type='int', help="McCabe complexity threshold") - parser.config_options.append('max-complexity') - - @classmethod - def parse_options(cls, options): - cls.max_complexity = options.max_complexity - - def run(self): - if self.max_complexity < 0: - return - visitor = PathGraphingAstVisitor() - visitor.preorder(self.tree, visitor) - for graph in visitor.graphs.values(): - if graph.complexity() >= self.max_complexity: - text = self._error_tmpl % (graph.entity, graph.complexity()) - yield graph.lineno, 0, text, type(self) - -Since that is the defined entry point in the above ``setup.py``, flake8 finds -it and uses it to register the extension. - -If we wanted the extension or a check to be optional, you can add -``off_by_default = True`` to our entry point. For example, we could -update ``mccabe.py`` with this variable as shown below: - -.. code-block:: python - - # https://github.com/flintwork/mccabe/blob/0.2/mccabe.py#L225:L254 - class McCabeChecker(object): - """McCabe cyclomatic complexity checker.""" - name = 'mccabe' - version = __version__ - off_by_default = True - -If we wanted to run the optional extension or check, we need to specify the -error and warnings via the ``--enable-extension`` command line argument. In our -case, we could run ``flake8 --enable-extension=C90`` which would enable our -off_by_default example version of the mccabe extension. - - -Existing Extensions -=================== - -This is not at all a comprehensive listing of existing extensions but simply a -listing of the ones we are aware of: - -* `flake8-debugger `_ - -* `flake8-immediate `_ - -* `flake8-print `_ - -* `flake8-todo `_ - -* `pep8-naming `_ - -* `radon `_ - -* `flake8-import-order `_ - -* `flake8-respect-noqa `_ - -.. links -.. _mccabe: https://github.com/flintwork/mccabe -.. _PyPI: https://pypi.python.org/pypi/ diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index dcd2501..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. include:: ../README.rst - - -Documentation -============= - -.. toctree:: - - config - warnings - vcs - buildout - setuptools - api - extensions - changes - -Original Projects -================= - -Flake8 is just a glue project, all the merits go to the creators and maintainers -of the original projects: - -- pycodestyle: https://github.com/pycqa/pycodestyle -- PyFlakes: https://launchpad.net/pyflakes -- McCabe: http://nedbatchelder.com/blog/200803/python_code_complexity_microtool.html diff --git a/docs/setuptools.rst b/docs/setuptools.rst deleted file mode 100644 index 8bc080d..0000000 --- a/docs/setuptools.rst +++ /dev/null @@ -1,25 +0,0 @@ -Setuptools integration -====================== - -Upon installation, Flake8 enables a setuptools command that checks Python -files declared by your project. - -Running ``python setup.py flake8`` on the command line will check the files -listed in your ``py_modules`` and ``packages``. If any warning is found, -the command will exit with an error code:: - - $ python setup.py flake8 - -Also, to allow users to be able to use the command without having to install -flake8 themselves, add flake8 to the setup_requires of your setup() like so:: - - setup( - name="project", - packages=["project"], - - setup_requires=[ - "flake8" - ] - ) - - diff --git a/docs/conf.py b/docs/source/conf.py similarity index 67% rename from docs/conf.py rename to docs/source/conf.py index 0bcaa6f..c7a0fd0 100644 --- a/docs/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- # +# flake8 documentation build configuration file, created by +# sphinx-quickstart on Tue Jan 19 07:14:10 2016. +# # This file is execfile()d with the current directory set to its # containing dir. # @@ -12,29 +15,35 @@ import sys import os -# This environment variable makes decorators not decorate functions, so their -# signatures in the generated documentation are still correct -os.environ['GENERATING_DOCUMENTATION'] = "flake8" - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) -import flake8 +#sys.path.insert(0, os.path.abspath('.')) -# -- General configuration ---------------------------------------------------- +# -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +needs_sphinx = '1.3' # Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc'] +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx-prompt', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix of source filenames. +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. @@ -45,8 +54,10 @@ master_doc = 'index' # General information about the project. project = u'flake8' -copyright = u'2012-2013 - Tarek Ziade, Ian Cordasco, Florent Xicluna' +copyright = u'2016, Ian Cordasco' +author = u'Ian Cordasco' +import flake8 # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. @@ -54,11 +65,18 @@ copyright = u'2012-2013 - Tarek Ziade, Ian Cordasco, Florent Xicluna' # The short X.Y version. version = flake8.__version__ # The full version, including alpha/beta/rc tags. -release = version +release = flake8.__version__ + +rst_epilog = """ +.. |Flake8| replace:: :program:`Flake8` +""" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -68,10 +86,10 @@ release = version # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = [] -# The reST default role (used for this markup: `text`) to use for -# all documents. +# The reST default role (used for this markup: `text`) to use for all +# documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. @@ -86,17 +104,23 @@ exclude_patterns = ['_build'] #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -# pygments_style = 'flask_theme_support.FlaskyStyle' +pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False -# -- Options for HTML output -------------------------------------------------- +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -#html_theme = 'nature' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -125,7 +149,12 @@ exclude_patterns = ['_build'] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -152,10 +181,10 @@ exclude_patterns = ['_build'] #html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = False +#html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = False +#html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True @@ -168,24 +197,45 @@ exclude_patterns = ['_build'] # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + # Output file base name for HTML help builder. -htmlhelp_basename = 'flake8_doc' +htmlhelp_basename = 'flake8doc' +# -- Options for LaTeX output --------------------------------------------- -# -- Options for LaTeX output ------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, -# author, documentclass [howto/manual]). +# author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'flake8.tex', u'flake8 Documentation', - u'Tarek Ziade', 'manual'), + (master_doc, 'flake8.tex', u'flake8 Documentation', + u'Ian Cordasco', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -202,9 +252,6 @@ latex_documents = [ # If true, show URL addresses after external links. #latex_show_urls = False -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - # Documents to append as an appendix to all manuals. #latex_appendices = [] @@ -212,29 +259,42 @@ latex_documents = [ #latex_domain_indices = True -# -- Options for manual page output ------------------------------------------- +# -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'flake8', u'flake8 Documentation', - [u'Tarek Ziade', u'Ian Cordasco', u'Florent Xicluna'], 1) + (master_doc, 'flake8', u'flake8 Documentation', + [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False -# -- Options for Texinfo output ----------------------------------------------- +# -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'flake8', u'flake8 Documentation', u'Tarek Ziade', - 'flake8', 'Code checking using pycodestyle, pyflakes and mccabe', + ('index', 'Flake8', u'Flake8 Documentation', u'Tarek Ziade', + 'Flake8', 'Code checking using pycodestyle, pyflakes and mccabe', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -texinfo_appendices = [] +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'python': ('https://docs.python.org/3.4', None)} diff --git a/docs/source/faq.rst b/docs/source/faq.rst new file mode 100644 index 0000000..3f2bc04 --- /dev/null +++ b/docs/source/faq.rst @@ -0,0 +1,56 @@ +============================ + Frequently Asked Questions +============================ + +When is Flake8 released? +======================== + +|Flake8| is released *as necessary*. Sometimes there are specific goals and +drives to get to a release. Usually, we release as users report and fix +bugs. + + +How can I help Flake8 release faster? +===================================== + +Look at the next milestone. If there's work you can help us complete, that +will help us get to the next milestone. If there's a show-stopping bug that +needs to be released, let us know but please be kind. |Flake8| is developed +and released entirely on volunteer time. + + +What is the next version of Flake8? +=================================== + +In general we try to use milestones to indicate this. If the last release +on PyPI is 3.1.5 and you see a milestone for 3.2.0 in GitLab, there's a +good chance that 3.2.0 is the next release. + + +Why does Flake8 use ranges for its dependencies? +================================================ + +|Flake8| uses ranges for mccabe, pyflakes, and pycodestyle because each of +those projects tend to add *new* checks in minor releases. It has been an +implicit design goal of |Flake8|'s to make the list of error codes stable in +its own minor releases. That way if you install something from the 2.5 +series today, you will not find new checks in the same series in a month +from now when you install it again. + +|Flake8|'s dependencies tend to avoid new checks in patch versions which is +why |Flake8| expresses its dependencies roughly as:: + + pycodestyle >= 2.0.0, < 2.1.0 + pyflakes >= 0.8.0, != 1.2.0, != 1.2.1, != 1.2.2, < 1.3.0 + mccabe >= 0.5.0, < 0.6.0 + +This allows those projects to release patch versions that fix bugs and for +|Flake8| users to consume those fixes. + + +Should I file an issue when a new version of a dependency is available? +======================================================================= + +**No.** The current Flake8 core team (of one person) is also +a core developer of pycodestyle, pyflakes, and mccabe. They are aware of +these releases. diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst new file mode 100644 index 0000000..dd691b6 --- /dev/null +++ b/docs/source/glossary.rst @@ -0,0 +1,56 @@ +.. _glossary: + +================================================ + Glossary of Terms Used in Flake8 Documentation +================================================ + +.. glossary:: + :sorted: + + formatter + A :term:`plugin` that augments the output of |Flake8| when passed + to :option:`flake8 --format`. + + plugin + A package that is typically installed from PyPI to augment the + behaviour of |Flake8| either through adding one or more additional + :term:`check`\ s or providing additional :term:`formatter`\ s. + + check + A piece of logic that corresponds to an error code. A check may + be a style check (e.g., check the length of a given line against + the user configured maximum) or a lint check (e.g., checking for + unused imports) or some other check as defined by a plugin. + + error + error code + The symbol associated with a specific :term:`check`. For example, + pycodestyle implements :term:`check`\ s that look for whitespace + around binary operators and will either return an error code of + ``W503`` or ``W504``. + + warning + Typically the ``W`` class of :term:`error code`\ s from pycodestyle. + + class + error class + A larger grouping of related :term:`error code`\ s. For example, + ``W503`` and ``W504`` are two codes related to whitespace. ``W50`` + would be the most specific class of codes relating to whitespace. + ``W`` would be the warning class that subsumes all whitespace + errors. + + pyflakes + The project |Flake8| depends on to lint files (check for unused + imports, variables, etc.). This uses the ``F`` :term:`class` of + :term:`error code`\ s reported by |Flake8|. + + pycodestyle + The project |Flake8| depends on to provide style enforcement. + pycodestyle implements :term:`check`\ s for :pep:`8`. This uses the + ``E`` and ``W`` :term:`class`\ es of :term:`error code`\ s. + + mccabe + The project |Flake8| depends on to calculate the McCabe complexity + of a unit of code (e.g., a function). This uses the ``C`` + :term:`class` of :term`error code`\ s. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..51b0189 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,131 @@ +.. flake8 documentation master file, created by + sphinx-quickstart on Tue Jan 19 07:14:10 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +=============================================== + Flake8: Your Tool For Style Guide Enforcement +=============================================== + +Quickstart +========== + +.. _installation-guide: + +Installation +------------ + +To install |Flake8|, open an interactive shell and run: + +.. code:: + + python -m pip install flake8 + +If you want |Flake8| to be installed for your default Python installation, you +can instead use: + +.. code:: + + python -m pip install flake8 + +.. note:: + + It is **very** important to install |Flake8| on the *correct* version of + Python for your needs. If you want |Flake8| to properly parse new language + features in Python 3.5 (for example), you need it to be installed on 3.5 + for |Flake8| to understand those features. In many ways, Flake8 is tied to + the version of Python on which it runs. + +Using Flake8 +------------ + +To start using |Flake8|, open an interactive shell and run: + +.. code:: + + flake8 path/to/code/to/check.py + # or + flake8 path/to/code/ + +.. note:: + + If you have installed |Flake8| on a particular version of Python (or on + several versions), it may be best to instead run ``python -m + flake8``. + +If you only want to see the instances of a specific warning or error, you can +*select* that error like so: + +.. code:: + + flake8 --select E123,W503 path/to/code/ + +Alternatively, if you want to *ignore* only one specific warning or error: + +.. code:: + + flake8 --ignore E24,W504 path/to/code/ + +Please read our user guide for more information about how to use and configure +|Flake8|. + +FAQ and Glossary +================ + +.. toctree:: + :maxdepth: 2 + + faq + glossary + +User Guide +========== + +All users of |Flake8| should read this portion of the documentation. This +provides examples and documentation around |Flake8|'s assortment of options +and how to specify them on the command-line or in configuration files. + +.. toctree:: + :maxdepth: 2 + + user/index + +Plugin Developer Guide +====================== + +If you're maintaining a plugin for |Flake8| or creating a new one, you should +read this section of the documentation. It explains how you can write your +plugins and distribute them to others. + +.. toctree:: + :maxdepth: 2 + + plugin-development/index + +Contributor Guide +================= + +If you are reading |Flake8|'s source code for fun or looking to contribute, +you should read this portion of the documentation. This is a mix of documenting +the internal-only interfaces |Flake8| and documenting reasoning for Flake8's +design. + +.. toctree:: + :maxdepth: 2 + + internal/index + +Release Notes and History +========================= + +.. toctree:: + :maxdepth: 2 + + release-notes/index + +General Indices +=============== + +* :ref:`genindex` +* :ref:`Index of Documented Public Modules ` +* :ref:`Glossary of terms ` diff --git a/docs/source/internal/.keep b/docs/source/internal/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/internal/checker.rst b/docs/source/internal/checker.rst new file mode 100644 index 0000000..19bcf7d --- /dev/null +++ b/docs/source/internal/checker.rst @@ -0,0 +1,70 @@ +==================== + How Checks are Run +==================== + +In |Flake8| 2.x, |Flake8| delegated check running to pep8. In 3.0 |Flake8| +takes on that responsibility. This has allowed for simpler +handling of the ``--jobs`` parameter (using :mod:`multiprocessing`) and +simplified our fallback if something goes awry with concurency. +At the lowest level we have a |FileChecker|. Instances of |FileChecker| are +created for *each* file to be analyzed by |Flake8|. Each instance, has a copy +of all of the plugins registered with setuptools in the ``flake8.extension`` +entry-point group. + +The |FileChecker| instances are managed by an instance of |Manager|. The +|Manager| instance handles creating sub-processes with +:mod:`multiprocessing` module and falling back to running checks in serial if +an operating system level error arises. When creating |FileChecker| instances, +the |Manager| is responsible for determining if a particular file has been +excluded. + + +Processing Files +---------------- + +Unfortunately, since |Flake8| took over check running from pep8/pycodestyle, +it also had to take over parsing and processing files for the checkers +to use. Since it couldn't reuse pycodestyle's functionality (since it did not +separate cleanly the processing from check running) that function was isolated +into the :class:`~flake8.processor.FileProcessor` class. We moved +several helper functions into the :mod:`flake8.processor` module (see also +:ref:`Processor Utility Functions `). + + +API Reference +------------- + +.. autoclass:: flake8.checker.FileChecker + :members: + +.. autoclass:: flake8.checker.Manager + :members: + +.. autoclass:: flake8.processor.FileProcessor + :members: + + +.. _processor_utility_functions: + +Utility Functions +````````````````` + +.. autofunction:: flake8.processor.count_parentheses + +.. autofunction:: flake8.processor.expand_indent + +.. autofunction:: flake8.processor.is_eol_token + +.. autofunction:: flake8.processor.is_multiline_string + +.. autofunction:: flake8.processor.log_token + +.. autofunction:: flake8.processor.mutate_string + +.. autofunction:: flake8.processor.token_is_comment + +.. autofunction:: flake8.processor.token_is_newline + +.. Substitutions +.. |FileChecker| replace:: :class:`~flake8.checker.FileChecker` +.. |Manager| replace:: :class:`~flake8.checker.Manager` diff --git a/docs/source/internal/cli.rst b/docs/source/internal/cli.rst new file mode 100644 index 0000000..f203125 --- /dev/null +++ b/docs/source/internal/cli.rst @@ -0,0 +1,26 @@ +Command Line Interface +====================== + +The command line interface of |Flake8| is modeled as an application via +:class:`~flake8.main.cli.Application`. When a user runs ``flake8`` at their +command line, :func:`~flake8.main.cli.main` is run which handles +management of the application. + +User input is parsed *twice* to accomodate logging and verbosity options +passed by the user as early as possible. +This is so as much logging can be produced as possible. + +The default |Flake8| options are registered by +:func:`~flake8.main.options.register_default_options`. Trying to register +these options in plugins will result in errors. + + +API Documentation +----------------- + +.. autofunction:: flake8.main.cli.main + +.. autoclass:: flake8.main.application.Application + :members: + +.. autofunction:: flake8.main.options.register_default_options diff --git a/docs/source/internal/contributing.rst b/docs/source/internal/contributing.rst new file mode 100644 index 0000000..29afc6a --- /dev/null +++ b/docs/source/internal/contributing.rst @@ -0,0 +1,202 @@ +======================== + Contributing to Flake8 +======================== + +There are many ways to contriubte to |Flake8|, and we encourage them all: + +- contributing bug reports and feature requests + +- contributing documenation (and yes that includes this document) + +- reviewing and triaging bugs and merge requests + +Before you go any further, please allow me to reassure you that I do want +*your* contribution. If you think your contribution might not be valuable, I +reassure you that any help you can provide *is* valuable. + + +Code of Conduct +=============== + +|Flake8| adheres to the `Python Code Quality Authority's Code of Conduct`_. +Any violations of the Code of Conduct should be reported to Ian Cordasco +(graffatcolmingov [at] gmail [dot] com). + + +Setting Up A Development Environment +==================================== + +To contribute to |Flake8|'s development, you simply need: + +- Python (one of the versions we support) + +- `tox`_ + + We suggest installing this like: + + .. prompt:: bash + + pip install --user tox + + Or + + .. prompt:: bash + + python -m pip install --user tox + +- your favorite editor + + +Filing a Bug +============ + +When filing a bug against |Flake8|, please fill out the issue template as it +is provided to you by `GitLab`_. If your bug is in reference to one of the +checks that |Flake8| reports by default, please do not report them to |Flake8| +unless |Flake8| is doing something to prevent the check from running or you +have some reason to believe |Flake8| is inhibiting the effectiveness of the +check. + +**Please search for closed and open bug reports before opening new ones.** + +All bug reports about checks should go to their respective projects: + +- Error codes starting with ``E`` and ``W`` should be reported to + `pycodestyle`_. + +- Error codes starting with ``F`` should be reported to `pyflakes`_ + +- Error codes starting with ``C`` should be reported to `mccabe`_ + + +Requesting a New Feature +======================== + +When requesting a new feature in |Flake8|, please fill out the issue template. +Please also note if there are any existing alternatives to your new feature +either via plugins, or combining command-line options. Please provide example +use cases. For example, do not ask for a feature like this: + + I need feature frobulate for my job. + +Instead ask: + + I need |Flake8| to frobulate these files because my team expects them to + frobulated but |Flake8| currently does not frobulate them. We tried using + ``--filename`` but we could not create a pattern that worked. + +The more you explain about *why* you need a feature, the more likely we are to +understand your needs and help you to the best of our ability. + + +Contributing Documentation +========================== + +To contribute to |Flake8|'s documentation, you might want to first read a +little about reStructuredText or Sphinx. |Flake8| has a :ref:`guide of best +practices ` when contributing to our documentation. For the most +part, you should be fine following the structure and style of the rest of +|Flake8|'s documentation. + +All of |Flake8|'s documentation is written in reStructuredText and rendered by +Sphinx. The source (reStructuredText) lives in ``docs/source/``. To build +the documentation the way our Continuous Integration does, run: + +.. prompt:: bash + + tox -e docs + +To view the documentation locally, you can also run: + +.. prompt:: bash + + tox -e serve-docs + +You can run the latter in a separate terminal and continuously re-run the +documentation generation and refresh the documentation you're working on. + +.. note:: + + We lint our documentation just like we lint our code. + You should also run: + + .. prompt:: bash + + tox -e linters + + After making changes and before pushing them to ensure that they will + pass our CI tests. + + +Contributing Code +================= + +|Flake8| development happens on `GitLab`_. Code contributions should be +submitted there. + +Merge requests should: + +- Fix one issue and fix it well + + Fix the issue, but do not include extraneous refactoring or code + reformatting. In other words, keep the diff short, but only as short + as is necessary to fix the bug appropriately and add sufficient testing + around it. Long diffs are fine, so long as everything that it includes + is necessary to the purpose of the merge request. + +- Have descriptive titles and descriptions + + Searching old merge requests is made easier when a merge request is well + described. + +- Have commits that follow this style: + + .. code:: + + Create a short title that is 50 characters long + + Ensure the title and commit message use the imperative voice. The + commit and you are doing something. Also, please ensure that the + body of the commit message does not exceed 72 characters. + + The body may have multiple paragraphs as necessary. + + The final line of the body references the issue appropriately. + + +Reviewing and Triaging Issues and Merge Requests +================================================ + +When reviewing other people's merge requests and issues, please be +**especially** mindful of how the words you choose can be read by someone +else. We strive for professional code reviews that do not insult the +contributor's intelligence or impugn their character. The code review +should be focused on the code, it's effectiveness, and whether it is +appropriate for |Flake8|. + +If you have the ability to edit an issue or merge request's labels, please do +so to make search and prioritization easier. + +|Flake8| uses milestones with both issues and merge requests. This provides +direction for other contributors about when an issue or merge request will be +delivered. + + +.. links +.. _Python Code Quality Authority's Code of Conduct: + http://meta.pycqa.org/en/latest/code-of-conduct.html + +.. _tox: + https://tox.readthedocs.io/ + +.. _GitLab: + https://gitlab.com/pycqa/flake8 + +.. _pycodestyle: + https://github.com/pycqa/pycodestyle + +.. _pyflakes: + https://github.com/pyflakes/pyflakes + +.. _mccabe: + https://github.com/pycqa/mccabe diff --git a/docs/source/internal/formatters.rst b/docs/source/internal/formatters.rst new file mode 100644 index 0000000..c58189b --- /dev/null +++ b/docs/source/internal/formatters.rst @@ -0,0 +1,47 @@ +===================== + Built-in Formatters +===================== + +By default |Flake8| has two formatters built-in, ``default`` and ``pylint``. +These correspond to two classes |DefaultFormatter| and |PylintFormatter|. + +In |Flake8| 2.0, pep8 handled formatting of errors and also allowed users to +specify an arbitrary format string as a parameter to ``--format``. In order +to allow for this backwards compatibility, |Flake8| 3.0 made two choices: + +#. To not limit a user's choices for ``--format`` to the format class names + +#. To make the default formatter attempt to use the string provided by the + user if it cannot find a formatter with that name. + +Default Formatter +================= + +The |DefaultFormatter| continues to use the same default format string as +pep8: ``'%(path)s:%(row)d:%(col)d: %(code)s %(text)s'``. + +To provide the default functionality it overrides two methods: + +#. ``after_init`` + +#. ``format`` + +The former allows us to inspect the value provided to ``--format`` by the +user and alter our own format based on that value. The second simply uses +that format string to format the error. + +.. autoclass:: flake8.formatting.default.Default + :members: + +Pylint Formatter +================ + +The |PylintFormatter| simply defines the default Pylint format string from +pep8: ``'%(path)s:%(row)d: [%(code)s] %(text)s'``. + +.. autoclass:: flake8.formatting.default.Pylint + :members: + + +.. |DefaultFormatter| replace:: :class:`~flake8.formatting.default.Default` +.. |PylintFormatter| replace:: :class:`~flake8.formatting.default.Pylint` diff --git a/docs/source/internal/index.rst b/docs/source/internal/index.rst new file mode 100644 index 0000000..482b898 --- /dev/null +++ b/docs/source/internal/index.rst @@ -0,0 +1,26 @@ +============================== + Exploring Flake8's Internals +============================== + +While writing |Flake8| 3.0, the developers attempted to capture some reasoning +and decision information in internal documentation meant for future developers +and maintaners. Most of this information is unnecessary for users and plugin +developers. Some of it, however, is linked to from the plugin development +documentation. + +Keep in mind that not everything will be here and you may need to help pull +information out of the developers' heads and into these documents. Please +pull gently. + +.. toctree:: + :maxdepth: 2 + + contributing + writing-documentation + releases + checker + cli + formatters + option_handling + plugin_handling + utils diff --git a/docs/source/internal/option_handling.rst b/docs/source/internal/option_handling.rst new file mode 100644 index 0000000..74ecb76 --- /dev/null +++ b/docs/source/internal/option_handling.rst @@ -0,0 +1,234 @@ +Option and Configuration Handling +================================= + +Option Management +----------------- + +Command-line options are often also set in configuration files for |Flake8|. +While not all options are meant to be parsed from configuration files, many +default options are also parsed from configuration files as well as +most plugin options. + +In |Flake8| 2, plugins received a :class:`optparse.OptionParser` instance and +called :meth:`optparse.OptionParser.add_option` to register options. If the +plugin author also wanted to have that option parsed from config files they +also had to do something like: + +.. code-block:: python + + parser.config_options.append('my_config_option') + parser.config_options.extend(['config_opt1', 'config_opt2']) + +This was previously undocumented and led to a lot of confusion about why +registered options were not automatically parsed from configuration files. + +Since |Flake8| 3 was rewritten from scratch, we decided to take a different +approach to configuration file parsing. Instead of needing to know about an +undocumented attribute that pep8 looks for, |Flake8| 3 now accepts a parameter +to ``add_option``, specifically ``parse_from_config`` which is a boolean +value. + +|Flake8| does this by creating its own abstractions on top of :mod:`optparse`. +The first abstraction is the :class:`flake8.options.manager.Option` class. The +second is the :class:`flake8.options.manager.OptionManager`. In fact, we add +three new parameters: + +- ``parse_from_config`` + +- ``comma_separated_list`` + +- ``normalize_paths`` + +The last two are not specifically for configuration file handling, but they +do improve that dramatically. We found that there were options that, when +specified in a configuration file, often necessitated being spit +multiple lines and those options were almost always comma-separated. For +example, let's consider a user's list of ignored error codes for a project: + +.. code-block:: ini + + [flake8] + ignore = + # Reasoning + E111, + # Reasoning + E711, + # Reasoning + E712, + # Reasoning + E121, + # Reasoning + E122, + # Reasoning + E123, + # Reasoning + E131, + # Reasoning + E251 + +It makes sense here to allow users to specify the value this way, but, the +standard libary's :class:`configparser.RawConfigParser` class does returns a +string that looks like + +.. code-block:: python + + "\nE111, \nE711, \nE712, \nE121, \nE122, \nE123, \nE131, \nE251 " + +This means that a typical call to :meth:`str.split` with ``','`` will not be +sufficient here. Telling |Flake8| that something is a comma-separated list +(e.g., ``comma_separated_list=True``) will handle this for you. |Flake8| will +return: + +.. code-block:: python + + ["E111", "E711", "E712", "E121", "E122", "E123", "E131", "E251"] + +Next let's look at how users might like to specify their ``exclude`` list. +Presently OpenStack's Nova project has this line in their `tox.ini`_: + +.. code-block:: ini + + exclude = .venv,.git,.tox,dist,doc,*openstack/common/*,*lib/python*,*egg,build,tools/xenserver*,releasenotes + +We think we can all agree that this would be easier to read like this: + +.. code-block:: ini + + exclude = + .venv, + .git, + .tox, + dist, + doc, + *openstack/common/*, + *lib/python*, + *egg, + build, + tools/xenserver*, + releasenotes + +In this case, since these are actually intended to be paths, we would specify +both ``comma_separated_list=True`` and ``normalize_paths=True`` because we +want the paths to be provided to us with some consistency (either all absolute +paths or not). + +Now let's look at how this will actually be used. Most plugin developers +will receive an instance of :class:`~flake8.options.manager.OptionManager` so +to ease the transition we kept the same API as the +:class:`optparse.OptionParser` object. The only difference is that +:meth:`~flake8.options.manager.OptionManager.add_option` accepts the three +extra arguments we highlighted above. + +.. _tox.ini: + https://github.com/openstack/nova/blob/3eb190c4cfc0eefddac6c2cc1b94a699fb1687f8/tox.ini#L155 + +Configuration File Management +----------------------------- + +In |Flake8| 2, configuration file discovery and management was handled by +pep8. In pep8's 1.6 release series, it drastically broke how discovery and +merging worked (as a result of trying to improve it). To avoid a dependency +breaking |Flake8| again in the future, we have created our own discovery and +management. +As part of managing this ourselves, we decided to change management/discovery +for 3.0.0. We have done the following: + +- User files (files stored in a user's home directory or in the XDG directory + inside their home directory) are the first files read. For example, if the + user has a ``~/.flake8`` file, we will read that first. + +- Project files (files stored in the current directory) are read next and + merged on top of the user file. In other words, configuration in project + files takes precedence over configuration in user files. + +- **New in 3.0.0** The user can specify ``--append-config `` + repeatedly to include extra configuration files that should be read and + take precedence over user and project files. + +- **New in 3.0.0** The user can specify ``--config `` to so this + file is the only configuration file used. This is a change from |Flake8| 2 + where pep8 would simply merge this configuration file into the configuration + generated by user and project files (where this takes precedence). + +- **New in 3.0.0** The user can specify ``--isolated`` to disable + configuration via discovered configuration files. + +To facilitate the configuration file management, we've taken a different +approach to discovery and management of files than pep8. In pep8 1.5, 1.6, and +1.7 configuration discovery and management was centralized in `66 lines of +very terse python`_ which was confusing and not very explicit. The terseness +of this function (|Flake8|'s authors believe) caused the confusion and +problems with pep8's 1.6 series. As such, |Flake8| has separated out +discovery, management, and merging into a module to make reasoning about each +of these pieces easier and more explicit (as well as easier to test). + +Configuration file discovery is managed by the +:class:`~flake8.options.config.ConfigFileFinder` object. This object needs to +know information about the program's name, any extra arguments passed to it, +and any configuration files that should be appended to the list of discovered +files. It provides methods for finding the files and similiar methods for +parsing those fles. For example, it provides +:meth:`~flake8.options.config.ConfigFileFinder.local_config_files` to find +known local config files (and append the extra configuration files) and it +also provides :meth:`~flake8.options.config.ConfigFileFinder.local_configs` +to parse those configuration files. + +.. note:: ``local_config_files`` also filters out non-existent files. + +Configuration file merging and managemnt is controlled by the +:class:`~flake8.options.config.MergedConfigParser`. This requires the instance +of :class:`~flake8.options.manager.OptionManager` that the program is using, +the list of appended config files, and the list of extra arguments. This +object is currently the sole user of the +:class:`~flake8.options.config.ConfigFileFinder` object. It appropriately +initializes the object and uses it in each of + +- :meth:`~flake8.options.config.MergedConfigParser.parse_cli_config` +- :meth:`~flake8.options.config.MergedConfigParser.parse_local_config` +- :meth:`~flake8.options.config.MergedConfigParser.parse_user_config` + +Finally, +:meth:`~flake8.options.config.MergedConfigParser.merge_user_and_local_config` +takes the user and local configuration files that are parsed by +:meth:`~flake8.options.config.MergedConfigParser.parse_local_config` and +:meth:`~flake8.options.config.MergedConfigParser.parse_user_config`. The +main usage of the ``MergedConfigParser`` is in +:func:`~flake8.options.aggregator.aggregate_options`. + +Aggregating Configuration File and Command Line Arguments +--------------------------------------------------------- + +:func:`~flake8.options.aggregator.aggregate_options` accepts an instance of +:class:`~flake8.options.maanger.OptionManager` and does the work to parse the +command-line arguments passed by the user necessary for creating an instance +of :class:`~flake8.options.config.MergedConfigParser`. + +After parsing the configuration file, we determine the default ignore list. We +use the defaults from the OptionManager and update those with the parsed +configuration files. Finally we parse the user-provided options one last time +using the option defaults and configuration file values as defaults. The +parser merges on the command-line specified arguments for us so we have our +final, definitive, aggregated options. + +.. _66 lines of very terse python: + https://github.com/PyCQA/pep8/blob/b8088a2b6bc5b76bece174efad877f764529bc74/pep8.py#L1981..L2047 + +API Documentation +----------------- + +.. autofunction:: flake8.options.aggregator.aggregate_options + +.. autoclass:: flake8.options.manager.Option + :members: __init__, normalize, to_optparse + +.. autoclass:: flake8.options.manager.OptionManager + :members: + :special-members: + +.. autoclass:: flake8.options.config.ConfigFileFinder + :members: + :special-members: + +.. autoclass:: flake8.options.config.MergedConfigParser + :members: + :special-members: diff --git a/docs/source/internal/plugin_handling.rst b/docs/source/internal/plugin_handling.rst new file mode 100644 index 0000000..9af3182 --- /dev/null +++ b/docs/source/internal/plugin_handling.rst @@ -0,0 +1,129 @@ +Plugin Handling +=============== + +Plugin Management +----------------- + +|Flake8| 3.0 added support for two other plugins besides those which define +new checks. It now supports: + +- extra checks + +- alternative report formatters + +- listeners to auto-correct violations of checks + +To facilitate this, |Flake8| needed a more mature way of managing plugins. +Thus, we developed the |PluginManager| which accepts a namespace and will load +the plugins for that namespace. A |PluginManager| creates and manages many +|Plugin| instances. + +A |Plugin| lazily loads the underlying entry-point provided by setuptools. +The entry-point will be loaded either by calling +:meth:`~flake8.plugins.manager.Plugin.load_plugin` or accessing the ``plugin`` +attribute. We also use this abstraction to retrieve options that the plugin +wishes to register and parse. + +The only public method the |PluginManager| provides is +:meth:`~flake8.plugins.manager.PluginManager.map`. This will accept a function +(or other callable) and call it with each plugin as the first parameter. + +We build atop the |PluginManager| with the |PTM|. It is expected that users of +the |PTM| will subclass it and specify the ``namespace``, e.g., + +.. code-block:: python + + class ExamplePluginType(flake8.plugin.manager.PluginTypeManager): + namespace = 'example-plugins' + +This provides a few extra methods via the |PluginManager|'s ``map`` method. + +Finally, we create three classes of plugins: + +- :class:`~flake8.plugins.manager.Checkers` + +- :class:`~flake8.plugins.manager.Listeners` + +- :class:`~flake8.plugins.manager.ReportFormatters` + +These are used to interact with each of the types of plugins individually. + +.. note:: + + Our inspiration for our plugin handling comes from the author's extensive + experience with ``stevedore``. + +Notifying Listener Plugins +-------------------------- + +One of the interesting challenges with allowing plugins to be notified each +time an error or warning is emitted by a checker is finding listeners quickly +and efficiently. It makes sense to allow a listener to listen for a certain +class of warnings or just a specific warning. Hence, we need to allow all +plugins that listen to a specific warning or class to be notified. For +example, someone might register a listener for ``E1`` and another for ``E111`` +if ``E111`` is triggered by the code, both listeners should be notified. +If ``E112`` is returned, then only ``E1`` (and any other listeners) would be +notified. + +To implement this goal, we needed an object to store listeners in that would +allow for efficient look up - a Trie (or Prefix Tree). Given that none of the +existing packages on PyPI allowed for storing data on each node of the trie, +it was left up to write our own as :class:`~flake8.plugins._trie.Trie`. On +top of that we layer our :class:`~flake8.plugins.notifier.Notifier` class. + +Now when |Flake8| receives an error or warning, we can easily call the +:meth:`~flake8.plugins.notifier.Notifier.notify` method and let plugins act on +that knowledge. + +Default Plugins +--------------- + +Finally, |Flake8| has always provided its own plugin shim for Pyflakes. As +part of that we carry our own shim in-tree and now store that in +:mod:`flake8.plugins.pyflakes`. + +|Flake8| also registers plugins for pep8. Each check in pep8 requires +different parameters and it cannot easily be shimmed together like Pyflakes +was. As such, plugins have a concept of a "group". If you look at our +:file:`setup.py` you will see that we register pep8 checks roughly like so: + +.. code:: + + pep8. = pep8: + +We do this to identify that ``>`` is part of a group. This also +enables us to special-case how we handle reporting those checks. Instead of +reporting each check in the ``--version`` output, we report ``pep8`` and check +``pep8`` the module for a ``__version__`` attribute. We only report it once +to avoid confusing users. + +API Documentation +----------------- + +.. autoclass:: flake8.plugins.manager.PluginManager + :members: + :special-members: __init__, __contains__, __getitem__ + +.. autoclass:: flake8.plugins.manager.Plugin + :members: + :special-members: __init__ + +.. autoclass:: flake8.plugins.manager.PluginTypeManager + :members: + +.. autoclass:: flake8.plugins.manager.Checkers + :members: + +.. autoclass:: flake8.plugins.manager.Listeners + :members: build_notifier + +.. autoclass:: flake8.plugins.manager.ReportFormatters + +.. autoclass:: flake8.plugins.notifier.Notifier + +.. autoclass:: flake8.plugins._trie.Trie + +.. |PluginManager| replace:: :class:`~flake8.plugins.manager.PluginManager` +.. |Plugin| replace:: :class:`~flake8.plugins.manager.Plugin` +.. |PTM| replace:: :class:`~flake8.plugins.manager.PluginTypeManager` diff --git a/docs/source/internal/releases.rst b/docs/source/internal/releases.rst new file mode 100644 index 0000000..a624930 --- /dev/null +++ b/docs/source/internal/releases.rst @@ -0,0 +1,99 @@ +================== + Releasing Flake8 +================== + +There is not much that is hard to find about how |Flake8| is released. + +- We use **major** releases (e.g., 2.0.0, 3.0.0, etc.) for big, potentially + backwards incompatible, releases. + +- We use **minor** releases (e.g., 2.1.0, 2.2.0, 3.1.0, 3.2.0, etc.) for + releases that contain features and dependency version changes. + +- We use **patch** releases (e.g., 2.1.1, 2.1.2, 3.0.1, 3.0.10, etc.) for + releases that contain *only* bug fixes. + +In this sense we follow semantic versioning. But we follow it as more of a set +of guidelines. We're also not perfect, so we may make mistakes, and that's +fine. + + +Major Releases +============== + +Major releases are often associated with backwards incompatibility. |Flake8| +hopes to avoid those, but will occasionally need them. + +Historically, |Flake8| has generated major releases for: + +- Unvendoring dependencies (2.0) + +- Large scale refactoring (2.0, 3.0) + +- Subtly breaking CLI changes (3.0) + +- Breaking changes to its plugin interface (3.0) + +Major releases can also contain: + +- Bug fixes (which may have backwards incompatible solutions) + +- New features + +- Dependency changes + + +Minor Releases +============== + +Minor releases often have new features in them, which we define roughly as: + +- New command-line flags + +- New behaviour that does not break backwards compatibility + +- New errors detected by dependencies, e.g., by raising the upper limit on + PyFlakes we introduce F405 + +- Bug fixes + + +Patch Releases +============== + +Patch releases should only ever have bug fixes in them. + +We do not update dependency constraints in patch releases. If you do not +install |Flake8| from PyPI, there is a chance that your packager is using +different requirements. Some downstream redistributors have been known to +force a new version of PyFlakes, pep8/PyCodestyle, or McCabe into place. +Occasionally this will cause breakage when using |Flake8|. There is little +we can do to help you in those cases. + + +Process +======= + +To prepare a release, we create a file in :file:`docs/source/releases/` named: +``{{ release_number }}.rst`` (e.g., ``3.0.0.rst``). We note bug fixes, +improvements, and dependency version changes as well as other items of note +for users. + +Before releasing, the following tox test environments must pass: + +- Python 2.7 (a.k.a., ``tox -e py27``) + +- Python 3.4 (a.k.a., ``tox -e py34``) + +- Python 3.5 (a.k.a., ``tox -e py35``) + +- PyPy (a.k.a., ``tox -e pypy``) + +- Linters (a.k.a., ``tox -e linters``) + +We tag the most recent commit that passes those items and contains our release +notes. + +Finally, we run ``tox -e release`` to build source distributions (e.g., +``flake8-3.0.0.tar.gz``), universal wheels, and upload them to PyPI with +Twine. diff --git a/docs/source/internal/utils.rst b/docs/source/internal/utils.rst new file mode 100644 index 0000000..1b2bb1c --- /dev/null +++ b/docs/source/internal/utils.rst @@ -0,0 +1,127 @@ +=================== + Utility Functions +=================== + +|Flake8| has a few utility functions that it uses internally. + +.. warning:: + + As should be implied by where these are documented, these are all + **internal** utility functions. Their signatures and return types + may change between releases without notice. + + Bugs reported about these **internal** functions will be closed + immediately. + + If functions are needed by plugin developers, they may be requested + in the bug tracker and after careful consideration they *may* be added + to the *documented* stable API. + +.. autofunction:: flake8.utils.parse_comma_separated_list + +:func:`~flake8.utils.parse_comma_separated_list` takes either a string like + +.. code-block:: python + + "E121,W123,F904" + "E121,\nW123,\nF804" + "E121,\n\tW123,\n\tF804" + +Or it will take a list of strings (potentially with whitespace) such as + +.. code-block:: python + + [" E121\n", "\t\nW123 ", "\n\tF904\n "] + +And converts it to a list that looks as follows + +.. code-block:: python + + ["E121", "W123", "F904"] + +This function helps normalize any kind of comma-separated input you or |Flake8| +might receive. This is most helpful when taking advantage of |Flake8|'s +additional parameters to :class:`~flake8.options.manager.Option`. + +.. autofunction:: flake8.utils.normalize_path + +This utility takes a string that represents a path and returns the absolute +path if the string has a ``/`` in it. It also removes trailing ``/``\ s. + +.. autofunction:: flake8.utils.normalize_paths + +This function utilizes :func:`~flake8.utils.parse_comma_separated_list` and +:func:`~flake8.utils.normalize_path` to normalize it's input to a list of +strings that should be paths. + +.. autofunction:: flake8.utils.stdin_get_value + +This function retrieves and caches the value provided on ``sys.stdin``. This +allows plugins to use this to retrieve ``stdin`` if necessary. + +.. autofunction:: flake8.utils.is_windows + +This provides a convenient and explicitly named function that checks if we are +currently running on a Windows (or ``nt``) operating system. + +.. autofunction:: flake8.utils.can_run_multiprocessing_on_windows + +This provides a separate and distinct check from +:func:`~flake8.utils.is_windows` that allows us to check if the version of +Python we're using can actually use multiprocessing on Windows. + +.. autofunction:: flake8.utils.is_using_stdin + +Another helpful function that is named only to be explicit given it is a very +trivial check, this checks if the user specified ``-`` in their arguments to +|Flake8| to indicate we should read from stdin. + +.. autofunction:: flake8.utils.filenames_from + +When provided an argument to |Flake8|, we need to be able to traverse +directories in a convenient manner. For example, if someone runs + +.. code:: + + $ flake8 flake8/ + +Then they want us to check all of the files in the directory ``flake8/``. This +function will handle that while also handling the case where they specify a +file like: + +.. code:: + + $ flake8 flake8/__init__.py + + +.. autofunction:: flake8.utils.fnmatch + +The standard library's :func:`fnmatch.fnmatch` is excellent at deciding if a +filename matches a single pattern. In our use case, however, we typically have +a list of patterns and want to know if the filename matches any of them. This +function abstracts that logic away with a little extra logic. + +.. autofunction:: flake8.utils.parameters_for + +|Flake8| analyzes the parameters to plugins to determine what input they are +expecting. Plugins may expect one of the following: + +- ``physical_line`` to receive the line as it appears in the file + +- ``logical_line`` to receive the logical line (not as it appears in the file) + +- ``tree`` to receive the abstract syntax tree (AST) for the file + +We also analyze the rest of the parameters to provide more detail to the +plugin. This function will return the parameters in a consistent way across +versions of Python and will handle both classes and functions that are used as +plugins. Further, if the plugin is a class, it will strip the ``self`` +argument so we can check the parameters of the plugin consistently. + +.. autofunction:: flake8.utils.parse_unified_diff + +To handle usage of :option:`flake8 --diff`, |Flake8| needs to be able +to parse the name of the files in the diff as well as the ranges indicated the +sections that have been changed. This function either accepts the diff as an +argument or reads the diff from standard-in. It then returns a dictionary with +filenames as the keys and sets of line numbers as the value. diff --git a/docs/source/internal/writing-documentation.rst b/docs/source/internal/writing-documentation.rst new file mode 100644 index 0000000..ee37517 --- /dev/null +++ b/docs/source/internal/writing-documentation.rst @@ -0,0 +1,183 @@ +.. _docs-style: + +================================== + Writing Documentation for Flake8 +================================== + +The maintainers of |Flake8| believe strongly in benefit of style guides. +Hence, for all contributors who wish to work on our documentation, we've +put together a loose set of guidelines and best practices when adding to +our documentation. + + +View the docs locally before submitting +======================================= + +You can and should generate the docs locally before you submit a pull request +with your changes. You can build the docs by running: + +.. prompt:: bash + + tox -e docs + +From the directory containing the ``tox.ini`` file (which also contains the +``docs/`` directory that this file lives in). + +.. note:: + + If the docs don't build locally, they will not build in our continuous + integration system. We will generally not merge any pull request that + fails continuous integration. + + +Run the docs linter tests before submitting +=========================================== + +You should run the ``doc8`` linter job before you're ready to commit and fix +any errors found. + + +Capitalize Flake8 in prose +========================== + +We believe that by capitalizing |Flake8| in prose, we can help reduce +confusion between the command-line usage of ``flake8`` and the project. + +We also have defined a global replacement ``|Flake8|`` that should be used +and will replace each instance with ``:program:`Flake8```. + + +Use the prompt directive for command-line examples +================================================== + +When documenting something on the command-line, use the ``.. prompt::`` +directive to make it easier for users to copy and paste into their terminal. + +Example: + +.. code-block:: restructuredtext + + .. prompt:: bash + + flake8 --select E123,W503 dir/ + flake8 --ignore E24,W504 dir + + +Wrap lines around 79 characters +=============================== + +We use a maximum line-length in our documentation that is similar to the +default in |Flake8|. Please wrap lines at 79 characters (or less). + + +Use two new-lines before new sections +===================================== + +After the final paragraph of a section and before the next section title, +use two new-lines to separate them. This makes reading the plain-text +document a little nicer. Sphinx ignores these when rendering so they have +no semantic meaning. + +Example: + +.. code-block:: restructuredtext + + Section Header + ============== + + Paragraph. + + + Next Section Header + =================== + + Paragraph. + + +Surround document titles with equal symbols +=========================================== + +To indicate the title of a document, we place an equal number of ``=`` symbols +on the lines before and after the title. For example: + +.. code-block:: restructuredtext + + ================================== + Writing Documentation for Flake8 + ================================== + +Note also that we "center" the title by adding a leading space and having +extra ``=`` symbols at the end of those lines. + + +Use the option template for new options +======================================= + +All of |Flake8|'s command-line options are documented in the User Guide. Each +option is documented individually using the ``.. option::`` directive provided +by Sphinx. At the top of the document, in a reStructuredText comment, is a +template that should be copied and pasted into place when documening new +options. + +.. note:: + + The ordering of the options page is the order that options are printed + in the output of: + + .. prompt:: bash + + flake8 --help + + Please insert your option documentation according to that order. + + +Use anchors for easy reference linking +====================================== + +Use link anchors to allow for other areas of the documentation to use the +``:ref:`` role for intralinking documentation. Example: + +.. code-block:: restructuredtext + + .. _use-anchors: + + Use anchors for easy reference linking + ====================================== + +.. code-block:: restructuredtext + + Somewhere in this paragraph we will :ref:`reference anchors + `. + +.. note:: + + You do not need to provide custom text for the ``:ref:`` if the title of + the section has a title that is sufficient. + + +Keep your audience in mind +========================== + +|Flake8|'s documentation has three distinct (but not separate) audiences: + +#. Users + +#. Plugin Developers + +#. Flake8 Developers and Contributors + +At the moment, you're one of the third group (because you're contributing +or thinking of contributing). + +Consider that most Users aren't very interested in the internal working of +|Flake8|. When writing for Users, focus on how to do something or the +behaviour of a certain piece of configuration or invocation. + +Plugin developers will only care about the internals of |Flake8| as much as +they will have to interact with that. Keep discussions of internal to the +mininmum required. + +Finally, Flake8 Developers and Contributors need to know how everything fits +together. We don't need detail about every line of code, but cogent +explanations and design specifications will help future developers understand +the Hows and Whys of |Flake8|'s internal design. diff --git a/docs/source/plugin-development/.keep b/docs/source/plugin-development/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/plugin-development/cross-compatibility.rst b/docs/source/plugin-development/cross-compatibility.rst new file mode 100644 index 0000000..1aa45e3 --- /dev/null +++ b/docs/source/plugin-development/cross-compatibility.rst @@ -0,0 +1,150 @@ +==================================== + Writing Plugins For Flake8 2 and 3 +==================================== + +Plugins have existed for |Flake8| 2.x for a few years. There are a number of +these on PyPI already. While it did not seem reasonable for |Flake8| to attempt +to provide a backwards compatible shim for them, we did decide to try to +document the easiest way to write a plugin that's compatible across both +versions. + +.. note:: + + If your plugin does not register options, it *should* Just Work. + +The **only** breaking change in |Flake8| 3.0 is the fact that we no longer +check the option parser for a list of strings to parse from a config file. On +|Flake8| 2.x, to have an option parsed from the configuration files that +|Flake8| finds and parses you would have to do something like: + +.. code-block:: python + + parser.add_option('-X', '--example-flag', type='string', + help='...') + parser.config_options.append('example-flag') + +For |Flake8| 3.0, we have added *three* arguments to the +:meth:`~flake8.options.manager.OptionManager.add_option` method you will call +on the parser you receive: + +- ``parse_from_config`` which expects ``True`` or ``False`` + + When ``True``, |Flake8| will parse the option from the config files |Flake8| + finds. + +- ``comma_separated_list`` which expects ``True`` or ``False`` + + When ``True``, |Flake8| will split the string intelligently and handle + extra whitespace. The parsed value will be a list. + +- ``normalize_paths`` which expects ``True`` or ``False`` + + When ``True``, |Flake8| will: + + * remove trailing path separators (i.e., ``os.path.sep``) + + * return the absolute path for values that have the separator in them + +All three of these options can be combined or used separately. + + +Parsing Options from Configuration Files +======================================== + +The example from |Flake8| 2.x now looks like: + +.. code-block:: python + + parser.add_option('-X', '--example-flag', type='string', + parse_from_config=True, + help='...') + + +Parsing Comma-Separated Lists +============================= + +Now let's imagine that the option we want to add is expecting a comma-separatd +list of values from the user (e.g., ``--select E123,W503,F405``). |Flake8| 2.x +often forced users to parse these lists themselves since pep8 special-cased +certain flags and left others on their own. |Flake8| 3.0 adds +``comma_separated_list`` so that the parsed option is already a list for +plugin authors. When combined with ``parse_from_config`` this means that users +can also do something like: + +.. code-block:: ini + + example-flag = + first, + second, + third, + fourth, + fifth + +And |Flake8| will just return the list: + +.. code-block:: python + + ["first", "second", "third", "fourth", "fifth"] + + +Normalizing Values that Are Paths +================================= + +Finally, let's imagine that our new option wants a path or list of paths. To +ensure that these paths are semi-normalized (the way |Flake8| 2.x used to +work) we need only pass ``normalize_paths=True``. If you have specified +``comma_separated_list=True`` then this will parse the value as a list of +paths that have been normalized. Otherwise, this will parse the value +as a single path. + + +Option Handling on Flake8 2 and 3 +================================= + +So, in conclusion, we can now write our plugin that relies on registering +options with |Flake8| and have it work on |Flake8| 2.x and 3.x. + +.. code-block:: python + + option_args = ('-X', '--example-flag') + option_kwargs = { + 'type': 'string', + 'parse_from_config': True, + 'help': '...', + } + try: + # Flake8 3.x registration + parser.add_option(*option_args, **option_kwargs) + except TypeError: + # Flake8 2.x registration + parse_from_config = option_kwargs.pop('parse_from_config', False) + parser.add_option(*option_args, **option_kwargs) + if parse_from_config: + parser.config_options.append(option_args[-1].lstrip('-')) + + +Or, you can write a tiny helper function: + +.. code-block:: python + + def register_opt(parser, *args, **kwargs): + try: + # Flake8 3.x registration + parser.add_option(*args, **kwargs) + except TypeError: + # Flake8 2.x registration + parse_from_config = kwargs.pop('parse_from_config', False) + parser.add_option(*args, **kwargs) + if parse_from_config: + parser.config_options.append(args[-1].lstrip('-')) + +.. code-block:: python + + @classmethod + def register_options(cls, parser): + register_opt(parser, '-X', '--example-flag', type='string', + parse_from_config=True, help='...') + +The transition period is admittedly not fantastic, but we believe that this +is a worthwhile change for plugin developers going forward. We also hope to +help with the transition phase for as many plugins as we can manage. diff --git a/docs/source/plugin-development/formatters.rst b/docs/source/plugin-development/formatters.rst new file mode 100644 index 0000000..480ada0 --- /dev/null +++ b/docs/source/plugin-development/formatters.rst @@ -0,0 +1,54 @@ +.. _formatting-plugins: + +=========================================== + Developing a Formatting Plugin for Flake8 +=========================================== + +|Flake8| allowed for custom formatting plugins in version +3.0.0. Let's write a plugin together: + +.. code-block:: python + + from flake8.formatting import base + + + class Example(base.BaseFormatter): + """Flake8's example formatter.""" + + pass + +We notice, as soon as we start, that we inherit from |Flake8|'s +:class:`~flake8.formatting.base.BaseFormatter` class. If we follow the +:ref:`instructions to register a plugin ` and try to use +our example formatter, e.g., ``flake8 --format=example`` then +|Flake8| will fail because we did not implement the ``format`` method. +Let's do that next. + +.. code-block:: python + + class Example(base.BaseFormatter): + """Flake8's example formatter.""" + + def format(self, error): + return 'Example formatter: {0!r}'.format(error) + +With that we're done. Obviously this isn't a very useful formatter, but it +should highlight the simplicitly of creating a formatter with Flake8. If we +wanted to instead create a formatter that aggregated the results and returned +XML, JSON, or subunit we could also do that. |Flake8| interacts with the +formatter in two ways: + +#. It creates the formatter and provides it the options parsed from the + configuration files and command-line + +#. It uses the instance of the formatter and calls ``handle`` with the error. + +By default :meth:`flake8.formatting.base.BaseFormatter.handle` simply calls +the ``format`` method and then ``write``. Any extra handling you wish to do +for formatting purposes should override the ``handle`` method. + +API Documentation +================= + +.. autoclass:: flake8.formatting.base.BaseFormatter + :members: diff --git a/docs/source/plugin-development/index.rst b/docs/source/plugin-development/index.rst new file mode 100644 index 0000000..c3efb1d --- /dev/null +++ b/docs/source/plugin-development/index.rst @@ -0,0 +1,56 @@ +============================ + Writing Plugins for Flake8 +============================ + +Since |Flake8| 2.0, the |Flake8| tool has allowed for extensions and custom +plugins. In |Flake8| 3.0, we're expanding that ability to customize and +extend **and** we're attempting to thoroughly document it. Some of the +documentation in this section may reference third-party documentation to +reduce duplication and to point you, the developer, towards the authoritative +documentation for those pieces. + +Getting Started +=============== + +To get started writing a |Flake8| :term:`plugin` you first need: + +- An idea for a plugin + +- An available package name on PyPI + +- One or more versions of Python installed + +- A text editor or IDE of some kind + +- An idea of what *kind* of plugin you want to build: + + * Formatter + + * Check + +Once you've gathered these things, you can get started. + +All plugins for |Flake8| must be registered via `entry points`_. In this +section we cover: + +- How to register your plugin so |Flake8| can find it + +- How to make |Flake8| provide your check plugin with information (via + command-line flags, function/class parameters, etc.) + +- How to make a formatter plugin + +- How to write your check plugin so that it works with |Flake8| 2.x and 3.x + +.. toctree:: + :caption: Plugin Developer Documentation + :maxdepth: 2 + + registering-plugins + plugin-parameters + formatters + cross-compatibility + + +.. _entry points: + https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points diff --git a/docs/source/plugin-development/plugin-parameters.rst b/docs/source/plugin-development/plugin-parameters.rst new file mode 100644 index 0000000..527950c --- /dev/null +++ b/docs/source/plugin-development/plugin-parameters.rst @@ -0,0 +1,163 @@ +.. _plugin-parameters: + +========================================== + Receiving Information For A Check Plugin +========================================== + +Plugins to |Flake8| have a great deal of information that they can request +from a :class:`~flake8.processor.FileProcessor` instance. Historically, +|Flake8| has supported two types of plugins: + +#. classes that accept parsed abstract syntax trees (ASTs) + +#. functions that accept a range of arguments + +|Flake8| now does not distinguish between the two types of plugins. Any plugin +can accept either an AST or a range of arguments. Further, any plugin that has +certain callable attributes can also register options and receive parsed +options. + + +Indicating Desired Data +======================= + +|Flake8| inspects the plugin's signature to determine what parameters it +expects using :func:`flake8.utils.parameters_for`. +:attr:`flake8.plugins.manager.Plugin.parameters` caches the values so that +each plugin makes that fairly expensive call once per plugin. When processing +a file, a plugin can ask for any of the following: + +- :attr:`~flake8.processor.FileProcessor.blank_before` +- :attr:`~flake8.processor.FileProcessor.blank_lines` +- :attr:`~flake8.processor.FileProcessor.checker_state` +- :attr:`~flake8.processor.FileProcessor.indect_char` +- :attr:`~flake8.processor.FileProcessor.indent_level` +- :attr:`~flake8.processor.FileProcessor.line_number` +- :attr:`~flake8.processor.FileProcessor.logical_line` +- :attr:`~flake8.processor.FileProcessor.max_line_length` +- :attr:`~flake8.processor.FileProcessor.multiline` +- :attr:`~flake8.processor.FileProcessor.noqa` +- :attr:`~flake8.processor.FileProcessor.previous_indent_level` +- :attr:`~flake8.processor.FileProcessor.previous_logical` +- :attr:`~flake8.processor.FileProcessor.tokens` +- :attr:`~flake8.processor.FileProcessor.total_lines` +- :attr:`~flake8.processor.FileProcessor.verbose` + +Alternatively, a plugin can accept ``tree`` and ``filename``. +``tree`` will be a parsed abstract syntax tree that will be used by plugins +like PyFlakes and McCabe. + + +Registering Options +=================== + +Any plugin that has callable attributes ``provide_options`` and +``register_options`` can parse option information and register new options. + +Your ``register_options`` function should expect to receive an instance of +|OptionManager|. An |OptionManager| instance behaves very similarly to +:class:`optparse.OptionParser`. It, however, uses the layer that |Flake8| has +developed on top of :mod:`optparse` to also handle configuration file parsing. +:meth:`~flake8.options.manager.OptionManager.add_option` creates an |Option| +which accepts the same parameters as :mod:`optparse` as well as three extra +boolean parameters: + +- ``parse_from_config`` + + The command-line option should also be parsed from config files discovered + by |Flake8|. + + .. note:: + + This takes the place of appending strings to a list on the + :class:`optparse.OptionParser`. + +- ``comma_separated_list`` + + The value provided to this option is a comma-separated list. After parsing + the value, it should be further broken up into a list. This also allows us + to handle values like: + + .. code:: + + E123,E124, + E125, + E126 + +- ``normalize_paths`` + + The value provided to this option is a path. It should be normalized to be + an absolute path. This can be combined with ``comma_separated_list`` to + allow a comma-separated list of paths. + +Each of these options works individually or can be combined. Let's look at a +couple examples from |Flake8|. In each example, we will have +``option_manager`` which is an instance of |OptionManager|. + +.. code-block:: python + + option_manager.add_option( + '--max-line-length', type='int', metavar='n', + default=defaults.MAX_LINE_LENGTH, parse_from_config=True, + help='Maximum allowed line length for the entirety of this run. ' + '(Default: %default)', + ) + +Here we are adding the ``--max-line-length`` command-line option which is +always an integer and will be parsed from the configuration file. Since we +provide a default, we take advantage of :mod:`optparse`\ 's willingness to +display that in the help text with ``%default``. + +.. code-block:: python + + option_manager.add_option( + '--select', metavar='errors', default='', + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to enable.' + ' For example, ``--select=E4,E51,W234``. (Default: %default)', + ) + +In adding the ``--select`` command-line option, we're also indicating to the +|OptionManager| that we want the value parsed from the config files and parsed +as a comma-separated list. + +.. code-block:: python + + option_manager.add_option( + '--exclude', metavar='patterns', default=defaults.EXCLUDE, + comma_separated_list=True, parse_from_config=True, + normalize_paths=True, + help='Comma-separated list of files or directories to exclude.' + '(Default: %default)', + ) + +Finally, we show an option that uses all three extra flags. Values from +``--exclude`` will be parsed from the config, converted from a comma-separated +list, and then each item will be normalized. + +For information about other parameters to +:meth:`~flake8.options.manager.OptionManager.add_option` refer to the +documentation of :mod:`optparse`. + + +Accessing Parsed Options +======================== + +When a plugin has a callable ``provide_options`` attribute, |Flake8| will call +it and attempt to provide the |OptionManager| instance, the parsed options +which will be an instance of :class:`optparse.Values`, and the extra arguments +that were not parsed by the |OptionManager|. If that fails, we will just pass +the :class:`optparse.Values`. In other words, your ``provide_options`` +callable will have one of the following signatures: + +.. code-block:: python + + def provide_options(option_manager, options, args): + pass + # or + def provide_options(options): + pass + +.. substitutions +.. |OptionManager| replace:: :class:`~flake8.options.manager.OptionManager` +.. |Option| replace:: :class:`~flake8.options.manager.Option` diff --git a/docs/source/plugin-development/registering-plugins.rst b/docs/source/plugin-development/registering-plugins.rst new file mode 100644 index 0000000..5d01f99 --- /dev/null +++ b/docs/source/plugin-development/registering-plugins.rst @@ -0,0 +1,115 @@ +.. _register-a-plugin: + +================================== + Registering a Plugin with Flake8 +================================== + +To register any kind of plugin with |Flake8|, you need: + +#. A way to install the plugin (whether it is packaged on its own or + as part of something else). In this section, we will use a ``setup.py`` + written for an example plugin. + +#. A name for your plugin that will (ideally) be unique. + +#. A somewhat recent version of setuptools (newer than 0.7.0 but preferably as + recent as you can attain). + +|Flake8| relies on functionality provided by setuptools called +`Entry Points`_. These allow any package to register a plugin with |Flake8| +via that package's ``setup.py`` file. + +Let's presume that we already have our plugin written and it's in a module +called ``flake8_example``. We might have a ``setup.py`` that looks something +like: + +.. code-block:: python + + from __future__ import with_statement + import setuptools + + requires = [ + "flake8 > 3.0.0", + ] + + flake8_entry_point = # ... + + setuptools.setup( + name="flake8_example", + license="MIT", + version="0.1.0", + description="our extension to flake8", + author="Me", + author_email="example@example.com", + url="https://gitlab.com/me/flake8_example", + packages=[ + "flake8_example", + ], + install_requires=requires, + entry_points={ + flake8_entry_point: [ + 'X = flake8_example:ExamplePlugin', + ], + }, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", + ], + ) + +Note specifically these lines: + +.. code-block:: python + + flake8_entry_point = # ... + + setuptools.setup( + # snip ... + entry_points={ + flake8_entry_point: [ + 'X = flake8_example:ExamplePlugin', + ], + }, + # snip ... + ) + +We tell setuptools to register our entry point "X" inside the specific +grouping of entry-points that flake8 should look in. + +|Flake8| presently looks at three groups: + +- ``flake8.extension`` + +- ``flake8.listen`` + +- ``flake8.report`` + +If your plugin is one that adds checks to |Flake8|, you will use +``flake8.extension``. If your plugin automatically fixes errors in code, you +will use ``flake8.listen``. Finally, if your plugin performs extra report +handling (formatting, filtering, etc.) it will use ``flake8.report``. + +If our ``ExamplePlugin`` is something that adds checks, our code would look +like: + +.. code-block:: python + + setuptools.setup( + # snip ... + entry_points={ + 'flake8.extension': [ + 'X = flake8_example:ExamplePlugin', + ], + }, + # snip ... + ) + + +.. _Entry Points: + https://pythonhosted.org/setuptools/pkg_resources.html#entry-points diff --git a/docs/source/release-notes/0.6.0.rst b/docs/source/release-notes/0.6.0.rst new file mode 100644 index 0000000..16590fa --- /dev/null +++ b/docs/source/release-notes/0.6.0.rst @@ -0,0 +1,4 @@ +0.6 - 2010-02-15 +---------------- + +- Fix the McCabe metric on some loops diff --git a/docs/source/release-notes/0.7.0.rst b/docs/source/release-notes/0.7.0.rst new file mode 100644 index 0000000..f842060 --- /dev/null +++ b/docs/source/release-notes/0.7.0.rst @@ -0,0 +1,6 @@ +0.7 - 2010-02-18 +---------------- + +- Fix pep8 initialization when run through Hg +- Make pep8 short options work when run through the command line +- Skip duplicates when controlling files via Hg diff --git a/docs/source/release-notes/0.8.0.rst b/docs/source/release-notes/0.8.0.rst new file mode 100644 index 0000000..66b90b5 --- /dev/null +++ b/docs/source/release-notes/0.8.0.rst @@ -0,0 +1,5 @@ +0.8 - 2011-02-27 +---------------- + +- fixed hg hook +- discard unexisting files on hook check diff --git a/docs/source/release-notes/0.9.0.rst b/docs/source/release-notes/0.9.0.rst new file mode 100644 index 0000000..be6c41b --- /dev/null +++ b/docs/source/release-notes/0.9.0.rst @@ -0,0 +1,5 @@ +0.9 - 2011-11-09 +---------------- + +- update pep8 version to 0.6.1 +- mccabe check: gracefully handle compile failure diff --git a/docs/source/release-notes/1.0.0.rst b/docs/source/release-notes/1.0.0.rst new file mode 100644 index 0000000..6882a43 --- /dev/null +++ b/docs/source/release-notes/1.0.0.rst @@ -0,0 +1,5 @@ +1.0 - 2011-11-29 +---------------- + +- Deactivates by default the complexity checker +- Introduces the complexity option in the HG hook and the command line. diff --git a/docs/source/release-notes/1.1.0.rst b/docs/source/release-notes/1.1.0.rst new file mode 100644 index 0000000..dadbe2e --- /dev/null +++ b/docs/source/release-notes/1.1.0.rst @@ -0,0 +1,8 @@ +1.1 - 2012-02-14 +---------------- + +- fixed the value returned by --version +- allow the flake8: header to be more generic +- fixed the "hg hook raises 'physical lines'" bug +- allow three argument form of raise +- now uses setuptools if available, for 'develop' command diff --git a/docs/source/release-notes/1.2.0.rst b/docs/source/release-notes/1.2.0.rst new file mode 100644 index 0000000..de86237 --- /dev/null +++ b/docs/source/release-notes/1.2.0.rst @@ -0,0 +1,6 @@ +1.2 - 2012-02-12 +---------------- + +- added a git hook +- now Python 3 compatible +- mccabe and pyflakes have warning codes like pep8 now diff --git a/docs/source/release-notes/1.3.0.rst b/docs/source/release-notes/1.3.0.rst new file mode 100644 index 0000000..0ddfe78 --- /dev/null +++ b/docs/source/release-notes/1.3.0.rst @@ -0,0 +1,4 @@ +1.3 - 2012-03-12 +---------------- + +- fixed false W402 warning on exception blocks. diff --git a/docs/source/release-notes/1.3.1.rst b/docs/source/release-notes/1.3.1.rst new file mode 100644 index 0000000..b2e34ee --- /dev/null +++ b/docs/source/release-notes/1.3.1.rst @@ -0,0 +1,4 @@ +1.3.1 - 2012-05-19 +------------------ + +- fixed support for Python 2.5 diff --git a/docs/source/release-notes/1.4.0.rst b/docs/source/release-notes/1.4.0.rst new file mode 100644 index 0000000..51f34cf --- /dev/null +++ b/docs/source/release-notes/1.4.0.rst @@ -0,0 +1,5 @@ +1.4 - 2012-07-12 +---------------- + +- git_hook: Only check staged changes for compliance +- use pep8 1.2 diff --git a/docs/source/release-notes/1.5.0.rst b/docs/source/release-notes/1.5.0.rst new file mode 100644 index 0000000..cd0a88d --- /dev/null +++ b/docs/source/release-notes/1.5.0.rst @@ -0,0 +1,9 @@ +1.5 - 2012-10-13 +---------------- + +- fixed the stdin +- make sure mccabe catches the syntax errors as warnings +- pep8 upgrade +- added max_line_length default value +- added Flake8Command and entry points if setuptools is around +- using the setuptools console wrapper when available diff --git a/docs/source/release-notes/1.6.0.rst b/docs/source/release-notes/1.6.0.rst new file mode 100644 index 0000000..658be2f --- /dev/null +++ b/docs/source/release-notes/1.6.0.rst @@ -0,0 +1,14 @@ +1.6 - 2012-11-16 +---------------- + +- changed the signatures of the ``check_file`` function in flake8/run.py, + ``skip_warning`` in flake8/util.py and the ``check``, ``checkPath`` + functions in flake8/pyflakes.py. +- fix ``--exclude`` and ``--ignore`` command flags (#14, #19) +- fix the git hook that wasn't catching files not already added to the index + (#29) +- pre-emptively includes the addition to pep8 to ignore certain lines. + Add ``# nopep8`` to the end of a line to ignore it. (#37) +- ``check_file`` can now be used without any special prior setup (#21) +- unpacking exceptions will no longer cause an exception (#20) +- fixed crash on non-existent file (#38) diff --git a/docs/source/release-notes/1.6.1.rst b/docs/source/release-notes/1.6.1.rst new file mode 100644 index 0000000..194dfa5 --- /dev/null +++ b/docs/source/release-notes/1.6.1.rst @@ -0,0 +1,7 @@ +1.6.1 - 2012-11-24 +------------------ + +- fixed the mercurial hook, a change from a previous patch was not properly + applied +- fixed an assumption about warnings/error messages that caused an exception + to be thrown when McCabe is used diff --git a/docs/source/release-notes/1.6.2.rst b/docs/source/release-notes/1.6.2.rst new file mode 100644 index 0000000..64633bd --- /dev/null +++ b/docs/source/release-notes/1.6.2.rst @@ -0,0 +1,4 @@ +1.6.2 - 2012-11-25 +------------------ + +- fixed the NameError: global name 'message' is not defined (#46) diff --git a/docs/source/release-notes/1.7.0.rst b/docs/source/release-notes/1.7.0.rst new file mode 100644 index 0000000..a3a4725 --- /dev/null +++ b/docs/source/release-notes/1.7.0.rst @@ -0,0 +1,9 @@ +1.7.0 - 2012-12-21 +------------------ + +- Fixes part of #35: Exception for no WITHITEM being an attribute of Checker + for Python 3.3 +- Support stdin +- Incorporate @phd's builtins pull request +- Fix the git hook +- Update pep8.py to the latest version diff --git a/docs/source/release-notes/2.0.0.rst b/docs/source/release-notes/2.0.0.rst new file mode 100644 index 0000000..4c1ff04 --- /dev/null +++ b/docs/source/release-notes/2.0.0.rst @@ -0,0 +1,13 @@ +2.0.0 - 2013-02-23 +------------------ + +- Pyflakes errors are prefixed by an ``F`` instead of an ``E`` +- McCabe complexity warnings are prefixed by a ``C`` instead of a ``W`` +- Flake8 supports extensions through entry points +- Due to the above support, we **require** setuptools +- We publish the `documentation `_ +- Fixes #13: pep8, pyflakes and mccabe become external dependencies +- Split run.py into main.py, engine.py and hooks.py for better logic +- Expose our parser for our users +- New feature: Install git and hg hooks automagically +- By relying on pyflakes (0.6.1), we also fixed #45 and #35 diff --git a/docs/source/release-notes/2.1.0.rst b/docs/source/release-notes/2.1.0.rst new file mode 100644 index 0000000..c9e3c60 --- /dev/null +++ b/docs/source/release-notes/2.1.0.rst @@ -0,0 +1,12 @@ +2.1.0 - 2013-10-26 +------------------ + +- Add FLAKE8_LAZY and FLAKE8_IGNORE environment variable support to git and + mercurial hooks +- Force git and mercurial hooks to repsect configuration in setup.cfg +- Only check staged files if that is specified +- Fix hook file permissions +- Fix the git hook on python 3 +- Ignore non-python files when running the git hook +- Ignore .tox directories by default +- Flake8 now reports the column number for PyFlakes messages diff --git a/docs/source/release-notes/2.2.0.rst b/docs/source/release-notes/2.2.0.rst new file mode 100644 index 0000000..357b6d2 --- /dev/null +++ b/docs/source/release-notes/2.2.0.rst @@ -0,0 +1,12 @@ +2.2.0 - 2014-06-22 +------------------ + +- New option ``doctests`` to run Pyflakes checks on doctests too +- New option ``jobs`` to launch multiple jobs in parallel +- Turn on using multiple jobs by default using the CPU count +- Add support for ``python -m flake8`` on Python 2.7 and Python 3 +- Fix Git and Mercurial hooks: issues #88, #133, #148 and #149 +- Fix crashes with Python 3.4 by upgrading dependencies +- Fix traceback when running tests with Python 2.6 +- Fix the setuptools command ``python setup.py flake8`` to read + the project configuration diff --git a/docs/source/release-notes/2.2.1.rst b/docs/source/release-notes/2.2.1.rst new file mode 100644 index 0000000..5575f8f --- /dev/null +++ b/docs/source/release-notes/2.2.1.rst @@ -0,0 +1,5 @@ +2.2.1 - 2014-06-30 +------------------ + +- Turn off multiple jobs by default. To enable automatic use of all CPUs, use + ``--jobs=auto``. Fixes #155 and #154. diff --git a/docs/source/release-notes/2.2.2.rst b/docs/source/release-notes/2.2.2.rst new file mode 100644 index 0000000..8fcff88 --- /dev/null +++ b/docs/source/release-notes/2.2.2.rst @@ -0,0 +1,5 @@ +2.2.2 - 2014-07-04 +------------------ + +- Re-enable multiprocessing by default while fixing the issue Windows users + were seeing. diff --git a/docs/source/release-notes/2.2.3.rst b/docs/source/release-notes/2.2.3.rst new file mode 100644 index 0000000..e7430f0 --- /dev/null +++ b/docs/source/release-notes/2.2.3.rst @@ -0,0 +1,4 @@ +2.2.3 - 2014-08-25 +------------------ + +- Actually turn multiprocessing on by default diff --git a/docs/source/release-notes/2.2.4.rst b/docs/source/release-notes/2.2.4.rst new file mode 100644 index 0000000..2564948 --- /dev/null +++ b/docs/source/release-notes/2.2.4.rst @@ -0,0 +1,20 @@ +2.2.4 - 2014-10-09 +------------------ + +- Fix bugs triggered by turning multiprocessing on by default (again) + + Multiprocessing is forcibly disabled in the following cases: + + - Passing something in via stdin + + - Analyzing a diff + + - Using windows + +- Fix --install-hook when there are no config files present for pep8 or + flake8. + +- Fix how the setuptools command parses excludes in config files + +- Fix how the git hook determines which files to analyze (Thanks Chris + Buccella!) diff --git a/docs/source/release-notes/2.2.5.rst b/docs/source/release-notes/2.2.5.rst new file mode 100644 index 0000000..540278f --- /dev/null +++ b/docs/source/release-notes/2.2.5.rst @@ -0,0 +1,6 @@ +2.2.5 - 2014-10-19 +------------------ + +- Flush standard out when using multiprocessing + +- Make the check for "# flake8: noqa" more strict diff --git a/docs/source/release-notes/2.3.0.rst b/docs/source/release-notes/2.3.0.rst new file mode 100644 index 0000000..120efa9 --- /dev/null +++ b/docs/source/release-notes/2.3.0.rst @@ -0,0 +1,10 @@ +2.3.0 - 2015-01-04 +------------------ + +- **Feature**: Add ``--output-file`` option to specify a file to write to + instead of ``stdout``. + +- **Bug** Fix interleaving of output while using multiprocessing + (`GitLab#17`_) + +.. _GitLab#17: https://gitlab.com/pycqa/flake8/issues/17 diff --git a/docs/source/release-notes/2.4.0.rst b/docs/source/release-notes/2.4.0.rst new file mode 100644 index 0000000..6c470cd --- /dev/null +++ b/docs/source/release-notes/2.4.0.rst @@ -0,0 +1,33 @@ +2.4.0 - 2015-03-07 +------------------ + +- **Bug** Print filenames when using multiprocessing and ``-q`` option. + (`GitLab#31`_) + +- **Bug** Put upper cap on dependencies. The caps for 2.4.0 are: + + - ``pep8 < 1.6`` (Related to `GitLab#35`_) + + - ``mccabe < 0.4`` + + - ``pyflakes < 0.9`` + + See also `GitLab#32`_ + +- **Bug** Files excluded in a config file were not being excluded when flake8 + was run from a git hook. (`GitHub#2`_) + +- **Improvement** Print warnings for users who are providing mutually + exclusive options to flake8. (`GitLab#8`_, `GitLab!18`_) + +- **Feature** Allow git hook configuration to live in ``.git/config``. + See the updated `VCS hooks docs`_ for more details. (`GitLab!20`_) + +.. _GitHub#2: https://github.com/pycqa/flake8/pull/2 +.. _GitLab#8: https://gitlab.com/pycqa/flake8/issues/8 +.. _GitLab#31: https://gitlab.com/pycqa/flake8/issues/31 +.. _GitLab#32: https://gitlab.com/pycqa/flake8/issues/32 +.. _GitLab#35: https://gitlab.com/pycqa/flake8/issues/35 +.. _GitLab!18: https://gitlab.com/pycqa/flake8/merge_requests/18 +.. _GitLab!20: https://gitlab.com/pycqa/flake8/merge_requests/20 +.. _VCS hooks docs: https://flake8.readthedocs.org/en/latest/vcs.html diff --git a/docs/source/release-notes/2.4.1.rst b/docs/source/release-notes/2.4.1.rst new file mode 100644 index 0000000..3448bc4 --- /dev/null +++ b/docs/source/release-notes/2.4.1.rst @@ -0,0 +1,12 @@ +2.4.1 - 2015-05-18 +------------------ + +- **Bug** Do not raise a ``SystemError`` unless there were errors in the + setuptools command. (`GitLab#39`_, `GitLab!23`_) + +- **Bug** Do not verify dependencies of extensions loaded via entry-points. + +- **Improvement** Blacklist versions of pep8 we know are broken + +.. _GitLab#39: https://gitlab.com/pycqa/flake8/issues/39 +.. _GitLab!23: https://gitlab.com/pycqa/flake8/merge_requests/23 diff --git a/docs/source/release-notes/2.5.0.rst b/docs/source/release-notes/2.5.0.rst new file mode 100644 index 0000000..1558fcf --- /dev/null +++ b/docs/source/release-notes/2.5.0.rst @@ -0,0 +1,25 @@ +2.5.0 - 2015-10-26 +------------------ + +- **Improvement** Raise cap on PyFlakes for Python 3.5 support + +- **Improvement** Avoid deprecation warnings when loading extensions + (`GitLab#59`_, `GitLab#90`_) + +- **Improvement** Separate logic to enable "off-by-default" extensions + (`GitLab#67`_) + +- **Bug** Properly parse options to setuptools Flake8 command (`GitLab!41`_) + +- **Bug** Fix exceptions when output on stdout is truncated before Flake8 + finishes writing the output (`GitLab#69`_) + +- **Bug** Fix error on OS X where Flake8 can no longer acquire or create new + semaphores (`GitLab#74`_) + +.. _GitLab!41: https://gitlab.com/pycqa/flake8/merge_requests/41 +.. _GitLab#59: https://gitlab.com/pycqa/flake8/issues/59 +.. _GitLab#67: https://gitlab.com/pycqa/flake8/issues/67 +.. _GitLab#69: https://gitlab.com/pycqa/flake8/issues/69 +.. _GitLab#74: https://gitlab.com/pycqa/flake8/issues/74 +.. _GitLab#90: https://gitlab.com/pycqa/flake8/issues/90 diff --git a/docs/source/release-notes/2.5.1.rst b/docs/source/release-notes/2.5.1.rst new file mode 100644 index 0000000..8a114c8 --- /dev/null +++ b/docs/source/release-notes/2.5.1.rst @@ -0,0 +1,13 @@ +2.5.1 - 2015-12-08 +------------------ + +- **Bug** Properly look for ``.flake8`` in current working directory + (`GitLab#103`_) + +- **Bug** Monkey-patch ``pep8.stdin_get_value`` to cache the actual value in + stdin. This helps plugins relying on the function when run with + multiprocessing. (`GitLab#105`_, `GitLab#107`_) + +.. _GitLab#103: https://gitlab.com/pycqa/flake8/issues/103 +.. _GitLab#105: https://gitlab.com/pycqa/flake8/issues/105 +.. _GitLab#107: https://gitlab.com/pycqa/flake8/issues/107 diff --git a/docs/source/release-notes/2.5.2.rst b/docs/source/release-notes/2.5.2.rst new file mode 100644 index 0000000..a093c9f --- /dev/null +++ b/docs/source/release-notes/2.5.2.rst @@ -0,0 +1,7 @@ +2.5.2 - 2016-01-30 +------------------ + +- **Bug** Parse ``output_file`` and ``enable_extensions`` from config files + +- **Improvement** Raise upper bound on mccabe plugin to allow for version + 0.4.0 diff --git a/docs/source/release-notes/2.5.3.rst b/docs/source/release-notes/2.5.3.rst new file mode 100644 index 0000000..85dbf33 --- /dev/null +++ b/docs/source/release-notes/2.5.3.rst @@ -0,0 +1,5 @@ +2.5.3 - 2016-02-11 +------------------ + +- **Bug** Actually parse ``output_file`` and ``enable_extensions`` from config + files diff --git a/docs/source/release-notes/2.5.4.rst b/docs/source/release-notes/2.5.4.rst new file mode 100644 index 0000000..5ba03ba --- /dev/null +++ b/docs/source/release-notes/2.5.4.rst @@ -0,0 +1,4 @@ +2.5.4 - 2016-02-11 +------------------ + +- **Bug** Missed an attribute rename during the v2.5.3 release. diff --git a/docs/source/release-notes/2.5.5.rst b/docs/source/release-notes/2.5.5.rst new file mode 100644 index 0000000..683cfb6 --- /dev/null +++ b/docs/source/release-notes/2.5.5.rst @@ -0,0 +1,7 @@ +2.5.5 - 2016-06-14 +------------------ + +- **Bug** Fix setuptools integration when parsing config files + +- **Bug** Don't pass the user's config path as the config_file when creating a + StyleGuide diff --git a/docs/source/release-notes/2.6.1.rst b/docs/source/release-notes/2.6.1.rst new file mode 100644 index 0000000..d05bd18 --- /dev/null +++ b/docs/source/release-notes/2.6.1.rst @@ -0,0 +1,6 @@ +2.6.1 - 2016-06-25 +------------------ + +- **Bug** Update the config files to search for to include ``setup.cfg`` and + ``tox.ini``. This was broken in 2.5.5 when we stopped passing + ``config_file`` to our Style Guide diff --git a/docs/source/release-notes/3.0.0.rst b/docs/source/release-notes/3.0.0.rst new file mode 100644 index 0000000..8f13cfe --- /dev/null +++ b/docs/source/release-notes/3.0.0.rst @@ -0,0 +1,42 @@ +3.0.0b1 -- 2016-06-25 +--------------------- + +- Rewrite our documentation from scratch! (http://flake8.pycqa.org) + +- Drop explicit support for Pythons 2.6, 3.2, and 3.3. + +- Remove dependence on pep8/pycodestyle for file processing, plugin + dispatching, and more. We now control all of this while keeping backwards + compatibility. + +- ``--select`` and ``--ignore`` can now both be specified and try to find the + most specific rule from each. For example, if you do ``--select E --ignore + E123`` then we will report everything that starts with ``E`` except for + ``E123``. Previously, you would have had to do ``--ignore E123,F,W`` which + will also still work, but the former should be far more intuitive. + +- Add support for in-line ``# noqa`` comments to specify **only** the error + codes to be ignored, e.g., ``# noqa: E123,W503`` + +- Add entry-point for formatters as well as a base class that new formatters + can inherit from. See the documentation for more details. + +- Add detailed verbose output using the standard library logging module. + +- Enhance our usage of optparse for plugin developers by adding new parameters + to the ``add_option`` that plugins use to register new options. + +- Update ``--install-hook`` to require the name of version control system hook + you wish to install a Flake8. + +- Stop checking sub-directories more than once via the setuptools command + +- When passing a file on standard-in, allow the caller to specify + ``--stdin-display-name`` so the output is properly formatted + +- The Git hook now uses ``sys.executable`` to format the shebang line. + This allows Flake8 to install a hook script from a virtualenv that points to + that virtualenv's Flake8 as opposed to a global one (without the virtualenv + being sourced). + +- When using ``--count``, the output is no longer written to stderr. diff --git a/docs/source/release-notes/index.rst b/docs/source/release-notes/index.rst new file mode 100644 index 0000000..d91324f --- /dev/null +++ b/docs/source/release-notes/index.rst @@ -0,0 +1,41 @@ +=========================== + Release Notes and History +=========================== + +All of the release notes that have been recorded for Flake8 are organized here +with the newest releases first. + +.. toctree:: + 3.0.0 + 2.5.5 + 2.5.4 + 2.5.3 + 2.5.2 + 2.5.1 + 2.5.0 + 2.4.1 + 2.4.0 + 2.3.0 + 2.2.5 + 2.2.4 + 2.2.3 + 2.2.2 + 2.2.1 + 2.2.0 + 2.1.0 + 2.0.0 + 1.7.0 + 1.6.2 + 1.6.1 + 1.6.0 + 1.5.0 + 1.4.0 + 1.3.1 + 1.3.0 + 1.2.0 + 1.1.0 + 1.0.0 + 0.9.0 + 0.8.0 + 0.7.0 + 0.6.0 diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt new file mode 100644 index 0000000..77bd874 --- /dev/null +++ b/docs/source/requirements.txt @@ -0,0 +1,4 @@ +sphinx>=1.3.0 +sphinx_rtd_theme +sphinx-prompt +configparser diff --git a/docs/source/user/.keep b/docs/source/user/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/user/configuration.rst b/docs/source/user/configuration.rst new file mode 100644 index 0000000..439aeae --- /dev/null +++ b/docs/source/user/configuration.rst @@ -0,0 +1,227 @@ +.. _configuration: + +==================== + Configuring Flake8 +==================== + +Once you have learned how to :ref:`invoke ` |Flake8|, you will soon +want to learn how to configure it so you do not have to specify the same +options every time you use it. + +This section will show you how to make + +.. prompt:: bash + + flake8 + +Remember that you want to specify certain options without writing + +.. prompt:: bash + + flake8 --select E123,W456 --enable-extensions H111 + + +Configuration Locations +======================= + +|Flake8| supports storing its configuration in the following places: + +- Your top-level user directory + +- In your project in one of ``setup.cfg``, ``tox.ini``, or ``.flake8``. + + +"User" Configuration +-------------------- + +|Flake8| allows a user to use "global" configuration file to store preferences. +The user configuration file is expected to be stored somewhere in the user's +"home" directory. + +- On Windows the "home" directory will be something like + ``C:\\Users\sigmavirus24``, a.k.a, ``~\``. + +- On Linux and other Unix like systems (including OS X) we will look in + ``~/``. + +Note that |Flake8| looks for ``~\.flake8`` on Windows and ``~/.config/flake8`` +on Linux and other Unix systems. + +User configuration files use the same syntax as Project Configuration files. +Keep reading to see that syntax. + + +Project Configuration +--------------------- + +|Flake8| is written with the understanding that people organize projects into +sub-directories. Let's take for example |Flake8|'s own project structure + +.. code:: + + flake8 + ├── docs + │   ├── build + │   └── source + │   ├── _static + │   ├── _templates + │   ├── dev + │   ├── internal + │   └── user + ├── flake8 + │   ├── formatting + │   ├── main + │   ├── options + │   └── plugins + └── tests + ├── fixtures + │   └── config_files + ├── integration + └── unit + +In the top-level ``flake8`` directory (which contains ``docs``, ``flake8``, +and ``tests``) there's also ``tox.ini`` and ``setup.cfg`` files. In our case, +we keep our |Flake8| configuration in ``tox.ini``. Regardless of whether you +keep your config in ``.flake8``, ``setup.cfg``, or ``tox.ini`` we expect you +to use INI to configure |Flake8| (since each of these files already uses INI +as a format). This means that any |Flake8| configuration you wish to set needs +to be in the ``flake8`` section, which means it needs to start like so: + +.. code-block:: ini + + [flake8] + +Each command-line option that you want to specify in your config file can +be named in either of two ways: + +#. Using underscores (``_``) instead of hyphens (``-``) + +#. Simply using hyphens (without the leading hyphens) + +.. note:: + + Not every |Flake8| command-line option can be specified in the + configuration file. See :ref:`our list of options ` to + determine which options will be parsed from the configuration files. + +Let's actually look at |Flake8|'s own configuration section: + +.. code-block:: ini + + [flake8] + ignore = D203 + exclude = .git,__pycache__,docs/source/conf.py,old,build,dist + max-complexity = 10 + +This is equivalent to: + +.. prompt:: bash + + flake8 --ignore D203 \ + --exclude .git,__pycache__,docs/source/conf.py,old,build,dist \ + --max-complexity 10 + +In our case, if we wanted to, we could also do + +.. code-block:: ini + + [flake8] + ignore = D203 + exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist + max-complexity = 10 + +This would allow us to add comments for why we're excluding items, e.g., + +.. code-block:: ini + + [flake8] + ignore = D203 + exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, + # The conf file is mostly autogenerated, ignore it + docs/source/conf.py, + # The old directory contains Flake8 2.0 + old, + # This contains our built documentation + build, + # This contains builds of flake8 that we don't want to check + dist + max-complexity = 10 + +.. note:: + + If you're using Python 2, you will notice that we download the + :mod:`configparser` backport from PyPI. That backport enables us to + support this behaviour on all supported versions of Python. + + Please do **not** open issues about this dependency to |Flake8|. + +.. note:: + + You can also specify ``--max-complexity`` as ``max_complexity = 10``. + +This is also useful if you have a long list of error codes to ignore. Let's +look at a portion of OpenStack's Swift `project configuration`_: + +.. code-block:: ini + + [flake8] + # it's not a bug that we aren't using all of hacking, ignore: + # F812: list comprehension redefines ... + # H101: Use TODO(NAME) + # H202: assertRaises Exception too broad + # H233: Python 3.x incompatible use of print operator + # H301: one import per line + # H306: imports not in alphabetical order (time, os) + # H401: docstring should not start with a space + # H403: multi line docstrings should end on a new line + # H404: multi line docstring should start without a leading new line + # H405: multi line docstring summary not separated with an empty line + # H501: Do not use self.__dict__ for string formatting + ignore = F812,H101,H202,H233,H301,H306,H401,H403,H404,H405,H501 + +They use the comments to describe the check but they could also write this as: + +.. code-block:: ini + + [flake8] + # it's not a bug that we aren't using all of hacking + ignore = + # F812: list comprehension redefines ... + F812, + # H101: Use TODO(NAME) + H101, + # H202: assertRaises Exception too broad + H202, + # H233: Python 3.x incompatible use of print operator + H233, + # H301: one import per line + H301, + # H306: imports not in alphabetical order (time, os) + H306, + # H401: docstring should not start with a space + H401, + # H403: multi line docstrings should end on a new line + H403, + # H404: multi line docstring should start without a leading new line + H404, + # H405: multi line docstring summary not separated with an empty line + H405, + # H501: Do not use self.__dict__ for string formatting + H501 + +Or they could use each comment to describe **why** they've ignored the check. +|Flake8| knows how to parse these lists and will appropriatey handle +these situations. + +.. _project configuration: + https://github.com/openstack/swift/blob/3944d820387f08372c1a29444f4af7d8e6090ae9/tox.ini#L66..L81 diff --git a/docs/source/user/ignoring-errors.rst b/docs/source/user/ignoring-errors.rst new file mode 100644 index 0000000..ff37c4a --- /dev/null +++ b/docs/source/user/ignoring-errors.rst @@ -0,0 +1,90 @@ +============================= + Ignoring Errors with Flake8 +============================= + +By default, |Flake8| has a list of error codes that it ignores. The list used +by a version of |Flake8| may be different than the list used by a different +version. To see the default list, :option:`flake8 --help` will +show the output with the current default list. + + +Changing the Ignore List +======================== + +If we want to change the list of ignored codes for a single run, we can use +:option:`flake8 --ignore` to specify a comma-separated list of codes for a +specific run on the command-line, e.g., + +.. prompt:: bash + + flake8 --ignore=E1,E23,W503 path/to/files/ path/to/more/files/ + +This tells |Flake8| to ignore any error codes starting with ``E1``, ``E23``, +or ``W503`` while it is running. + +.. note:: + + The documentation for :option:`flake8 --ignore` shows examples for how + to change the ignore list in the configuration file. See also + :ref:`configuration` as well for details about how to use configuration + files. + + +In-line Ignoring Errors +======================= + +In some cases, we might not want to ignore an error code (or class of error +codes) for the entirety of our project. Instead, we might want to ignore the +specific error code on a specific line. Let's take for example a line like + +.. code-block:: python + + example = lambda: 'example' + +Sometimes we genuinely need something this simple. We could instead define +a function like we normally would. Note, in some contexts this distracts from +what is actually happening. In those cases, we can also do: + +.. code-block:: python + + example = lambda: 'example' # noqa: E731 + +This will only ignore the error from pycodestyle that checks for lambda +assignments and generates an ``E731``. If there are other errors on the line +then those will be reported. + +.. note:: + + If we ever want to disable |Flake8| respecting ``# noqa`` comments, we can + can refer to :option:`flake8 --disable-noqa`. + +If we instead had more than one error that we wished to ignore, we could +list all of the errors with commas separating them: + +.. code-block:: python + + # noqa: E731,E123 + +Finally, if we have a particularly bad line of code, we can ignore every error +using simply ``# noqa`` with nothing after it. + + +Ignoring Entire Files +===================== + +Imagine a situation where we are adding |Flake8| to a codebase. Let's further +imagine that with the exception of a few particularly bad files, we can add +|Flake8| easily and move on with our lives. There are two ways to ignore the +file: + +#. By explicitly adding it to our list of excluded paths (see: :option:`flake8 + --exclude`) + +#. By adding ``# flake8: noqa`` to the file + +The former is the **recommended** way of ignoring entire files. By using our +exclude list, we can include it in our configuration file and have one central +place to find what files aren't included in |Flake8| checks. The latter has the +benefit that when we run |Flake8| with :option:`flake8 --disable-noqa` all of +the errors in that file will show up without having to modify our +configuration. Both exist so we can choose which is better for us. diff --git a/docs/source/user/index.rst b/docs/source/user/index.rst new file mode 100644 index 0000000..0df911e --- /dev/null +++ b/docs/source/user/index.rst @@ -0,0 +1,33 @@ +============== + Using Flake8 +============== + +|Flake8| can be used in many ways. A few: + +- invoked on the command-line + +- invoked via Python + +- called by Git or Mercurial on or around committing + +This guide will cover all of these and the nuances for using |Flake8|. + +.. note:: + + This portion of |Flake8|'s documentation does not cover installation. See + the :ref:`installation-guide` section for how to install |Flake8|. + +.. toctree:: + :maxdepth: 2 + + invocation + configuration + options + ignoring-errors + using-plugins + python-api + +.. config files +.. command-line tutorial +.. VCS usage +.. installing and using plugins diff --git a/docs/source/user/invocation.rst b/docs/source/user/invocation.rst new file mode 100644 index 0000000..383e93a --- /dev/null +++ b/docs/source/user/invocation.rst @@ -0,0 +1,144 @@ +.. _invocation: + +================= + Invoking Flake8 +================= + +Once you have :ref:`installed ` |Flake8|, you can begin +using it. Most of the time, you will be able to generically invoke |Flake8| +like so: + +.. prompt:: bash + + flake8 ... + +Where you simply allow the shell running in your terminal to locate |Flake8|. +In some cases, though, you may have installed |Flake8| for multiple versions +of Python (e.g., Python 2.7 and Python 3.5) and you need to call a specific +version. In that case, you will have much better results using: + +.. prompt:: bash + + python2.7 -m flake8 + +Or + +.. prompt:: bash + + python3.5 -m flake8 + +Since that will tell the correct version of Python to run |Flake8|. + +.. note:: + + Installing |Flake8| once will not install it on both Python 2.7 and + Python 3.5. It will only install it for the version of Python that + is running pip. + +It is also possible to specify command-line options directly to |Flake8|: + +.. prompt:: bash + + flake8 --select E123 + +Or + +.. prompt:: bash + + python -m flake8 --select E123 + +.. note:: + + This is the last time we will show both versions of an invocation. + From now on, we'll simply use ``flake8`` and assume that the user + knows they can instead use ``python -m flake8`` instead. + +It's also possible to narrow what |Flake8| will try to check by specifying +exactly the paths and directories you want it to check. Let's assume that +we have a directory with python files and sub-directories which have python +files (and may have more sub-directories) called ``my_project``. Then if +we only want errors from files found inside ``my_project`` we can do: + +.. prompt:: bash + + flake8 my_project + +And if we only want certain errors (e.g., ``E123``) from files in that +directory we can also do: + +.. prompt:: bash + + flake8 --select E123 my_project + +If you want to explore more options that can be passed on the command-line, +you can use the ``--help`` option: + +.. prompt:: bash + + flake8 --help + +And you should see something like: + +.. code:: + + Usage: flake8 [options] file file ... + + Options: + --version show program's version number and exit + -h, --help show this help message and exit + -v, --verbose Print more information about what is happening in + flake8. This option is repeatable and will increase + verbosity each time it is repeated. + -q, --quiet Report only file names, or nothing. This option is + repeatable. + --count Print total number of errors and warnings to standard + error and set the exit code to 1 if total is not + empty. + --diff Report changes only within line number ranges in the + unified diff provided on standard in by the user. + --exclude=patterns Comma-separated list of files or directories to + exclude.(Default: + .svn,CVS,.bzr,.hg,.git,__pycache__,.tox) + --filename=patterns Only check for filenames matching the patterns in this + comma-separated list. (Default: *.py) + --format=format Format errors according to the chosen formatter. + --hang-closing Hang closing bracket instead of matching indentation + of opening bracket's line. + --ignore=errors Comma-separated list of errors and warnings to ignore + (or skip). For example, ``--ignore=E4,E51,W234``. + (Default: E121,E123,E126,E226,E24,E704) + --max-line-length=n Maximum allowed line length for the entirety of this + run. (Default: 79) + --select=errors Comma-separated list of errors and warnings to enable. + For example, ``--select=E4,E51,W234``. (Default: ) + --disable-noqa Disable the effect of "# noqa". This will report + errors on lines with "# noqa" at the end. + --show-source Show the source generate each error or warning. + --statistics Count errors and warnings. + --enabled-extensions=ENABLED_EXTENSIONS + Enable plugins and extensions that are otherwise + disabled by default + --exit-zero Exit with status code "0" even if there are errors. + -j JOBS, --jobs=JOBS Number of subprocesses to use to run checks in + parallel. This is ignored on Windows. The default, + "auto", will auto-detect the number of processors + available to use. (Default: auto) + --output-file=OUTPUT_FILE + Redirect report to a file. + --append-config=APPEND_CONFIG + Provide extra config files to parse in addition to the + files found by Flake8 by default. These files are the + last ones read and so they take the highest precedence + when multiple files provide the same option. + --config=CONFIG Path to the config file that will be the authoritative + config source. This will cause Flake8 to ignore all + other configuration files. + --isolated Ignore all found configuration files. + --builtins=BUILTINS define more built-ins, comma separated + --doctests check syntax of the doctests + --include-in-doctest=INCLUDE_IN_DOCTEST + Run doctests only on these files + --exclude-from-doctest=EXCLUDE_FROM_DOCTEST + Skip these files when running doctests + + Installed plugins: pyflakes: 1.0.0, pep8: 1.7.0 diff --git a/docs/source/user/options.rst b/docs/source/user/options.rst new file mode 100644 index 0000000..acaa67c --- /dev/null +++ b/docs/source/user/options.rst @@ -0,0 +1,730 @@ +.. _options-list: + +================================================ + Full Listing of Options and Their Descriptions +================================================ + +.. + NOTE(sigmavirus24): When adding new options here, please follow the + following _rough_ template: + + .. option:: --[=] + + Active description of option's purpose (note that each description + starts with an active verb) + + Command-line usage: + + .. prompt:: bash + + flake8 --[=] [positional params] + + This **can[ not]** be specified in config files. + + (If it can be, an example using .. code-block:: ini) + + Thank you for your contribution to Flake8's documentation. + +.. program:: flake8 + +.. option:: --version + + Show |Flake8|'s version as well as the versions of all plugins + installed. + + Command-line usage: + + .. prompt:: bash + + flake8 --version + + This **can not** be specified in config files. + + +.. option:: -h, --help + + Show a description of how to use |Flake8| and its options. + + Command-line usage: + + .. prompt:: bash + + flake8 --help + flake8 -h + + This **can not** be specified in config files. + + +.. option:: -v, --verbose + + Increase the verbosity of |Flake8|'s output. Each time you specify + it, it will print more and more information. + + Command-line example: + + .. prompt:: bash + + flake8 -vv + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + verbose = 2 + + +.. option:: -q, --quiet + + Decrease the verbosity of |Flake8|'s output. Each time you specify it, + it will print less and less information. + + Command-line example: + + .. prompt:: bash + + flake8 -q + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + quiet = 1 + + +.. option:: --count + + Print the total number of errors. + + Command-line example: + + .. prompt:: bash + + flake8 --count dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + count = True + + +.. option:: --diff + + Use the unified diff provided on standard in to only check the modified + files and report errors included in the diff. + + Command-line example: + + .. prompt:: bash + + git diff -u | flake8 --diff + + This **can not** be specified in config files. + + +.. option:: --exclude= + + Provide a comma-separated list of glob patterns to exclude from checks. + + This defaults to: ``.svn,CVS,.bzr,.hg,.git,__pycache__,.tox`` + + Example patterns: + + - ``*.pyc`` will match any file that ends with ``.pyc`` + + - ``__pycache__`` will match any path that has ``__pycache__`` in it + + - ``lib/python`` will look expand that using :func:`os.path.abspath` and + look for matching paths + + Command-line example: + + .. prompt:: bash + + flake8 --exclude=*.pyc dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + exclude = + .tox, + __pycache__ + + +.. option:: --filename= + + Provide a comma-separate list of glob patterns to include for checks. + + This defaults to: ``*.py`` + + Example patterns: + + - ``*.py`` will match any file that ends with ``.py`` + + - ``__pycache__`` will match any path that has ``__pycache__`` in it + + - ``lib/python`` will look expand that using :func:`os.path.abspath` and + look for matching paths + + Command-line example: + + .. prompt:: bash + + flake8 --filename=*.py dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + filename = + example.py, + another-example*.py + + +.. option:: --stdin-display-name= + + Provide the name to use to report warnings and errors from code on stdin. + + Instead of reporting an error as something like: + + .. code:: + + stdin:82:73 E501 line too long + + You can specify this option to have it report whatever value you want + instead of stdin. + + This defaults to: ``stdin`` + + Command-line example: + + .. prompt:: bash + + cat file.py | flake8 --stdin-display-name=file.py - + + This **can not** be specified in config files. + + +.. option:: --format= + + Select the formatter used to display errors to the user. + + This defaults to: ``default`` + + By default, there are two formatters available: + + - default + - pylint + + Other formatters can be installed. Refer to their documentation for the + name to use to select them. Further, users can specify their own format + string. The variables available are: + + - code + - col + - path + - row + - text + + The default formatter has a format string of: + + .. code-block:: python + + '%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + + Command-line example: + + .. prompt:: bash + + flake8 --format=pylint dir/ + flake8 --format='%(path)s::%(row)d,%(col)d::%(code)s::%(text)s' dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + format=pylint + format=%(path)s::%(row)d,%(col)d::%(code)s::%(text)s + + +.. option:: --hang-closing + + Toggle whether pycodestyle should enforce matching the indentation of the + opening bracket's line. When you specify this, it will prefer that you + hang the closing bracket rather than match the indentation. + + Command-line example: + + .. prompt:: bash + + flake8 --hang-closing dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + hang_closing = True + hang-closing = True + + +.. option:: --ignore= + + Specify a list of codes to ignore. The list is expected to be + comma-separated, and does not need to specify an error code exactly. + Since |Flake8| 3.0, this **can** be combined with :option:`--select`. See + :option:`--select` for more information. + + For example, if you wish to only ignore ``W234``, then you can specify + that. But if you want to ignore all codes that start with ``W23`` you + need only specify ``W23`` to ignore them. This also works for ``W2`` and + ``W`` (for example). + + This defaults to: ``E121,E123,E126,E226,E24,E704`` + + Command-line example: + + .. prompt:: bash + + flake8 --ignore=E121,E123 dir/ + flake8 --ignore=E24,E704 dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + ignore = + E121, + E123 + ignore = E121,E123 + + +.. option:: --max-line-length= + + Set the maximum length that any line (with some exceptions) may be. + + Exceptions include lines that are either strings or comments which are + entirely URLs. For example: + + .. code-block:: python + + # https://some-super-long-domain-name.com/with/some/very/long/path + + url = ( + 'http://...' + ) + + This defaults to: 79 + + Command-line example: + + .. prompt:: bash + + flake8 --max-line-length 99 dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + max-line-length = 79 + + +.. option:: --select= + + Specify the list of error codes you wish |Flake8| to report. Similarly to + :option:`--ignore`. You can specify a portion of an error code to get all + that start with that string. For example, you can use ``E``, ``E4``, + ``E43``, and ``E431``. + + This defaults to: E,F,W,C + + Command-line example: + + .. prompt:: bash + + flake8 --select=E431,E5,W,F dir/ + flake8 --select=E,W dir/ + + This can also be combined with :option:`--ignore`: + + .. prompt:: bash + + flake8 --select=E --ignore=E432 dir/ + + This will report all codes that start with ``E``, but ignore ``E432`` + specifically. This is more flexibly than the |Flake8| 2.x and 1.x used + to be. + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + select = + E431, + W, + F + + +.. option:: --disable-noqa + + Report all errors, even if it is on the same line as a ``# NOQA`` comment. + ``# NOQA`` can be used to silence messages on specific lines. Sometimes, + users will want to see what errors are being silenced without editing the + file. This option allows you to see all the warnings, errors, etc. + reported. + + Command-line example: + + .. prompt:: bash + + flake8 --disable-noqa dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + disable_noqa = True + disable-noqa = True + + +.. option:: --show-source + + Print the source code generating the error/warning in question. + + Command-line example: + + .. prompt:: bash + + flake8 --show-source dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + show_source = True + show-source = True + + +.. option:: --statistics + + Count the number of occurrences of each error/warning code and + print a report. + + Command-line example: + + .. prompt:: bash + + flake8 --statistics + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + statistics = True + + +.. option:: --enable-extensions= + + Enable off-by-default extensions. + + Plugins to |Flake8| have the option of registering themselves as + off-by-default. These plugins effectively add themselves to the + default ignore list. + + Command-line example: + + .. prompt:: bash + + flake8 --enable-extensions=H111 dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + enable-extensions = + H111, + G123 + enable_extensions = + H111, + G123 + + +.. option:: --exit-zero + + Force |Flake8| to use the exit status code 0 even if there are errors. + + By default |Flake8| will exit with a non-zero integer if there are errors. + + Command-line example: + + .. prompt:: bash + + flake8 --exit-zero dir/ + + This **can not** be specified in config files. + + +.. option:: --install-hook=VERSION_CONTROL_SYSTEM + + Install a hook for your version control system that is executed before + or during commit. + + The available options are: + + - git + - mercurial + + Command-line usage: + + .. prompt:: bash + + flake8 --install-hook=git + flake8 --install-hook=mercurial + + This **can not** be specified in config files. + + +.. option:: --jobs= + + Specify the number of subprocesses that |Flake8| will use to run checks in + parallel. + + .. note:: + + This option is ignored on Windows because :mod:`multiprocessing` does + not support Windows across all supported versions of Python. + + This defaults to: ``auto`` + + The default behaviour will use the number of CPUs on your machine as + reported by :func:`multiprocessing.cpu_count`. + + Command-line example: + + .. prompt:: bash + + flake8 --jobs=8 dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + jobs = 8 + + +.. option:: --output-file= + + Redirect all output to the specified file. + + Command-line example: + + .. prompt:: bash + + flake8 --output-file=output.txt dir/ + flake8 -vv --output-file=output.txt dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + output-file = output.txt + output_file = output.txt + + +.. option:: --append-config= + + Provide extra config files to parse in after and in addition to the files + that |Flake8| found on its own. Since these files are the last ones read + into the Configuration Parser, so it has the highest precedence if it + provides an option specified in another config file. + + Command-line example: + + .. prompt:: bash + + flake8 --append-config=my-extra-config.ini dir/ + + This **can not** be specified in config files. + + +.. option:: --config= + + Provide a path to a config file that will be the only config file read and + used. This will cause |Flake8| to ignore all other config files that + exist. + + Command-line example: + + .. prompt:: bash + + flake8 --config=my-only-config.ini dir/ + + This **can not** be specified in config files. + + +.. option:: --isolated + + Ignore any config files and use |Flake8| as if there were no config files + found. + + Command-line example: + + .. prompt:: bash + + flake8 --isolated dir/ + + This **can not** be specified in config files. + + +.. option:: --builtins= + + Provide a custom list of builtin functions, objects, names, etc. + + This allows you to let pyflakes know about builtins that it may + not immediately recognize so it does not report warnings for using + an undefined name. + + This is registered by the default PyFlakes plugin. + + Command-line example: + + .. prompt:: bash + + flake8 --builtins=_,_LE,_LW dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + builtins = + _, + _LE, + _LW + + +.. option:: --doctests + + Enable PyFlakes syntax checking of doctests in docstrings. + + This is registered by the default PyFlakes plugin. + + Command-line example: + + .. prompt:: bash + + flake8 --doctests dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + doctests = True + + +.. option:: --include-in-doctest= + + Specify which files are checked by PyFlakes for doctest syntax. + + This is registered by the default PyFlakes plugin. + + Command-line example: + + .. prompt:: bash + + flake8 --include-in-doctest=dir/subdir/file.py,dir/other/file.py dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + include-in-doctest = + dir/subdir/file.py, + dir/other/file.py + include_in_doctest = + dir/subdir/file.py, + dir/other/file.py + + +.. option:: --exclude-from-doctest= + + Specify which files are not to be checked by PyFlakes for doctest syntax. + + This is registered by the default PyFlakes plugin. + + Command-line example: + + .. prompt:: bash + + flake8 --exclude-in-doctest=dir/subdir/file.py,dir/other/file.py dir/ + + This **can** be specified in config files. + + Example config file usage: + + .. code-block:: ini + + exclude-in-doctest = + dir/subdir/file.py, + dir/other/file.py + exclude_in_doctest = + dir/subdir/file.py, + dir/other/file.py + + +.. option:: --benchmark + + Collect and print benchmarks for this run of |Flake8|. This aggregates the + total number of: + + - tokens + - physical lines + - logical lines + - files + + and the number of elapsed seconds. + + Command-line usage: + + .. prompt:: bash + + flake8 --benchmark dir/ + + This **can not** be specified in config files. diff --git a/docs/source/user/python-api.rst b/docs/source/user/python-api.rst new file mode 100644 index 0000000..214565d --- /dev/null +++ b/docs/source/user/python-api.rst @@ -0,0 +1,11 @@ +=================== + Public Python API +=================== + +|Flake8| 3.0.0 presently does not have a public, stable Python API. + +When it does it will be located in :mod:`flake8.api` and that will +be documented here. + +.. automodule:: flake8.api + :members: diff --git a/docs/source/user/using-plugins.rst b/docs/source/user/using-plugins.rst new file mode 100644 index 0000000..fad1911 --- /dev/null +++ b/docs/source/user/using-plugins.rst @@ -0,0 +1,66 @@ +================================== + Using Plugins For Fun and Profit +================================== + +|Flake8| is useful on its own but a lot of |Flake8|'s popularity is due to +its extensibility. Our community has developed :term:`plugin`\ s that augment +|Flake8|'s behaviour. Most of these plugins are uploaded to PyPI_. The +developers of these plugins often have some style they wish to enforce. + +For example, `flake8-docstrings`_ adds a check for :pep:`257` style +conformance. Others attempt to enforce consistency, like `flake8-future`_. + +.. note:: + + The accuracy or reliability of these plugins may vary wildly from plugin + to plugin and not all plugins are guaranteed to work with |Flake8| 3.0. + +To install a third-party plugin, make sure that you know which version of +Python (or pip) you used to install |Flake8|. You can then use the most +appropriate of: + +.. prompt:: bash + + pip install + pip3 install + python -m pip install + python2.7 -m pip install + python3 -m pip install + python3.4 -m pip install + python3.5 -m pip install + +To install the plugin, where ```` is the package name on PyPI_. +To verify installation use: + +.. prompt:: bash + + flake8 --version + python -m flake8 --version + +To see the plugin's name and version in the output. + +.. seealso:: :ref:`How to Invoke Flake8 ` + +After installation, most plugins immediately start reporting :term:`error`\ s. +Check the plugin's documentation for which error codes it returns and if it +disables any by default. + +.. note:: + + You can use both :option:`flake8 --select` and :option:`flake8 --ignore` + with plugins. + +Some plugins register new options, so be sure to check :option:`flake8 --help` +for new flags and documentation. These plugins may also allow these flags to +be specified in your configuration file. Hopefully, the plugin authors have +documented this for you. + +.. seealso:: :ref:`Configuring Flake8 ` + + +.. _PyPI: + https://pypi.io/ +.. _flake8-docstrings: + https://pypi.io/project/flake8-docstrings/ +.. _flake8-future: + https://pypi.io/project/flake8-future/ diff --git a/docs/vcs.rst b/docs/vcs.rst deleted file mode 100644 index 66538b4..0000000 --- a/docs/vcs.rst +++ /dev/null @@ -1,50 +0,0 @@ -VCS Hooks -========= - -flake8 can install hooks for Mercurial and Git so that flake8 is run -automatically before commits. The commit will fail if there are any -flake8 issues. - -You can install the hook by issuing this command in the root of your -project:: - - $ flake8 --install-hook - -In the case of Git, the hook won't be installed if a custom -``pre-commit`` hook file is already present in -the ``.git/hooks`` directory. - -You can control the behavior of the pre-commit hook using configuration file -settings or environment variables: - -``flake8.complexity`` or ``FLAKE8_COMPLEXITY`` - Any value > 0 enables complexity checking with McCabe. (defaults - to 10) - -``flake8.strict`` or ``FLAKE8_STRICT`` - If True, this causes the commit to fail in case of any errors at - all. (defaults to False) - -``flake8.ignore`` or ``FLAKE8_IGNORE`` - Comma-separated list of errors and warnings to ignore. (defaults to - empty) - -``flake8.lazy`` or ``FLAKE8_LAZY`` - If True, also scans those files not added to the index before - commit. (defaults to False) - -You can set these either through the git command line - -.. code-block:: bash-session - - $ git config flake8.complexity 10 - $ git config flake8.strict true - -Or by directly editing ``.git/config`` and adding a section like - -.. code-block:: ini - - [flake8] - complexity = 10 - strict = true - lazy = false diff --git a/docs/warnings.rst b/docs/warnings.rst deleted file mode 100644 index a1d03fc..0000000 --- a/docs/warnings.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _error-codes: - -Warning / Error codes -===================== - -The convention of Flake8 is to assign a code to each error or warning, like -the ``pycodestyle`` tool. These codes are used to configure the list of errors -which are selected or ignored. - -Each code consists of an upper case ASCII letter followed by three digits. -The recommendation is to use a different prefix for each plugin. A list of the -known prefixes is published below: - -- ``E***``/``W***``: `pycodestyle errors and warnings - `_ -- ``F***``: PyFlakes codes (see below) -- ``C9**``: McCabe complexity plugin `mccabe - `_ -- ``N8**``: Naming Conventions plugin `pep8-naming - `_ - - -The original PyFlakes does not provide error codes. Flake8 patches the -PyFlakes messages to add the following codes: - -+------+--------------------------------------------------------------------+ -| code | sample message | -+======+====================================================================+ -| F401 | ``module`` imported but unused | -+------+--------------------------------------------------------------------+ -| F402 | import ``module`` from line ``N`` shadowed by loop variable | -+------+--------------------------------------------------------------------+ -| F403 | 'from ``module`` import \*' used; unable to detect undefined names | -+------+--------------------------------------------------------------------+ -| F404 | future import(s) ``name`` after other statements | -+------+--------------------------------------------------------------------+ -| F405 | ``name`` may be undefined, or defined from star imports: ``module``| -+------+--------------------------------------------------------------------+ -+------+--------------------------------------------------------------------+ -| F811 | redefinition of unused ``name`` from line ``N`` | -+------+--------------------------------------------------------------------+ -| F812 | list comprehension redefines ``name`` from line ``N`` | -+------+--------------------------------------------------------------------+ -| F821 | undefined name ``name`` | -+------+--------------------------------------------------------------------+ -| F822 | undefined name ``name`` in __all__ | -+------+--------------------------------------------------------------------+ -| F823 | local variable ``name`` ... referenced before assignment | -+------+--------------------------------------------------------------------+ -| F831 | duplicate argument ``name`` in function definition | -+------+--------------------------------------------------------------------+ -| F841 | local variable ``name`` is assigned to but never used | -+------+--------------------------------------------------------------------+ diff --git a/flake8/__init__.py b/flake8/__init__.py deleted file mode 100644 index 574f407..0000000 --- a/flake8/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '2.6.1' diff --git a/flake8/__main__.py b/flake8/__main__.py deleted file mode 100644 index aaa497b..0000000 --- a/flake8/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from flake8.main import main - -# python -m flake8 (with Python >= 2.7) -main() diff --git a/flake8/callbacks.py b/flake8/callbacks.py deleted file mode 100644 index 3767f30..0000000 --- a/flake8/callbacks.py +++ /dev/null @@ -1,27 +0,0 @@ -import atexit -import sys - - -def install_vcs_hook(option, option_str, value, parser): - # For now, there's no way to affect a change in how pep8 processes - # options. If no args are provided and there's no config file present, - # it will error out because no input was provided. To get around this, - # when we're using --install-hook, we'll say that there were arguments so - # we can actually attempt to install the hook. - # See: https://gitlab.com/pycqa/flake8/issues/2 and - # https://github.com/jcrocholl/pep8/blob/4c5bf00cb613be617c7f48d3b2b82a1c7b895ac1/pep8.py#L1912 - # for more context. - parser.values.install_hook = True - parser.rargs.append('.') - - -def restore_stdout(old_stdout): - sys.stdout.close() - sys.stdout = old_stdout - - -def redirect_stdout(option, option_str, value, parser): - fd = open(value, 'w') - old_stdout, sys.stdout = sys.stdout, fd - - atexit.register(restore_stdout, old_stdout) diff --git a/flake8/compat.py b/flake8/compat.py deleted file mode 100644 index 9bd00a7..0000000 --- a/flake8/compat.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -"""Compatibility shims for Flake8.""" -import os.path -import sys - - -def relpath(path, start='.'): - """Wallpaper over the differences between 2.6 and newer versions.""" - if sys.version_info < (2, 7) and path.startswith(start): - return path[len(start):] - else: - return os.path.relpath(path, start=start) diff --git a/flake8/engine.py b/flake8/engine.py deleted file mode 100644 index 885f9c6..0000000 --- a/flake8/engine.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding: utf-8 -*- -import errno -import io -import platform -import re -import sys -import warnings - -import pycodestyle as pep8 - -from flake8 import __version__ -from flake8 import callbacks -from flake8.reporter import (multiprocessing, BaseQReport, FileQReport, - QueueReport) -from flake8 import util - -_flake8_noqa = re.compile(r'\s*# flake8[:=]\s*noqa', re.I).search - -EXTRA_EXCLUDE = ['.tox', '.eggs', '*.egg'] - -pep8.PROJECT_CONFIG = ('.flake8',) + pep8.PROJECT_CONFIG - - -def _load_entry_point(entry_point, verify_requirements): - """Based on the version of setuptools load an entry-point correctly. - - setuptools 11.3 deprecated `require=False` in the call to EntryPoint.load. - To load entry points correctly after that without requiring all - dependencies be present, the proper way is to call EntryPoint.resolve. - - This function will provide backwards compatibility for older versions of - setuptools while also ensuring we do the right thing for the future. - """ - if hasattr(entry_point, 'resolve') and hasattr(entry_point, 'require'): - if verify_requirements: - entry_point.require() - plugin = entry_point.resolve() - else: - plugin = entry_point.load(require=verify_requirements) - - return plugin - - -def _register_extensions(): - """Register all the extensions.""" - extensions = util.OrderedSet() - extensions.add(('pycodestyle', pep8.__version__)) - parser_hooks = [] - options_hooks = [] - ignored_hooks = [] - try: - from pkg_resources import iter_entry_points - except ImportError: - pass - else: - for entry in iter_entry_points('flake8.extension'): - # Do not verify that the requirements versions are valid - checker = _load_entry_point(entry, verify_requirements=False) - pep8.register_check(checker, codes=[entry.name]) - extensions.add((checker.name, checker.version)) - if hasattr(checker, 'add_options'): - parser_hooks.append(checker.add_options) - if hasattr(checker, 'parse_options'): - options_hooks.append(checker.parse_options) - if getattr(checker, 'off_by_default', False) is True: - ignored_hooks.append(entry.name) - return extensions, parser_hooks, options_hooks, ignored_hooks - - -def get_parser(): - """This returns an instance of optparse.OptionParser with all the - extensions registered and options set. This wraps ``pep8.get_parser``. - """ - (extensions, parser_hooks, options_hooks, ignored) = _register_extensions() - details = ', '.join('%s: %s' % ext for ext in extensions) - python_version = get_python_version() - parser = pep8.get_parser('flake8', '%s (%s) %s' % ( - __version__, details, python_version - )) - for opt in ('--repeat', '--testsuite', '--doctest'): - try: - parser.remove_option(opt) - except ValueError: - pass - - if multiprocessing: - parser.config_options.append('jobs') - parser.add_option('-j', '--jobs', type='string', default='auto', - help="number of jobs to run simultaneously, " - "or 'auto'. This is ignored on Windows.") - - parser.add_option('--exit-zero', action='store_true', - help="exit with code 0 even if there are errors") - for parser_hook in parser_hooks: - parser_hook(parser) - # See comment above regarding why this has to be a callback. - parser.add_option('--install-hook', default=False, dest='install_hook', - help='Install the appropriate hook for this ' - 'repository.', action='callback', - callback=callbacks.install_vcs_hook) - parser.add_option('--output-file', default=None, - help='Redirect report to a file.', - type='string', nargs=1, action='callback', - callback=callbacks.redirect_stdout) - parser.add_option('--enable-extensions', default='', - dest='enable_extensions', - help='Enable plugins and extensions that are disabled ' - 'by default', - type='string') - parser.config_options.extend(['output-file', 'enable-extensions']) - parser.ignored_extensions = ignored - return parser, options_hooks - - -class NoQAStyleGuide(pep8.StyleGuide): - - def input_file(self, filename, lines=None, expected=None, line_offset=0): - """Run all checks on a Python source file.""" - if self.options.verbose: - print('checking %s' % filename) - fchecker = self.checker_class( - filename, lines=lines, options=self.options) - # Any "flake8: noqa" comments to ignore the entire file? - if any(_flake8_noqa(line) for line in fchecker.lines): - return 0 - return fchecker.check_all(expected=expected, line_offset=line_offset) - - -class StyleGuide(object): - """A wrapper StyleGuide object for Flake8 usage. - - This allows for OSErrors to be caught in the styleguide and special logic - to be used to handle those errors. - """ - - # Reasoning for error numbers is in-line below - serial_retry_errors = set([ - # ENOSPC: Added by sigmavirus24 - # > On some operating systems (OSX), multiprocessing may cause an - # > ENOSPC error while trying to trying to create a Semaphore. - # > In those cases, we should replace the customized Queue Report - # > class with pep8's StandardReport class to ensure users don't run - # > into this problem. - # > (See also: https://gitlab.com/pycqa/flake8/issues/74) - errno.ENOSPC, - # NOTE(sigmavirus24): When adding to this list, include the reasoning - # on the lines before the error code and always append your error - # code. Further, please always add a trailing `,` to reduce the visual - # noise in diffs. - ]) - - def __init__(self, **kwargs): - # This allows us to inject a mocked StyleGuide in the tests. - self._styleguide = kwargs.pop('styleguide', NoQAStyleGuide(**kwargs)) - - @property - def options(self): - return self._styleguide.options - - @property - def paths(self): - return self._styleguide.paths - - def _retry_serial(self, func, *args, **kwargs): - """This will retry the passed function in serial if necessary. - - In the event that we encounter an OSError with an errno in - :attr:`serial_retry_errors`, this function will retry this function - using pep8's default Report class which operates in serial. - """ - try: - return func(*args, **kwargs) - except OSError as oserr: - if oserr.errno in self.serial_retry_errors: - self.init_report(pep8.StandardReport) - else: - raise - return func(*args, **kwargs) - - def check_files(self, paths=None): - return self._retry_serial(self._styleguide.check_files, paths=paths) - - def excluded(self, filename, parent=None): - return self._styleguide.excluded(filename, parent=parent) - - def init_report(self, reporter=None): - return self._styleguide.init_report(reporter) - - def input_file(self, filename, lines=None, expected=None, line_offset=0): - return self._retry_serial( - self._styleguide.input_file, - filename=filename, - lines=lines, - expected=expected, - line_offset=line_offset, - ) - - -def _parse_multi_options(options, split_token=','): - r"""Split and strip and discard empties. - - Turns the following: - - A, - B, - - into ["A", "B"]. - - Credit: Kristian Glass as contributed to pep8 - """ - if options: - return [o.strip() for o in options.split(split_token) if o.strip()] - else: - return options - - -def _disable_extensions(parser, options): - ignored_extensions = set(getattr(parser, 'ignored_extensions', [])) - enabled = set(_parse_multi_options(options.enable_extensions)) - - # Remove any of the selected extensions from the extensions ignored by - # default. - ignored_extensions -= enabled - - # Whatever is left afterwards should be unioned with options.ignore and - # options.ignore should be updated with that. - options.ignore = tuple(ignored_extensions.union(options.ignore)) - - -def get_style_guide(**kwargs): - """Parse the options and configure the checker. This returns a sub-class - of ``pep8.StyleGuide``.""" - kwargs['parser'], options_hooks = get_parser() - styleguide = StyleGuide(**kwargs) - options = styleguide.options - _disable_extensions(kwargs['parser'], options) - - if options.exclude and not isinstance(options.exclude, list): - options.exclude = pep8.normalize_paths(options.exclude) - elif not options.exclude: - options.exclude = [] - - # Add patterns in EXTRA_EXCLUDE to the list of excluded patterns - options.exclude.extend(pep8.normalize_paths(EXTRA_EXCLUDE)) - - for options_hook in options_hooks: - options_hook(options) - - if util.warn_when_using_jobs(options): - if not multiprocessing: - warnings.warn("The multiprocessing module is not available. " - "Ignoring --jobs arguments.") - if util.is_windows(): - warnings.warn("The --jobs option is not available on Windows. " - "Ignoring --jobs arguments.") - if util.is_using_stdin(styleguide.paths): - warnings.warn("The --jobs option is not compatible with supplying " - "input using - . Ignoring --jobs arguments.") - if options.diff: - warnings.warn("The --diff option was specified with --jobs but " - "they are not compatible. Ignoring --jobs arguments." - ) - - if options.diff: - options.jobs = None - - force_disable_jobs = util.force_disable_jobs(styleguide) - - if multiprocessing and options.jobs and not force_disable_jobs: - if options.jobs.isdigit(): - n_jobs = int(options.jobs) - else: - try: - n_jobs = multiprocessing.cpu_count() - except NotImplementedError: - n_jobs = 1 - if n_jobs > 1: - options.jobs = n_jobs - reporter = QueueReport - if options.quiet: - reporter = BaseQReport - if options.quiet == 1: - reporter = FileQReport - report = styleguide.init_report(reporter) - report.input_file = styleguide.input_file - styleguide.runner = report.task_queue.put - - return styleguide - - -def get_python_version(): - # The implementation isn't all that important. - try: - impl = platform.python_implementation() + " " - except AttributeError: # Python 2.5 - impl = '' - return '%s%s on %s' % (impl, platform.python_version(), platform.system()) - - -def make_stdin_get_value(original): - def stdin_get_value(): - if not hasattr(stdin_get_value, 'cached_stdin'): - value = original() - if sys.version_info < (3, 0): - stdin = io.BytesIO(value) - else: - stdin = io.StringIO(value) - stdin_get_value.cached_stdin = stdin - else: - stdin = stdin_get_value.cached_stdin - return stdin.getvalue() - - return stdin_get_value - - -pep8.stdin_get_value = make_stdin_get_value(pep8.stdin_get_value) diff --git a/flake8/hooks.py b/flake8/hooks.py deleted file mode 100644 index 40cbf97..0000000 --- a/flake8/hooks.py +++ /dev/null @@ -1,295 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import with_statement -import os -import pycodestyle as pep8 -import sys -import stat -from subprocess import Popen, PIPE -import shutil -import tempfile -try: - from configparser import ConfigParser -except ImportError: # Python 2 - from ConfigParser import ConfigParser - -from flake8 import compat -from flake8.engine import get_parser, get_style_guide - - -def git_hook(complexity=-1, strict=False, ignore=None, lazy=False): - """This is the function used by the git hook. - - :param int complexity: (optional), any value > 0 enables complexity - checking with mccabe - :param bool strict: (optional), if True, this returns the total number of - errors which will cause the hook to fail - :param str ignore: (optional), a comma-separated list of errors and - warnings to ignore - :param bool lazy: (optional), allows for the instances where you don't add - the files to the index before running a commit, e.g., git commit -a - :returns: total number of errors if strict is True, otherwise 0 - """ - gitcmd = "git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD" - if lazy: - # Catch all files, including those not added to the index - gitcmd = gitcmd.replace('--cached ', '') - - if hasattr(ignore, 'split'): - ignore = ignore.split(',') - - # Returns the exit code, list of files modified, list of error messages - _, files_modified, _ = run(gitcmd) - - # We only want to pass ignore and max_complexity if they differ from the - # defaults so that we don't override a local configuration file - options = {} - if ignore: - options['ignore'] = ignore - if complexity > -1: - options['max_complexity'] = complexity - - tmpdir = tempfile.mkdtemp() - - flake8_style = get_style_guide(paths=['.'], **options) - filepatterns = flake8_style.options.filename - - # Copy staged versions to temporary directory - files_to_check = [] - try: - for file_ in files_modified: - # get the staged version of the file - gitcmd_getstaged = "git show :%s" % file_ - _, out, _ = run(gitcmd_getstaged, raw_output=True, decode=False) - # write the staged version to temp dir with its full path to - # avoid overwriting files with the same name - dirname, filename = os.path.split(os.path.abspath(file_)) - prefix = os.path.commonprefix([dirname, tmpdir]) - dirname = compat.relpath(dirname, start=prefix) - dirname = os.path.join(tmpdir, dirname) - if not os.path.isdir(dirname): - os.makedirs(dirname) - - # check_files() only does this check if passed a dir; so we do it - if ((pep8.filename_match(file_, filepatterns) and - not flake8_style.excluded(file_))): - - filename = os.path.join(dirname, filename) - files_to_check.append(filename) - # write staged version of file to temporary directory - with open(filename, "wb") as fh: - fh.write(out) - - # Run the checks - report = flake8_style.check_files(files_to_check) - # remove temporary directory - finally: - shutil.rmtree(tmpdir, ignore_errors=True) - - if strict: - return report.total_errors - - return 0 - - -def hg_hook(ui, repo, **kwargs): - """This is the function executed directly by Mercurial as part of the - hook. This is never called directly by the user, so the parameters are - undocumented. If you would like to learn more about them, please feel free - to read the official Mercurial documentation. - """ - complexity = ui.config('flake8', 'complexity', default=-1) - strict = ui.configbool('flake8', 'strict', default=True) - ignore = ui.config('flake8', 'ignore', default=None) - config = ui.config('flake8', 'config', default=None) - - paths = _get_files(repo, **kwargs) - - # We only want to pass ignore and max_complexity if they differ from the - # defaults so that we don't override a local configuration file - options = {} - if ignore: - options['ignore'] = ignore - if complexity > -1: - options['max_complexity'] = complexity - - flake8_style = get_style_guide(config_file=config, paths=['.'], - **options) - report = flake8_style.check_files(paths) - - if strict: - return report.total_errors - - return 0 - - -def run(command, raw_output=False, decode=True): - p = Popen(command.split(), stdout=PIPE, stderr=PIPE) - (stdout, stderr) = p.communicate() - # On python 3, subprocess.Popen returns bytes objects which expect - # endswith to be given a bytes object or a tuple of bytes but not native - # string objects. This is simply less mysterious than using b'.py' in the - # endswith method. That should work but might still fail horribly. - if decode: - if hasattr(stdout, 'decode'): - stdout = stdout.decode('utf-8') - if hasattr(stderr, 'decode'): - stderr = stderr.decode('utf-8') - if not raw_output: - stdout = [line.strip() for line in stdout.splitlines()] - stderr = [line.strip() for line in stderr.splitlines()] - return (p.returncode, stdout, stderr) - - -def _get_files(repo, **kwargs): - seen = set() - for rev in range(repo[kwargs['node']], len(repo)): - for file_ in repo[rev].files(): - file_ = os.path.join(repo.root, file_) - if file_ in seen or not os.path.exists(file_): - continue - seen.add(file_) - if file_.endswith('.py'): - yield file_ - - -def find_vcs(): - try: - _, git_dir, _ = run('git rev-parse --git-dir') - except OSError: - pass - else: - if git_dir and os.path.isdir(git_dir[0]): - if not os.path.isdir(os.path.join(git_dir[0], 'hooks')): - os.mkdir(os.path.join(git_dir[0], 'hooks')) - return os.path.join(git_dir[0], 'hooks', 'pre-commit') - try: - _, hg_dir, _ = run('hg root') - except OSError: - pass - else: - if hg_dir and os.path.isdir(hg_dir[0]): - return os.path.join(hg_dir[0], '.hg', 'hgrc') - return '' - - -def get_git_config(option, opt_type='', convert_type=True): - # type can be --bool, --int or an empty string - _, git_cfg_value, _ = run('git config --get %s %s' % (opt_type, option), - raw_output=True) - git_cfg_value = git_cfg_value.strip() - if not convert_type: - return git_cfg_value - if opt_type == '--bool': - git_cfg_value = git_cfg_value.lower() == 'true' - elif git_cfg_value and opt_type == '--int': - git_cfg_value = int(git_cfg_value) - return git_cfg_value - - -_params = { - 'FLAKE8_COMPLEXITY': '--int', - 'FLAKE8_STRICT': '--bool', - 'FLAKE8_IGNORE': '', - 'FLAKE8_LAZY': '--bool', -} - - -def get_git_param(option, default=''): - global _params - opt_type = _params[option] - param_value = get_git_config(option.lower().replace('_', '.'), - opt_type=opt_type, convert_type=False) - if param_value == '': - param_value = os.environ.get(option, default) - if opt_type == '--bool' and not isinstance(param_value, bool): - param_value = param_value.lower() == 'true' - elif param_value and opt_type == '--int': - param_value = int(param_value) - return param_value - - -git_hook_file = """#!/usr/bin/env python -import sys -from flake8.hooks import git_hook, get_git_param - -# `get_git_param` will retrieve configuration from your local git config and -# then fall back to using the environment variables that the hook has always -# supported. -# For example, to set the complexity, you'll need to do: -# git config flake8.complexity 10 -COMPLEXITY = get_git_param('FLAKE8_COMPLEXITY', 10) -STRICT = get_git_param('FLAKE8_STRICT', False) -IGNORE = get_git_param('FLAKE8_IGNORE', None) -LAZY = get_git_param('FLAKE8_LAZY', False) - -if __name__ == '__main__': - sys.exit(git_hook( - complexity=COMPLEXITY, - strict=STRICT, - ignore=IGNORE, - lazy=LAZY, - )) -""" - - -def _install_hg_hook(path): - getenv = os.environ.get - if not os.path.isfile(path): - # Make the file so we can avoid IOError's - open(path, 'w').close() - - c = ConfigParser() - c.readfp(open(path, 'r')) - if not c.has_section('hooks'): - c.add_section('hooks') - - if not c.has_option('hooks', 'commit'): - c.set('hooks', 'commit', 'python:flake8.hooks.hg_hook') - - if not c.has_option('hooks', 'qrefresh'): - c.set('hooks', 'qrefresh', 'python:flake8.hooks.hg_hook') - - if not c.has_section('flake8'): - c.add_section('flake8') - - if not c.has_option('flake8', 'complexity'): - c.set('flake8', 'complexity', str(getenv('FLAKE8_COMPLEXITY', 10))) - - if not c.has_option('flake8', 'strict'): - c.set('flake8', 'strict', getenv('FLAKE8_STRICT', False)) - - if not c.has_option('flake8', 'ignore'): - c.set('flake8', 'ignore', getenv('FLAKE8_IGNORE', '')) - - if not c.has_option('flake8', 'lazy'): - c.set('flake8', 'lazy', getenv('FLAKE8_LAZY', False)) - - with open(path, 'w') as fd: - c.write(fd) - - -def install_hook(): - vcs = find_vcs() - - if not vcs: - p = get_parser()[0] - sys.stderr.write('Error: could not find either a git or mercurial ' - 'directory. Please re-run this in a proper ' - 'repository.\n') - p.print_help() - sys.exit(1) - - status = 0 - if 'git' in vcs: - if os.path.exists(vcs): - sys.exit('Error: hook already exists (%s)' % vcs) - with open(vcs, 'w') as fd: - fd.write(git_hook_file) - # rwxr--r-- - os.chmod(vcs, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) - elif 'hg' in vcs: - _install_hg_hook(vcs) - else: - status = 1 - - sys.exit(status) diff --git a/flake8/main.py b/flake8/main.py deleted file mode 100644 index 9b1b0a6..0000000 --- a/flake8/main.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import re -import sys - -import pycodestyle as pep8 -import setuptools - -from flake8.engine import get_parser, get_style_guide -from flake8.util import option_normalizer - -if sys.platform.startswith('win'): - USER_CONFIG = os.path.expanduser(r'~\.flake8') -else: - USER_CONFIG = os.path.join( - os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), - 'flake8' - ) - -pep8.USER_CONFIG = USER_CONFIG - -EXTRA_IGNORE = [] - - -def main(): - """Parse options and run checks on Python source.""" - # Prepare - flake8_style = get_style_guide(parse_argv=True) - options = flake8_style.options - - if options.install_hook: - from flake8.hooks import install_hook - install_hook() - - # Run the checkers - report = flake8_style.check_files() - - exit_code = print_report(report, flake8_style) - if exit_code > 0: - raise SystemExit(exit_code > 0) - - -def print_report(report, flake8_style): - # Print the final report - options = flake8_style.options - if options.statistics: - report.print_statistics() - if options.benchmark: - report.print_benchmark() - if report.total_errors: - if options.count: - sys.stderr.write(str(report.total_errors) + '\n') - if not options.exit_zero: - return 1 - return 0 - - -def check_file(path, ignore=(), complexity=-1): - """Checks a file using pep8 and pyflakes by default and mccabe - optionally. - - :param str path: path to the file to be checked - :param tuple ignore: (optional), error and warning codes to be ignored - :param int complexity: (optional), enables the mccabe check for values > 0 - """ - ignore = set(ignore).union(EXTRA_IGNORE) - flake8_style = get_style_guide(ignore=ignore, max_complexity=complexity) - return flake8_style.input_file(path) - - -def check_code(code, ignore=(), complexity=-1): - """Checks code using pep8 and pyflakes by default and mccabe optionally. - - :param str code: code to be checked - :param tuple ignore: (optional), error and warning codes to be ignored - :param int complexity: (optional), enables the mccabe check for values > 0 - """ - ignore = set(ignore).union(EXTRA_IGNORE) - flake8_style = get_style_guide(ignore=ignore, max_complexity=complexity) - return flake8_style.input_file(None, lines=code.splitlines(True)) - - -class Flake8Command(setuptools.Command): - """The :class:`Flake8Command` class is used by setuptools to perform - checks on registered modules. - """ - - description = "Run flake8 on modules registered in setuptools" - user_options = [] - - def initialize_options(self): - self.option_to_cmds = {} - parser = get_parser()[0] - for opt in parser.option_list: - cmd_name = opt._long_opts[0][2:] - option_name = cmd_name.replace('-', '_') - self.option_to_cmds[option_name] = opt - setattr(self, option_name, None) - - def finalize_options(self): - self.options_dict = {} - for (option_name, opt) in self.option_to_cmds.items(): - if option_name in ['help', 'verbose']: - continue - value = getattr(self, option_name) - if value is None: - continue - value = option_normalizer(value, opt, option_name) - # Check if there's any values that need to be fixed. - if option_name == "include" and isinstance(value, str): - value = re.findall('[^,;\s]+', value) - - self.options_dict[option_name] = value - - def distribution_files(self): - if self.distribution.packages: - package_dirs = self.distribution.package_dir or {} - for package in self.distribution.packages: - pkg_dir = package - if package in package_dirs: - pkg_dir = package_dirs[package] - elif '' in package_dirs: - pkg_dir = package_dirs[''] + os.path.sep + pkg_dir - yield pkg_dir.replace('.', os.path.sep) - - if self.distribution.py_modules: - for filename in self.distribution.py_modules: - yield "%s.py" % filename - # Don't miss the setup.py file itself - yield "setup.py" - - def run(self): - # Prepare - paths = list(self.distribution_files()) - flake8_style = get_style_guide(paths=paths, **self.options_dict) - - # Run the checkers - report = flake8_style.check_files() - exit_code = print_report(report, flake8_style) - if exit_code > 0: - raise SystemExit(exit_code > 0) diff --git a/flake8/reporter.py b/flake8/reporter.py deleted file mode 100644 index cea3717..0000000 --- a/flake8/reporter.py +++ /dev/null @@ -1,152 +0,0 @@ -# -*- coding: utf-8 -*- -# Adapted from a contribution of Johan Dahlin - -import collections -import errno -import re -import sys -try: - import multiprocessing -except ImportError: # Python 2.5 - multiprocessing = None - -import pycodestyle as pep8 - -__all__ = ['multiprocessing', 'BaseQReport', 'QueueReport'] - - -class BaseQReport(pep8.BaseReport): - """Base Queue Report.""" - _loaded = False # Windows support - - # Reasoning for ignored error numbers is in-line below - ignored_errors = set([ - # EPIPE: Added by sigmavirus24 - # > If output during processing is piped to something that may close - # > its own stdin before we've finished printing results, we need to - # > catch a Broken pipe error and continue on. - # > (See also: https://gitlab.com/pycqa/flake8/issues/69) - errno.EPIPE, - # NOTE(sigmavirus24): When adding to this list, include the reasoning - # on the lines before the error code and always append your error - # code. Further, please always add a trailing `,` to reduce the visual - # noise in diffs. - ]) - - def __init__(self, options): - assert options.jobs > 0 - super(BaseQReport, self).__init__(options) - self.counters = collections.defaultdict(int) - self.n_jobs = options.jobs - - # init queues - self.task_queue = multiprocessing.Queue() - self.result_queue = multiprocessing.Queue() - if sys.platform == 'win32': - # Work around http://bugs.python.org/issue10845 - sys.modules['__main__'].__file__ = __file__ - - def _cleanup_queue(self, queue): - while not queue.empty(): - queue.get_nowait() - - def _put_done(self): - # collect queues - for i in range(self.n_jobs): - self.task_queue.put('DONE') - self.update_state(self.result_queue.get()) - - def _process_main(self): - if not self._loaded: - # Windows needs to parse again the configuration - from flake8.main import get_style_guide - get_style_guide(parse_argv=True) - for filename in iter(self.task_queue.get, 'DONE'): - self.input_file(filename) - - def start(self): - super(BaseQReport, self).start() - self.__class__._loaded = True - # spawn processes - for i in range(self.n_jobs): - p = multiprocessing.Process(target=self.process_main) - p.daemon = True - p.start() - - def stop(self): - try: - self._put_done() - except KeyboardInterrupt: - pass - finally: - # cleanup queues to unlock threads - self._cleanup_queue(self.result_queue) - self._cleanup_queue(self.task_queue) - super(BaseQReport, self).stop() - - def process_main(self): - try: - self._process_main() - except KeyboardInterrupt: - pass - except IOError as ioerr: - # If we happen across an IOError that we aren't certain can/should - # be ignored, we should re-raise the exception. - if ioerr.errno not in self.ignored_errors: - raise - finally: - # ensure all output is flushed before main process continues - sys.stdout.flush() - sys.stderr.flush() - self.result_queue.put(self.get_state()) - - def get_state(self): - return {'total_errors': self.total_errors, - 'counters': self.counters, - 'messages': self.messages} - - def update_state(self, state): - self.total_errors += state['total_errors'] - for key, value in state['counters'].items(): - self.counters[key] += value - self.messages.update(state['messages']) - - -class FileQReport(BaseQReport): - """File Queue Report.""" - print_filename = True - - -class QueueReport(pep8.StandardReport, BaseQReport): - """Standard Queue Report.""" - - def get_file_results(self): - """Print the result and return the overall count for this file.""" - self._deferred_print.sort() - - for line_number, offset, code, text, doc in self._deferred_print: - print(self._fmt % { - 'path': self.filename, - 'row': self.line_offset + line_number, 'col': offset + 1, - 'code': code, 'text': text, - }) - # stdout is block buffered when not stdout.isatty(). - # line can be broken where buffer boundary since other processes - # write to same file. - # flush() after print() to avoid buffer boundary. - # Typical buffer size is 8192. line written safely when - # len(line) < 8192. - sys.stdout.flush() - if self._show_source: - if line_number > len(self.lines): - line = '' - else: - line = self.lines[line_number - 1] - print(line.rstrip()) - sys.stdout.flush() - print(re.sub(r'\S', ' ', line[:offset]) + '^') - sys.stdout.flush() - if self._show_pep8 and doc: - print(' ' + doc.strip()) - sys.stdout.flush() - return self.file_errors diff --git a/flake8/run.py b/flake8/run.py deleted file mode 100644 index aca929e..0000000 --- a/flake8/run.py +++ /dev/null @@ -1,11 +0,0 @@ - -""" -Implementation of the command-line I{flake8} tool. -""" -from flake8.hooks import git_hook, hg_hook # noqa -from flake8.main import check_code, check_file, Flake8Command # noqa -from flake8.main import main - - -if __name__ == '__main__': - main() diff --git a/flake8/tests/__init__.py b/flake8/tests/__init__.py deleted file mode 100644 index 792d600..0000000 --- a/flake8/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/flake8/tests/_test_warnings.py b/flake8/tests/_test_warnings.py deleted file mode 100644 index e329199..0000000 --- a/flake8/tests/_test_warnings.py +++ /dev/null @@ -1,309 +0,0 @@ -""" - _test_warnings.py - - Tests for the warnings that are emitted by flake8. - - This module is named _test_warnings instead of test_warnings so that a - normal nosetests run does not collect it. The tests in this module pass - when they are run alone, but they fail when they are run along with other - tests (nosetests --with-isolation doesn't help). - - In tox.ini, these tests are run separately. - -""" - -from __future__ import with_statement - -import os -import warnings -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import engine -from flake8.util import is_windows - -# The Problem -# ------------ -# -# Some of the tests in this module pass when this module is run on its own, but -# they fail when this module is run as part of the whole test suite. These are -# the problematic tests: -# -# test_jobs_verbose -# test_stdin_jobs_warning -# -# On some platforms, the warnings.capture_warnings function doesn't work -# properly when run with the other flake8 tests. It drops some warnings, even -# though the warnings filter is set to 'always'. However, when run separately, -# these tests pass. -# -# This problem only occurs on Windows, with Python 3.3 and older. Maybe it's -# related to PEP 446 - Inheritable file descriptors? -# -# -# -# -# Things that didn't work -# ------------ -# -# Nose --attr -# I tried using the nosetests --attr feature to run the tests separately. I -# put the following in setup.cfg -# -# [nosetests] -# atttr=!run_alone -# -# Then I added a tox section thst did this -# -# nosetests --attr=run_alone -# -# However, the command line --attr would not override the config file --attr, -# so the special tox section wound up runing all the tests, and failing. -# -# -# -# Nose --with-isolation -# The nosetests --with-isolation flag did not help. -# -# -# -# unittest.skipIf -# I tried decorating the problematic tests with the unittest.skipIf -# decorator. -# -# @unittest.skipIf(is_windows() and sys.version_info < (3, 4), -# "Fails on Windows with Python < 3.4 when run with other" -# " tests.") -# -# The idea is, skip the tests in the main test run, on affected platforms. -# Then, only on those platforms, come back in later and run the tests -# separately. -# -# I added a new stanza to tox.ini, to run the tests separately on the -# affected platforms. -# -# nosetests --no-skip -# -# I ran in to a bug in the nosetests skip plugin. It would report the test as -# having been run, but it would not actually run the test. So, when run with -# --no-skip, the following test would be reported as having run and passed! -# -# @unittest.skip("This passes o_o") -# def test_should_fail(self): -# assert 0 -# -# This bug has been reported here: -# "--no-skip broken with Python 2.7" -# https://github.com/nose-devs/nose/issues/512 -# -# -# -# py.test -# -# I tried using py.test, and its @pytest.mark.xfail decorator. I added some -# separate stanzas in tox, and useing the pytest --runxfail option to run the -# tests separately. This allows us to run all the tests together, on -# platforms that allow it. On platforms that don't allow us to run the tests -# all together, this still runs all the tests, but in two separate steps. -# -# This is the same solution as the nosetests --no-skip solution I described -# above, but --runxfail does not have the same bug as --no-skip. -# -# This has the advantage that all tests are discoverable by default, outside -# of tox. However, nose does not recognize the pytest.mark.xfail decorator. -# So, if a user runs nosetests, it still tries to run the problematic tests -# together with the rest of the test suite, causing them to fail. -# -# -# -# -# -# -# Solution -# ------------ -# Move the problematic tests to _test_warnings.py, so nose.collector will not -# find them. Set up a separate section in tox.ini that runs this: -# -# nosetests flake8.tests._test_warnings -# -# This allows all tests to pass on all platforms, when run through tox. -# However, it means that, even on unaffected platforms, the problematic tests -# are not discovered and run outside of tox (if the user just runs nosetests -# manually, for example). - - -class IntegrationTestCaseWarnings(unittest.TestCase): - """Integration style tests to check that warnings are issued properly for - different command line options.""" - - windows_warning_text = ("The --jobs option is not available on Windows." - " Ignoring --jobs arguments.") - stdin_warning_text = ("The --jobs option is not compatible with" - " supplying input using - . Ignoring --jobs" - " arguments.") - - def this_file(self): - """Return the real path of this file.""" - this_file = os.path.realpath(__file__) - if this_file.endswith("pyc"): - this_file = this_file[:-1] - return this_file - - @staticmethod - def get_style_guide_with_warnings(engine, *args, **kwargs): - """ - Return a style guide object (obtained by calling - engine.get_style_guide) and a list of the warnings that were raised in - the process. - - Note: not threadsafe - """ - - # Note - # https://docs.python.org/2/library/warnings.html - # - # The catch_warnings manager works by replacing and then later - # restoring the module's showwarning() function and internal list of - # filter specifications. This means the context manager is modifying - # global state and therefore is not thread-safe - - with warnings.catch_warnings(record=True) as collected_warnings: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - - # Get the style guide - style_guide = engine.get_style_guide(*args, **kwargs) - - # Now that the warnings have been collected, return the style guide and - # the warnings. - return (style_guide, collected_warnings) - - def verify_warnings(self, collected_warnings, expected_warnings): - """ - Verifies that collected_warnings is a sequence that contains user - warnings that match the sequence of string values passed in as - expected_warnings. - """ - if expected_warnings is None: - expected_warnings = [] - - collected_user_warnings = [w for w in collected_warnings - if issubclass(w.category, UserWarning)] - - self.assertEqual(len(collected_user_warnings), - len(expected_warnings)) - - collected_warnings_set = set(str(warning.message) - for warning - in collected_user_warnings) - expected_warnings_set = set(expected_warnings) - self.assertEqual(collected_warnings_set, expected_warnings_set) - - def check_files_collect_warnings(self, - arglist=[], - explicit_stdin=False, - count=0, - verbose=False): - """Call check_files and collect any warnings that are issued.""" - if verbose: - arglist.append('--verbose') - if explicit_stdin: - target_file = "-" - else: - target_file = self.this_file() - argv = ['flake8'] + arglist + [target_file] - with mock.patch("sys.argv", argv): - (style_guide, - collected_warnings, - ) = self.get_style_guide_with_warnings(engine, - parse_argv=True) - report = style_guide.check_files() - self.assertEqual(report.total_errors, count) - return style_guide, report, collected_warnings - - def check_files_no_warnings_allowed(self, - arglist=[], - explicit_stdin=False, - count=0, - verbose=False): - """Call check_files, and assert that there were no warnings issued.""" - (style_guide, - report, - collected_warnings, - ) = self.check_files_collect_warnings(arglist=arglist, - explicit_stdin=explicit_stdin, - count=count, - verbose=verbose) - self.verify_warnings(collected_warnings, expected_warnings=None) - return style_guide, report - - def _job_tester(self, jobs, verbose=False): - # mock stdout.flush so we can count the number of jobs created - with mock.patch('sys.stdout.flush') as mocked: - (guide, - report, - collected_warnings, - ) = self.check_files_collect_warnings( - arglist=['--jobs=%s' % jobs], - verbose=verbose) - - if is_windows(): - # The code path where guide.options.jobs gets converted to an - # int is not run on windows. So, do the int conversion here. - self.assertEqual(int(guide.options.jobs), jobs) - # On windows, call count is always zero. - self.assertEqual(mocked.call_count, 0) - else: - self.assertEqual(guide.options.jobs, jobs) - self.assertEqual(mocked.call_count, jobs) - - expected_warings = [] - if verbose and is_windows(): - expected_warings.append(self.windows_warning_text) - self.verify_warnings(collected_warnings, expected_warings) - - def test_jobs(self, verbose=False): - self._job_tester(2, verbose=verbose) - self._job_tester(10, verbose=verbose) - - def test_no_args_no_warnings(self, verbose=False): - self.check_files_no_warnings_allowed(verbose=verbose) - - def test_stdin_jobs_warning(self, verbose=False): - self.count = 0 - - def fake_stdin(): - self.count += 1 - with open(self.this_file(), "r") as f: - return f.read() - - with mock.patch("pycodestyle.stdin_get_value", fake_stdin): - (style_guide, - report, - collected_warnings, - ) = self.check_files_collect_warnings(arglist=['--jobs=4'], - explicit_stdin=True, - verbose=verbose) - expected_warings = [] - if verbose: - expected_warings.append(self.stdin_warning_text) - if is_windows(): - expected_warings.append(self.windows_warning_text) - self.verify_warnings(collected_warnings, expected_warings) - self.assertEqual(self.count, 1) - - def test_jobs_verbose(self): - self.test_jobs(verbose=True) - - def test_no_args_no_warnings_verbose(self): - self.test_no_args_no_warnings(verbose=True) - - def test_stdin_jobs_warning_verbose(self): - self.test_stdin_jobs_warning(verbose=True) - - -if __name__ == '__main__': - unittest.main() diff --git a/flake8/tests/test_engine.py b/flake8/tests/test_engine.py deleted file mode 100644 index 5feafe1..0000000 --- a/flake8/tests/test_engine.py +++ /dev/null @@ -1,236 +0,0 @@ -from __future__ import with_statement - -import errno -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import engine, util, __version__, reporter -import pycodestyle as pep8 - - -class TestEngine(unittest.TestCase): - def setUp(self): - self.patches = {} - - def tearDown(self): - assert len(self.patches.items()) == 0 - - def start_patch(self, patch): - self.patches[patch] = mock.patch(patch) - return self.patches[patch].start() - - def stop_patches(self): - patches = self.patches.copy() - for k, v in patches.items(): - v.stop() - del(self.patches[k]) - - def test_get_style_guide(self): - with mock.patch('flake8.engine._register_extensions') as reg_ext: - reg_ext.return_value = ([], [], [], []) - g = engine.get_style_guide() - self.assertTrue(isinstance(g, engine.StyleGuide)) - reg_ext.assert_called_once_with() - - def test_get_style_guide_kwargs(self): - m = mock.Mock() - with mock.patch('flake8.engine.StyleGuide') as StyleGuide: - with mock.patch('flake8.engine.get_parser') as get_parser: - m.ignored_extensions = [] - StyleGuide.return_value.options.jobs = '42' - StyleGuide.return_value.options.diff = False - get_parser.return_value = (m, []) - engine.get_style_guide(foo='bar') - get_parser.assert_called_once_with() - StyleGuide.assert_called_once_with(**{'parser': m, 'foo': 'bar'}) - - def test_register_extensions(self): - with mock.patch('pycodestyle.register_check') as register_check: - registered_exts = engine._register_extensions() - self.assertTrue(isinstance(registered_exts[0], util.OrderedSet)) - self.assertTrue(len(registered_exts[0]) > 0) - for i in registered_exts[1:]: - self.assertTrue(isinstance(i, list)) - self.assertTrue(register_check.called) - - def test_disable_extensions(self): - parser = mock.MagicMock() - options = mock.MagicMock() - - parser.ignored_extensions = ['I123', 'I345', 'I678', 'I910'] - - options.enable_extensions = 'I345,\nI678,I910' - options.ignore = ('E121', 'E123') - - engine._disable_extensions(parser, options) - self.assertEqual(set(options.ignore), set(['E121', 'E123', 'I123'])) - - def test_get_parser(self): - # setup - re = self.start_patch('flake8.engine._register_extensions') - gpv = self.start_patch('flake8.engine.get_python_version') - pgp = self.start_patch('pycodestyle.get_parser') - m = mock.Mock() - re.return_value = ([('pyflakes', '0.7'), ('mccabe', '0.2')], [], [], - []) - gpv.return_value = 'Python Version' - pgp.return_value = m - # actual call we're testing - parser, hooks = engine.get_parser() - # assertions - self.assertTrue(re.called) - self.assertTrue(gpv.called) - pgp.assert_called_once_with( - 'flake8', - '%s (pyflakes: 0.7, mccabe: 0.2) Python Version' % __version__) - self.assertTrue(m.remove_option.called) - self.assertTrue(m.add_option.called) - self.assertEqual(parser, m) - self.assertEqual(hooks, []) - # clean-up - self.stop_patches() - - def test_get_python_version(self): - self.assertTrue('on' in engine.get_python_version()) - # Silly test but it will provide 100% test coverage - # Also we can never be sure (without reconstructing the string - # ourselves) what system we may be testing on. - - def test_windows_disables_jobs(self): - with mock.patch('flake8.util.is_windows') as is_windows: - is_windows.return_value = True - guide = engine.get_style_guide() - assert isinstance(guide, reporter.BaseQReport) is False - - def test_stdin_disables_jobs(self): - with mock.patch('flake8.util.is_using_stdin') as is_using_stdin: - is_using_stdin.return_value = True - guide = engine.get_style_guide() - assert isinstance(guide, reporter.BaseQReport) is False - - def test_disables_extensions_that_are_not_selected(self): - with mock.patch('flake8.engine._register_extensions') as re: - re.return_value = ([('fake_ext', '0.1a1')], [], [], ['X']) - sg = engine.get_style_guide() - assert 'X' in sg.options.ignore - - def test_enables_off_by_default_extensions(self): - with mock.patch('flake8.engine._register_extensions') as re: - re.return_value = ([('fake_ext', '0.1a1')], [], [], ['X']) - parser, options = engine.get_parser() - parser.parse_args(['--select=X']) - sg = engine.StyleGuide(parser=parser) - assert 'X' not in sg.options.ignore - - def test_load_entry_point_verifies_requirements(self): - entry_point = mock.Mock(spec=['require', 'resolve', 'load']) - - engine._load_entry_point(entry_point, verify_requirements=True) - entry_point.require.assert_called_once_with() - entry_point.resolve.assert_called_once_with() - - def test_load_entry_point_does_not_verify_requirements(self): - entry_point = mock.Mock(spec=['require', 'resolve', 'load']) - - engine._load_entry_point(entry_point, verify_requirements=False) - self.assertFalse(entry_point.require.called) - entry_point.resolve.assert_called_once_with() - - def test_load_entry_point_passes_require_argument_to_load(self): - entry_point = mock.Mock(spec=['load']) - - engine._load_entry_point(entry_point, verify_requirements=True) - entry_point.load.assert_called_once_with(require=True) - entry_point.reset_mock() - - engine._load_entry_point(entry_point, verify_requirements=False) - entry_point.load.assert_called_once_with(require=False) - - -def oserror_generator(error_number, message='Ominous OSError message'): - def oserror_side_effect(*args, **kwargs): - if hasattr(oserror_side_effect, 'used'): - return - - oserror_side_effect.used = True - raise OSError(error_number, message) - - return oserror_side_effect - - -class TestStyleGuide(unittest.TestCase): - def setUp(self): - mocked_styleguide = mock.Mock(spec=engine.NoQAStyleGuide) - self.styleguide = engine.StyleGuide(styleguide=mocked_styleguide) - self.mocked_sg = mocked_styleguide - - def test_proxies_excluded(self): - self.styleguide.excluded('file.py', parent='.') - - self.mocked_sg.excluded.assert_called_once_with('file.py', parent='.') - - def test_proxies_init_report(self): - reporter = object() - self.styleguide.init_report(reporter) - - self.mocked_sg.init_report.assert_called_once_with(reporter) - - def test_proxies_check_files(self): - self.styleguide.check_files(['foo', 'bar']) - - self.mocked_sg.check_files.assert_called_once_with( - paths=['foo', 'bar'] - ) - - def test_proxies_input_file(self): - self.styleguide.input_file('file.py', - lines=[9, 10], - expected='foo', - line_offset=20) - - self.mocked_sg.input_file.assert_called_once_with(filename='file.py', - lines=[9, 10], - expected='foo', - line_offset=20) - - def test_check_files_retries_on_specific_OSErrors(self): - self.mocked_sg.check_files.side_effect = oserror_generator( - errno.ENOSPC, 'No space left on device' - ) - - self.styleguide.check_files(['foo', 'bar']) - - self.mocked_sg.init_report.assert_called_once_with(pep8.StandardReport) - - def test_input_file_retries_on_specific_OSErrors(self): - self.mocked_sg.input_file.side_effect = oserror_generator( - errno.ENOSPC, 'No space left on device' - ) - - self.styleguide.input_file('file.py') - - self.mocked_sg.init_report.assert_called_once_with(pep8.StandardReport) - - def test_check_files_reraises_unknown_OSErrors(self): - self.mocked_sg.check_files.side_effect = oserror_generator( - errno.EADDRINUSE, - 'lol why are we talking about binding to sockets' - ) - - self.assertRaises(OSError, self.styleguide.check_files, - ['foo', 'bar']) - - def test_input_file_reraises_unknown_OSErrors(self): - self.mocked_sg.input_file.side_effect = oserror_generator( - errno.EADDRINUSE, - 'lol why are we talking about binding to sockets' - ) - - self.assertRaises(OSError, self.styleguide.input_file, - ['foo', 'bar']) - -if __name__ == '__main__': - unittest.main() diff --git a/flake8/tests/test_hooks.py b/flake8/tests/test_hooks.py deleted file mode 100644 index ba46794..0000000 --- a/flake8/tests/test_hooks.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Module containing the tests for flake8.hooks.""" -import os -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -import flake8.hooks -from flake8.util import is_windows - - -def excluded(filename): - return filename.endswith('afile.py') - - -class TestGitHook(unittest.TestCase): - if is_windows: - # On Windows, absolute paths start with a drive letter, for example C: - # Here we build a fake absolute path starting with the current drive - # letter, for example C:\fake\temp - current_drive, ignore_tail = os.path.splitdrive(os.getcwd()) - fake_abs_path = os.path.join(current_drive, os.path.sep, 'fake', 'tmp') - else: - fake_abs_path = os.path.join(os.path.sep, 'fake', 'tmp') - - @mock.patch('os.makedirs') - @mock.patch('flake8.hooks.open', create=True) - @mock.patch('shutil.rmtree') - @mock.patch('tempfile.mkdtemp', return_value=fake_abs_path) - @mock.patch('flake8.hooks.run', - return_value=(None, - [os.path.join('foo', 'afile.py'), - os.path.join('foo', 'bfile.py')], - None)) - @mock.patch('flake8.hooks.get_style_guide') - def test_prepends_tmp_directory_to_exclude(self, get_style_guide, run, - *args): - style_guide = get_style_guide.return_value = mock.Mock() - style_guide.options.exclude = [os.path.join('foo', 'afile.py')] - style_guide.options.filename = [os.path.join('foo', '*')] - style_guide.excluded = excluded - - flake8.hooks.git_hook() - - dirname, filename = os.path.split( - os.path.abspath(os.path.join('foo', 'bfile.py'))) - if is_windows: - # In Windows, the absolute path in dirname will start with a drive - # letter. Here, we discad the drive letter. - ignore_drive, dirname = os.path.splitdrive(dirname) - tmpdir = os.path.join(self.fake_abs_path, dirname[1:]) - tmpfile = os.path.join(tmpdir, 'bfile.py') - style_guide.check_files.assert_called_once_with([tmpfile]) - - -if __name__ == '__main__': - unittest.main() diff --git a/flake8/tests/test_integration.py b/flake8/tests/test_integration.py deleted file mode 100644 index f82a769..0000000 --- a/flake8/tests/test_integration.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import with_statement - -import os -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import engine -from flake8.util import is_windows - - -class IntegrationTestCase(unittest.TestCase): - """Integration style tests to exercise different command line options.""" - - def this_file(self): - """Return the real path of this file.""" - this_file = os.path.realpath(__file__) - if this_file.endswith("pyc"): - this_file = this_file[:-1] - return this_file - - def check_files(self, arglist=[], explicit_stdin=False, count=0): - """Call check_files.""" - if explicit_stdin: - target_file = "-" - else: - target_file = self.this_file() - argv = ['flake8'] + arglist + [target_file] - with mock.patch("sys.argv", argv): - style_guide = engine.get_style_guide(parse_argv=True) - report = style_guide.check_files() - self.assertEqual(report.total_errors, count) - return style_guide, report - - def test_no_args(self): - # assert there are no reported errors - self.check_files() - - def _job_tester(self, jobs): - # mock stdout.flush so we can count the number of jobs created - with mock.patch('sys.stdout.flush') as mocked: - guide, report = self.check_files(arglist=['--jobs=%s' % jobs]) - if is_windows(): - # The code path where guide.options.jobs gets converted to an - # int is not run on windows. So, do the int conversion here. - self.assertEqual(int(guide.options.jobs), jobs) - # On windows, call count is always zero. - self.assertEqual(mocked.call_count, 0) - else: - self.assertEqual(guide.options.jobs, jobs) - self.assertEqual(mocked.call_count, jobs) - - def test_jobs(self): - self._job_tester(2) - self._job_tester(10) - - def test_stdin(self): - self.count = 0 - - def fake_stdin(): - self.count += 1 - with open(self.this_file(), "r") as f: - return f.read() - - with mock.patch("pycodestyle.stdin_get_value", fake_stdin): - guide, report = self.check_files(arglist=['--jobs=4'], - explicit_stdin=True) - self.assertEqual(self.count, 1) - - def test_stdin_fail(self): - def fake_stdin(): - return "notathing\n" - with mock.patch("pycodestyle.stdin_get_value", fake_stdin): - # only assert needed is in check_files - guide, report = self.check_files(arglist=['--jobs=4'], - explicit_stdin=True, - count=1) diff --git a/flake8/tests/test_main.py b/flake8/tests/test_main.py deleted file mode 100644 index af08093..0000000 --- a/flake8/tests/test_main.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import with_statement - -import unittest - -import setuptools -from flake8 import main - - -class TestMain(unittest.TestCase): - def test_issue_39_regression(self): - distribution = setuptools.Distribution() - cmd = main.Flake8Command(distribution) - cmd.options_dict = {} - cmd.run() - - -if __name__ == '__main__': - unittest.main() diff --git a/flake8/tests/test_pyflakes.py b/flake8/tests/test_pyflakes.py deleted file mode 100644 index fb2f042..0000000 --- a/flake8/tests/test_pyflakes.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import with_statement - -import ast -import unittest - -from collections import namedtuple - -from flake8._pyflakes import FlakesChecker - -Options = namedtuple("Options", ['builtins', 'doctests', - 'include_in_doctest', - 'exclude_from_doctest']) - - -class TestFlakesChecker(unittest.TestCase): - - def setUp(self): - self.tree = ast.parse('print("cookies")') - - def test_doctest_flag_enabled(self): - options = Options(builtins=None, doctests=True, - include_in_doctest='', - exclude_from_doctest='') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, 'cookies.txt') - assert flake_checker.withDoctest is True - - def test_doctest_flag_disabled(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='', - exclude_from_doctest='') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, 'cookies.txt') - assert flake_checker.withDoctest is False - - def test_doctest_flag_enabled_exclude_file(self): - options = Options(builtins=None, doctests=True, - include_in_doctest='', - exclude_from_doctest='cookies.txt,' - 'hungry/cookies.txt') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is False - - def test_doctest_flag_disabled_include_file(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./cookies.txt,cake_yuck.txt', - exclude_from_doctest='') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is True - - def test_doctest_flag_disabled_include_file_exclude_dir(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./cookies.txt', - exclude_from_doctest='./') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is True - - def test_doctest_flag_disabled_include_dir_exclude_file(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./', - exclude_from_doctest='./cookies.txt') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is False - - def test_doctest_flag_disabled_include_file_exclude_file_error(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./cookies.txt', - exclude_from_doctest='./cookies.txt,cake_yuck.txt') - self.assertRaises(ValueError, FlakesChecker.parse_options, options) diff --git a/flake8/tests/test_reporter.py b/flake8/tests/test_reporter.py deleted file mode 100644 index f91bb52..0000000 --- a/flake8/tests/test_reporter.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import with_statement - -import errno -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import reporter - - -def ioerror_report_factory(errno_code): - class IOErrorBaseQReport(reporter.BaseQReport): - def _process_main(self): - raise IOError(errno_code, 'Fake bad pipe exception') - - options = mock.MagicMock() - options.jobs = 2 - return IOErrorBaseQReport(options) - - -class TestBaseQReport(unittest.TestCase): - def test_does_not_raise_a_bad_pipe_ioerror(self): - """Test that no EPIPE IOError exception is re-raised or leaked.""" - report = ioerror_report_factory(errno.EPIPE) - try: - report.process_main() - except IOError: - self.fail('BaseQReport.process_main raised an IOError for EPIPE' - ' but it should have caught this exception.') - - def test_raises_a_enoent_ioerror(self): - """Test that an ENOENT IOError exception is re-raised.""" - report = ioerror_report_factory(errno.ENOENT) - self.assertRaises(IOError, report.process_main) diff --git a/flake8/tests/test_util.py b/flake8/tests/test_util.py deleted file mode 100644 index 221a98e..0000000 --- a/flake8/tests/test_util.py +++ /dev/null @@ -1,121 +0,0 @@ -import optparse -import unittest - -from flake8.util import option_normalizer - - -class TestOptionSerializerParsesTrue(unittest.TestCase): - - def setUp(self): - self.option = optparse.Option('--foo', action='store_true') - self.option_name = 'fake_option' - - def test_1_is_true(self): - value = option_normalizer('1', self.option, self.option_name) - self.assertTrue(value) - - def test_T_is_true(self): - value = option_normalizer('T', self.option, self.option_name) - self.assertTrue(value) - - def test_TRUE_is_true(self): - value = option_normalizer('TRUE', self.option, self.option_name) - self.assertTrue(value, True) - - def test_ON_is_true(self): - value = option_normalizer('ON', self.option, self.option_name) - self.assertTrue(value) - - def test_t_is_true(self): - value = option_normalizer('t', self.option, self.option_name) - self.assertTrue(value) - - def test_true_is_true(self): - value = option_normalizer('true', self.option, self.option_name) - self.assertTrue(value) - - def test_on_is_true(self): - value = option_normalizer('on', self.option, self.option_name) - self.assertTrue(value) - - -class TestOptionSerializerParsesFalse(unittest.TestCase): - - def setUp(self): - self.option = optparse.Option('--foo', action='store_true') - self.option_name = 'fake_option' - - def test_0_is_false(self): - value = option_normalizer('0', self.option, self.option_name) - self.assertFalse(value) - - def test_F_is_false(self): - value = option_normalizer('F', self.option, self.option_name) - self.assertFalse(value) - - def test_FALSE_is_false(self): - value = option_normalizer('FALSE', self.option, self.option_name) - self.assertFalse(value) - - def test_OFF_is_false(self): - value = option_normalizer('OFF', self.option, self.option_name) - self.assertFalse(value) - - def test_f_is_false(self): - value = option_normalizer('f', self.option, self.option_name) - self.assertFalse(value) - - def test_false_is_false(self): - value = option_normalizer('false', self.option, self.option_name) - self.assertFalse(value) - - def test_off_is_false(self): - value = option_normalizer('off', self.option, self.option_name) - self.assertFalse(value) - - -class TestOptionSerializerParsesLists(unittest.TestCase): - - def setUp(self): - self.option = optparse.Option('--select') - self.option_name = 'select' - self.answer = ['F401', 'F402', 'F403', 'F404', 'F405'] - - def test_parses_simple_comma_separated_lists(self): - value = option_normalizer('F401,F402,F403,F404,F405', self.option, - self.option_name) - self.assertEqual(value, self.answer) - - def test_parses_less_simple_comma_separated_lists(self): - value = option_normalizer('F401 ,F402 ,F403 ,F404, F405', self.option, - self.option_name) - self.assertEqual(value, self.answer) - - value = option_normalizer('F401, F402, F403, F404, F405', self.option, - self.option_name) - self.assertEqual(value, self.answer) - - def test_parses_comma_separated_lists_with_newlines(self): - value = option_normalizer('''\ - F401, - F402, - F403, - F404, - F405, - ''', self.option, self.option_name) - self.assertEqual(value, self.answer) - - -class TestOptionSerializerParsesInts(unittest.TestCase): - - def setUp(self): - self.option = optparse.Option('--max-complexity', type='int') - self.option_name = 'max_complexity' - - def test_parses_an_int(self): - value = option_normalizer('2', self.option, self.option_name) - self.assertEqual(value, 2) - - -if __name__ == '__main__': - unittest.main() diff --git a/flake8/util.py b/flake8/util.py deleted file mode 100644 index 2751f6b..0000000 --- a/flake8/util.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import sys - -try: - import ast - iter_child_nodes = ast.iter_child_nodes -except ImportError: # Python 2.5 - import _ast as ast - - if 'decorator_list' not in ast.ClassDef._fields: - # Patch the missing attribute 'decorator_list' - ast.ClassDef.decorator_list = () - ast.FunctionDef.decorator_list = property(lambda s: s.decorators) - - def iter_child_nodes(node): - """ - Yield all direct child nodes of *node*, that is, all fields that - are nodes and all items of fields that are lists of nodes. - """ - if not node._fields: - return - for name in node._fields: - field = getattr(node, name, None) - if isinstance(field, ast.AST): - yield field - elif isinstance(field, list): - for item in field: - if isinstance(item, ast.AST): - yield item - - -class OrderedSet(list): - """List without duplicates.""" - __slots__ = () - - def add(self, value): - if value not in self: - self.append(value) - - -def is_windows(): - """Determine if the system is Windows.""" - return os.name == 'nt' - - -def is_using_stdin(paths): - """Determine if we're running checks on stdin.""" - return '-' in paths - - -def warn_when_using_jobs(options): - return (options.verbose and options.jobs and options.jobs.isdigit() and - int(options.jobs) > 1) - - -def force_disable_jobs(styleguide): - affected_mp_version = (sys.version_info <= (2, 7, 11) or - (3, 0) <= sys.version_info < (3, 2)) - return (is_windows() and affected_mp_version or - is_using_stdin(styleguide.paths)) - - -INT_TYPES = ('int', 'count') -BOOL_TYPES = ('store_true', 'store_false') -LIST_OPTIONS = ('select', 'ignore', 'exclude', 'enable_extensions') - - -def option_normalizer(value, option, option_name): - if option.action in BOOL_TYPES: - if str(value).upper() in ('1', 'T', 'TRUE', 'ON'): - value = True - if str(value).upper() in ('0', 'F', 'FALSE', 'OFF'): - value = False - elif option.type in INT_TYPES: - value = int(value) - elif option_name in LIST_OPTIONS: - if isinstance(value, str): - value = [opt.strip() for opt in value.split(',') if opt.strip()] - - return value diff --git a/run_tests.py b/run_tests.py deleted file mode 100755 index 09d6884..0000000 --- a/run_tests.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -import unittest -import os -import re -import sys -sys.path.insert(0, '.') - -TEST_DIR = 'flake8.tests' - - -def collect_tests(): - # list files in directory tests/ - names = os.listdir(TEST_DIR.replace('.', '/')) - regex = re.compile("(?!_+)\w+\.py$") - join = '.'.join - # Make a list of the names like 'tests.test_name' - names = [join([TEST_DIR, f[:-3]]) for f in names if regex.match(f)] - modules = [__import__(name, fromlist=[TEST_DIR]) for name in names] - load_tests = unittest.defaultTestLoader.loadTestsFromModule - suites = [load_tests(m) for m in modules] - suite = suites.pop() - for s in suites: - suite.addTests(s) - return suite - -if __name__ == "__main__": - suite = collect_tests() - res = unittest.TextTestRunner(verbosity=1).run(suite) - - # If it was successful, we don't want to exit with code 1 - raise SystemExit(not res.wasSuccessful()) diff --git a/setup.cfg b/setup.cfg index 89f3026..218ddcc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,17 @@ -[wheel] -universal = 1 +[aliases] +test=pytest -; Publish a universal wheel to PyPI: -; $ pip install -U pip wheel -; $ python setup.py sdist bdist_wheel upload +[bdist_wheel] +universal=1 + +[metadata] +requires-dist = + enum34; python_version<"3.4" + configparser; python_version<"3.2" + pyflakes >= 0.8.1, != 1.2.0, != 1.2.1, != 1.2.2, < 1.3.0 + pycodestyle >= 2.0.0, < 2.1.0 + mccabe >= 0.5.0, < 0.6.0 + +[pytest] +norecursedirs = .git .* *.egg* old docs dist build +addopts = -rw diff --git a/setup.py b/setup.py index 653ba01..0df57b7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,17 @@ +"""Packaging logic for Flake8.""" # -*- coding: utf-8 -*- from __future__ import with_statement -from setuptools import setup + +import functools +import os +import sys + +import setuptools + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) + +import flake8 # noqa + try: # Work around a traceback with Nose on Python 2.6 # http://bugs.python.org/issue15881#msg170215 @@ -16,48 +27,104 @@ except ImportError: mock = None -tests_require = ['nose'] +tests_require = ['pytest'] if mock is None: - tests_require += ['mock'] + tests_require.append('mock') -def get_version(fname='flake8/__init__.py'): - with open(fname) as f: - for line in f: - if line.startswith('__version__'): - return eval(line.split('=')[-1]) +requires = [ + "pyflakes >= 0.8.1, != 1.2.0, != 1.2.1, != 1.2.2, < 1.3.0", + "pycodestyle >= 2.0.0, < 2.1.0", + "mccabe >= 0.5.0, < 0.6.0", +] + +if sys.version_info < (3, 4): + requires.append("enum34") + +if sys.version_info < (3, 2): + requires.append("configparser") def get_long_description(): + """Generate a long description from the README and CHANGES files.""" descr = [] - for fname in ('README.rst', 'CHANGES.rst'): + for fname in ('README.rst',): with open(fname) as f: descr.append(f.read()) return '\n\n'.join(descr) +PEP8 = 'pycodestyle' +_FORMAT = '{0}.{1} = {0}:{1}' +PEP8_PLUGIN = functools.partial(_FORMAT.format, PEP8) -setup( + +setuptools.setup( name="flake8", license="MIT", - version=get_version(), + version=flake8.__version__, description="the modular source code checker: pep8, pyflakes and co", - long_description=get_long_description(), + # long_description=get_long_description(), author="Tarek Ziade", author_email="tarek@ziade.org", maintainer="Ian Cordasco", maintainer_email="graffatcolmingov@gmail.com", url="https://gitlab.com/pycqa/flake8", - packages=["flake8", "flake8.tests"], - install_requires=[ - "pyflakes >= 0.8.1, < 1.3, != 1.2.0, != 1.2.1, != 1.2.2", - "pycodestyle >= 2.0, < 2.1", - "mccabe >= 0.2.1, < 0.6", + package_dir={"": "src"}, + packages=[ + "flake8", + "flake8.api", + "flake8.formatting", + "flake8.main", + "flake8.options", + "flake8.plugins", ], + install_requires=requires, entry_points={ - 'distutils.commands': ['flake8 = flake8.main:Flake8Command'], - 'console_scripts': ['flake8 = flake8.main:main'], + 'distutils.commands': [ + 'flake8 = flake8.main.setuptools_command:Flake8' + ], + 'console_scripts': [ + 'flake8 = flake8.main.cli:main' + ], 'flake8.extension': [ - 'F = flake8._pyflakes:FlakesChecker', + 'F = flake8.plugins.pyflakes:FlakesChecker', + # PEP-0008 checks provied by PyCQA/pycodestyle + PEP8_PLUGIN('tabs_or_spaces'), + PEP8_PLUGIN('tabs_obsolete'), + PEP8_PLUGIN('trailing_whitespace'), + PEP8_PLUGIN('trailing_blank_lines'), + PEP8_PLUGIN('maximum_line_length'), + PEP8_PLUGIN('blank_lines'), + PEP8_PLUGIN('extraneous_whitespace'), + PEP8_PLUGIN('whitespace_around_keywords'), + PEP8_PLUGIN('missing_whitespace_after_import_keyword'), + PEP8_PLUGIN('missing_whitespace'), + PEP8_PLUGIN('indentation'), + PEP8_PLUGIN('continued_indentation'), + PEP8_PLUGIN('whitespace_before_parameters'), + PEP8_PLUGIN('whitespace_around_operator'), + PEP8_PLUGIN('missing_whitespace_around_operator'), + PEP8_PLUGIN('whitespace_around_comma'), + PEP8_PLUGIN('whitespace_around_named_parameter_equals'), + PEP8_PLUGIN('whitespace_before_comment'), + PEP8_PLUGIN('imports_on_separate_lines'), + PEP8_PLUGIN('module_imports_on_top_of_file'), + PEP8_PLUGIN('compound_statements'), + PEP8_PLUGIN('explicit_line_join'), + PEP8_PLUGIN('break_around_binary_operator'), + PEP8_PLUGIN('comparison_to_singleton'), + PEP8_PLUGIN('comparison_negative'), + PEP8_PLUGIN('comparison_type'), + # NOTE(sigmavirus24): Add this back once PyCodestyle 2.1.0 is out + # PEP8_PLUGIN('ambiguous_identifier'), + PEP8_PLUGIN('python_3000_has_key'), + PEP8_PLUGIN('python_3000_raise_comma'), + PEP8_PLUGIN('python_3000_not_equal'), + PEP8_PLUGIN('python_3000_backticks'), + ], + 'flake8.report': [ + 'default = flake8.formatting.default:Default', + 'pylint = flake8.formatting.default:Pylint', ], }, classifiers=[ @@ -66,10 +133,13 @@ setup( "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance", ], tests_require=tests_require, - test_suite='nose.collector', + setup_requires=['pytest-runner'], ) diff --git a/src/flake8/__init__.py b/src/flake8/__init__.py new file mode 100644 index 0000000..d6c6915 --- /dev/null +++ b/src/flake8/__init__.py @@ -0,0 +1,82 @@ +"""Top-level module for Flake8. + +This module + +- initializes logging for the command-line tool +- tracks the version of the package +- provides a way to configure logging for the command-line tool + +.. autofunction:: flake8.configure_logging + +""" +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + """Shim for version of Python < 2.7.""" + + def emit(self, record): + """Do nothing.""" + pass +import sys + +LOG = logging.getLogger(__name__) +LOG.addHandler(NullHandler()) + +# Clean up after LOG config +del NullHandler + +__version__ = '3.0.0b1' +__version_info__ = tuple(int(i) for i in __version__.split('.') if i.isdigit()) + + +# There is nothing lower than logging.DEBUG (10) in the logging library, +# but we want an extra level to avoid being too verbose when using -vv. +_EXTRA_VERBOSE = 5 +logging.addLevelName(_EXTRA_VERBOSE, 'VERBOSE') + +_VERBOSITY_TO_LOG_LEVEL = { + # output more than warnings but not debugging info + 1: logging.INFO, # INFO is a numerical level of 20 + # output debugging information + 2: logging.DEBUG, # DEBUG is a numerical level of 10 + # output extra verbose debugging information + 3: _EXTRA_VERBOSE, +} + +LOG_FORMAT = ('%(name)-25s %(processName)-11s %(relativeCreated)6d ' + '%(levelname)-8s %(message)s') + + +def configure_logging(verbosity, filename=None, logformat=LOG_FORMAT): + """Configure logging for flake8. + + :param int verbosity: + How verbose to be in logging information. + :param str filename: + Name of the file to append log information to. + If ``None`` this will log to ``sys.stderr``. + If the name is "stdout" or "stderr" this will log to the appropriate + stream. + """ + if verbosity <= 0: + return + if verbosity > 3: + verbosity = 3 + + log_level = _VERBOSITY_TO_LOG_LEVEL[verbosity] + + if not filename or filename in ('stderr', 'stdout'): + fileobj = getattr(sys, filename or 'stderr') + handler_cls = logging.StreamHandler + else: + fileobj = filename + handler_cls = logging.FileHandler + + handler = handler_cls(fileobj) + handler.setFormatter(logging.Formatter(logformat)) + LOG.addHandler(handler) + LOG.setLevel(log_level) + LOG.debug('Added a %s logging handler to logger root at %s', + filename, __name__) diff --git a/src/flake8/__main__.py b/src/flake8/__main__.py new file mode 100644 index 0000000..42bc428 --- /dev/null +++ b/src/flake8/__main__.py @@ -0,0 +1,4 @@ +"""Module allowing for ``python -m flake8 ...``.""" +from flake8.main import cli + +cli.main() diff --git a/src/flake8/api/__init__.py b/src/flake8/api/__init__.py new file mode 100644 index 0000000..9f95557 --- /dev/null +++ b/src/flake8/api/__init__.py @@ -0,0 +1,10 @@ +"""Module containing all public entry-points for Flake8. + +This is the only submodule in Flake8 with a guaranteed stable API. All other +submodules are considered internal only and are subject to change. +""" + + +def get_style_guide(**kwargs): + """Stub out the only function I'm aware of people using.""" + pass diff --git a/src/flake8/checker.py b/src/flake8/checker.py new file mode 100644 index 0000000..b875f44 --- /dev/null +++ b/src/flake8/checker.py @@ -0,0 +1,610 @@ +"""Checker Manager and Checker classes.""" +import errno +import logging +import os +import sys +import tokenize + +try: + import multiprocessing +except ImportError: + multiprocessing = None + +try: + import Queue as queue +except ImportError: + import queue + +from flake8 import defaults +from flake8 import exceptions +from flake8 import processor +from flake8 import utils + +LOG = logging.getLogger(__name__) + +SERIAL_RETRY_ERRNOS = set([ + # ENOSPC: Added by sigmavirus24 + # > On some operating systems (OSX), multiprocessing may cause an + # > ENOSPC error while trying to trying to create a Semaphore. + # > In those cases, we should replace the customized Queue Report + # > class with pep8's StandardReport class to ensure users don't run + # > into this problem. + # > (See also: https://gitlab.com/pycqa/flake8/issues/74) + errno.ENOSPC, + # NOTE(sigmavirus24): When adding to this list, include the reasoning + # on the lines before the error code and always append your error + # code. Further, please always add a trailing `,` to reduce the visual + # noise in diffs. +]) + + +class Manager(object): + """Manage the parallelism and checker instances for each plugin and file. + + This class will be responsible for the following: + + - Determining the parallelism of Flake8, e.g.: + + * Do we use :mod:`multiprocessing` or is it unavailable? + + * Do we automatically decide on the number of jobs to use or did the + user provide that? + + - Falling back to a serial way of processing files if we run into an + OSError related to :mod:`multiprocessing` + + - Organizing the results of each checker so we can group the output + together and make our output deterministic. + """ + + def __init__(self, style_guide, arguments, checker_plugins): + """Initialize our Manager instance. + + :param style_guide: + The instantiated style guide for this instance of Flake8. + :type style_guide: + flake8.style_guide.StyleGuide + :param list arguments: + The extra arguments parsed from the CLI (if any) + :param checker_plugins: + The plugins representing checks parsed from entry-points. + :type checker_plugins: + flake8.plugins.manager.Checkers + """ + self.arguments = arguments + self.style_guide = style_guide + self.options = style_guide.options + self.checks = checker_plugins + self.jobs = self._job_count() + self.process_queue = None + self.results_queue = None + self.statistics_queue = None + self.using_multiprocessing = self.jobs > 1 + self.processes = [] + self.checkers = [] + self.statistics = { + 'files': 0, + 'logical lines': 0, + 'physical lines': 0, + 'tokens': 0, + } + + if self.using_multiprocessing: + try: + self.process_queue = multiprocessing.Queue() + self.results_queue = multiprocessing.Queue() + self.statistics_queue = multiprocessing.Queue() + except OSError as oserr: + if oserr.errno not in SERIAL_RETRY_ERRNOS: + raise + self.using_multiprocessing = False + + @staticmethod + def _cleanup_queue(q): + while not q.empty(): + q.get_nowait() + + def _force_cleanup(self): + if self.using_multiprocessing: + for proc in self.processes: + proc.join(0.2) + self._cleanup_queue(self.process_queue) + self._cleanup_queue(self.results_queue) + self._cleanup_queue(self.statistics_queue) + + def _process_statistics(self): + all_statistics = self.statistics + if self.using_multiprocessing: + total_number_of_checkers = len(self.checkers) + statistics_gathered = 0 + while statistics_gathered < total_number_of_checkers: + try: + statistics = self.statistics_queue.get(block=False) + statistics_gathered += 1 + except queue.Empty: + break + + for statistic in defaults.STATISTIC_NAMES: + all_statistics[statistic] += statistics[statistic] + else: + statistics_generator = (checker.statistics + for checker in self.checkers) + for statistics in statistics_generator: + for statistic in defaults.STATISTIC_NAMES: + all_statistics[statistic] += statistics[statistic] + all_statistics['files'] += len(self.checkers) + + def _job_count(self): + # type: () -> Union[int, NoneType] + # First we walk through all of our error cases: + # - multiprocessing library is not present + # - we're running on windows in which case we know we have significant + # implemenation issues + # - the user provided stdin and that's not something we can handle + # well + # - we're processing a diff, which again does not work well with + # multiprocessing and which really shouldn't require multiprocessing + # - the user provided some awful input + if not multiprocessing: + LOG.warning('The multiprocessing module is not available. ' + 'Ignoring --jobs arguments.') + return 0 + + if (utils.is_windows() and + not utils.can_run_multiprocessing_on_windows()): + LOG.warning('The --jobs option is only available on Windows on ' + 'Python 2.7.11+ and 3.3+. We have detected that you ' + 'are running an unsupported version of Python on ' + 'Windows. Ignoring --jobs arguments.') + return 0 + + if utils.is_using_stdin(self.arguments): + LOG.warning('The --jobs option is not compatible with supplying ' + 'input using - . Ignoring --jobs arguments.') + return 0 + + if self.options.diff: + LOG.warning('The --diff option was specified with --jobs but ' + 'they are not compatible. Ignoring --jobs arguments.') + return 0 + + jobs = self.options.jobs + if jobs != 'auto' and not jobs.isdigit(): + LOG.warning('"%s" is not a valid parameter to --jobs. Must be one ' + 'of "auto" or a numerical value, e.g., 4.', jobs) + return 0 + + # If the value is "auto", we want to let the multiprocessing library + # decide the number based on the number of CPUs. However, if that + # function is not implemented for this particular value of Python we + # default to 1 + if jobs == 'auto': + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 0 + + # Otherwise, we know jobs should be an integer and we can just convert + # it to an integer + return int(jobs) + + def _results(self): + seen_done = 0 + LOG.info('Retrieving results') + while True: + result = self.results_queue.get() + if result == 'DONE': + seen_done += 1 + if seen_done >= self.jobs: + break + continue + + yield result + + def _handle_results(self, filename, results): + style_guide = self.style_guide + reported_results_count = 0 + for (error_code, line_number, column, text, physical_line) in results: + reported_results_count += style_guide.handle_error( + code=error_code, + filename=filename, + line_number=line_number, + column_number=column, + text=text, + physical_line=physical_line, + ) + return reported_results_count + + def _run_checks_from_queue(self): + LOG.info('Running checks in parallel') + for checker in iter(self.process_queue.get, 'DONE'): + LOG.debug('Running checker for file "%s"', checker.filename) + checker.run_checks(self.results_queue, self.statistics_queue) + self.results_queue.put('DONE') + + def is_path_excluded(self, path): + # type: (str) -> bool + """Check if a path is excluded. + + :param str path: + Path to check against the exclude patterns. + :returns: + True if there are exclude patterns and the path matches, + otherwise False. + :rtype: + bool + """ + exclude = self.options.exclude + if not exclude: + return False + basename = os.path.basename(path) + if utils.fnmatch(basename, exclude): + LOG.info('"%s" has been excluded', basename) + return True + + absolute_path = os.path.abspath(path) + match = utils.fnmatch(absolute_path, exclude) + LOG.info('"%s" has %sbeen excluded', absolute_path, + '' if match else 'not ') + return match + + def make_checkers(self, paths=None): + # type: (List[str]) -> NoneType + """Create checkers for each file.""" + if paths is None: + paths = self.arguments + filename_patterns = self.options.filename + + # NOTE(sigmavirus24): Yes this is a little unsightly, but it's our + # best solution right now. + def should_create_file_checker(filename): + """Determine if we should create a file checker.""" + matches_filename_patterns = utils.fnmatch( + filename, filename_patterns + ) + is_stdin = filename == '-' + file_exists = os.path.exists(filename) + return (file_exists and matches_filename_patterns) or is_stdin + + self.checkers = [ + FileChecker(filename, self.checks, self.style_guide) + for argument in paths + for filename in utils.filenames_from(argument, + self.is_path_excluded) + if should_create_file_checker(filename) + ] + + def report(self): + # type: () -> (int, int) + """Report all of the errors found in the managed file checkers. + + This iterates over each of the checkers and reports the errors sorted + by line number. + + :returns: + A tuple of the total results found and the results reported. + :rtype: + tuple(int, int) + """ + results_reported = results_found = 0 + for checker in self.checkers: + results = sorted(checker.results, key=lambda tup: (tup[2], tup[3])) + results_reported += self._handle_results(checker.filename, + results) + results_found += len(results) + return (results_found, results_reported) + + def run_parallel(self): + """Run the checkers in parallel.""" + LOG.info('Starting %d process workers', self.jobs) + for i in range(self.jobs): + proc = multiprocessing.Process( + target=self._run_checks_from_queue + ) + proc.daemon = True + proc.start() + self.processes.append(proc) + + final_results = {} + for (filename, results) in self._results(): + final_results[filename] = results + + for checker in self.checkers: + filename = checker.filename + checker.results = sorted(final_results.get(filename, []), + key=lambda tup: (tup[1], tup[2])) + + def run_serial(self): + """Run the checkers in serial.""" + for checker in self.checkers: + checker.run_checks(self.results_queue, self.statistics_queue) + + def run(self): + """Run all the checkers. + + This will intelligently decide whether to run the checks in parallel + or whether to run them in serial. + + If running the checks in parallel causes a problem (e.g., + https://gitlab.com/pycqa/flake8/issues/74) this also implements + fallback to serial processing. + """ + try: + if self.using_multiprocessing: + self.run_parallel() + else: + self.run_serial() + except OSError as oserr: + if oserr.errno not in SERIAL_RETRY_ERRNOS: + LOG.exception(oserr) + raise + LOG.warning('Running in serial after OS exception, %r', oserr) + self.run_serial() + + def start(self, paths=None): + """Start checking files. + + :param list paths: + Path names to check. This is passed directly to + :meth:`~Manager.make_checkers`. + """ + LOG.info('Making checkers') + self.make_checkers(paths) + if not self.using_multiprocessing: + return + + LOG.info('Populating process queue') + for checker in self.checkers: + self.process_queue.put(checker) + + for i in range(self.jobs): + self.process_queue.put('DONE') + + def stop(self): + """Stop checking files.""" + self._process_statistics() + for proc in self.processes: + LOG.info('Joining %s to the main process', proc.name) + proc.join() + + +class FileChecker(object): + """Manage running checks for a file and aggregate the results.""" + + def __init__(self, filename, checks, style_guide): + """Initialize our file checker. + + :param str filename: + Name of the file to check. + :param checks: + The plugins registered to check the file. + :type checks: + flake8.plugins.manager.Checkers + :param style_guide: + The initialized StyleGuide for this particular run. + :type style_guide: + flake8.style_guide.StyleGuide + """ + self.filename = filename + self.checks = checks + self.style_guide = style_guide + self.results = [] + self.processor = self._make_processor() + self.statistics = { + 'tokens': 0, + 'logical lines': 0, + 'physical lines': len(self.processor.lines), + } + + def _make_processor(self): + try: + return processor.FileProcessor(self.filename, + self.style_guide.options) + except IOError: + # If we can not read the file due to an IOError (e.g., the file + # does not exist or we do not have the permissions to open it) + # then we need to format that exception for the user. + # NOTE(sigmavirus24): Historically, pep8 has always reported this + # as an E902. We probably *want* a better error code for this + # going forward. + (exc_type, exception) = sys.exc_info()[:2] + message = '{0}: {1}'.format(exc_type.__name__, exception) + self.report('E902', 0, 0, message) + return None + + def report(self, error_code, line_number, column, text): + # type: (str, int, int, str) -> str + """Report an error by storing it in the results list.""" + if error_code is None: + error_code, text = text.split(' ', 1) + + physical_line = '' + # If we're recovering from a problem in _make_processor, we will not + # have this attribute. + if getattr(self, 'processor', None): + physical_line = self.processor.line_for(line_number) + + error = (error_code, line_number, column, text, physical_line) + self.results.append(error) + return error_code + + def run_check(self, plugin, **arguments): + """Run the check in a single plugin.""" + LOG.debug('Running %r with %r', plugin, arguments) + self.processor.keyword_arguments_for(plugin.parameters, arguments) + return plugin.execute(**arguments) + + def run_ast_checks(self): + """Run all checks expecting an abstract syntax tree.""" + try: + ast = self.processor.build_ast() + except (ValueError, SyntaxError, TypeError): + (exc_type, exception) = sys.exc_info()[:2] + if len(exception.args) > 1: + offset = exception.args[1] + if len(offset) > 2: + offset = offset[1:3] + else: + offset = (1, 0) + + self.report('E999', offset[0], offset[1], '%s: %s' % + (exc_type.__name__, exception.args[0])) + return + + for plugin in self.checks.ast_plugins: + checker = self.run_check(plugin, tree=ast) + # NOTE(sigmavirus24): If we want to allow for AST plugins that are + # not classes exclusively, we can do the following: + # retrieve_results = getattr(checker, 'run', lambda: checker) + # Otherwise, we just call run on the checker + for (line_number, offset, text, check) in checker.run(): + self.report( + error_code=None, + line_number=line_number, + column=offset, + text=text, + ) + + def run_logical_checks(self): + """Run all checks expecting a logical line.""" + comments, logical_line, mapping = self.processor.build_logical_line() + if not mapping: + return + self.processor.update_state(mapping) + + LOG.debug('Logical line: "%s"', logical_line.rstrip()) + + for plugin in self.checks.logical_line_plugins: + self.processor.update_checker_state_for(plugin) + results = self.run_check(plugin, logical_line=logical_line) or () + for offset, text in results: + offset = find_offset(offset, mapping) + line_number, column_offset = offset + self.report( + error_code=None, + line_number=line_number, + column=column_offset, + text=text, + ) + + self.processor.next_logical_line() + + def run_physical_checks(self, physical_line): + """Run all checks for a given physical line.""" + for plugin in self.checks.physical_line_plugins: + self.processor.update_checker_state_for(plugin) + result = self.run_check(plugin, physical_line=physical_line) + if result is not None: + column_offset, text = result + error_code = self.report( + error_code=None, + line_number=self.processor.line_number, + column=column_offset, + text=text, + ) + + self.processor.check_physical_error(error_code, physical_line) + + def process_tokens(self): + """Process tokens and trigger checks. + + This can raise a :class:`flake8.exceptions.InvalidSyntax` exception. + Instead of using this directly, you should use + :meth:`flake8.checker.FileChecker.run_checks`. + """ + parens = 0 + statistics = self.statistics + file_processor = self.processor + for token in file_processor.generate_tokens(): + statistics['tokens'] += 1 + self.check_physical_eol(token) + token_type, text = token[0:2] + processor.log_token(LOG, token) + if token_type == tokenize.OP: + parens = processor.count_parentheses(parens, text) + elif parens == 0: + if processor.token_is_newline(token): + self.handle_newline(token_type) + elif (processor.token_is_comment(token) and + len(file_processor.tokens) == 1): + self.handle_comment(token, text) + + if file_processor.tokens: + # If any tokens are left over, process them + self.run_physical_checks(file_processor.lines[-1]) + self.run_logical_checks() + + def run_checks(self, results_queue, statistics_queue): + """Run checks against the file.""" + if self.processor.should_ignore_file(): + return + + try: + self.process_tokens() + except exceptions.InvalidSyntax as exc: + self.report(exc.error_code, exc.line_number, exc.column_number, + exc.error_message) + + self.run_ast_checks() + + if results_queue is not None: + results_queue.put((self.filename, self.results)) + + logical_lines = self.processor.statistics['logical lines'] + self.statistics['logical lines'] = logical_lines + if statistics_queue is not None: + statistics_queue.put(self.statistics) + + def handle_comment(self, token, token_text): + """Handle the logic when encountering a comment token.""" + # The comment also ends a physical line + token = list(token) + token[1] = token_text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.processor.tokens = [tuple(token)] + self.run_logical_checks() + + def handle_newline(self, token_type): + """Handle the logic when encountering a newline token.""" + if token_type == tokenize.NEWLINE: + self.run_logical_checks() + self.processor.reset_blank_before() + elif len(self.processor.tokens) == 1: + # The physical line contains only this token. + self.processor.visited_new_blank_line() + self.processor.delete_first_token() + else: + self.run_logical_checks() + + def check_physical_eol(self, token): + """Run physical checks if and only if it is at the end of the line.""" + if processor.is_eol_token(token): + # Obviously, a newline token ends a single physical line. + self.run_physical_checks(token[4]) + elif processor.is_multiline_string(token): + # Less obviously, a string that contains newlines is a + # multiline string, either triple-quoted or with internal + # newlines backslash-escaped. Check every physical line in the + # string *except* for the last one: its newline is outside of + # the multiline string, so we consider it a regular physical + # line, and will check it like any other physical line. + # + # Subtleties: + # - have to wind self.line_number back because initially it + # points to the last line of the string, and we want + # check_physical() to give accurate feedback + line_no = token[2][0] + with self.processor.inside_multiline(line_number=line_no): + for line in self.processor.split_line(token): + self.run_physical_checks(line + '\n') + + +def find_offset(offset, mapping): + """Find the offset tuple for a single offset.""" + if isinstance(offset, tuple): + return offset + + for token_offset, position in mapping: + if offset <= token_offset: + break + return (position[0], position[1] + offset - token_offset) diff --git a/src/flake8/defaults.py b/src/flake8/defaults.py new file mode 100644 index 0000000..d9f5a0b --- /dev/null +++ b/src/flake8/defaults.py @@ -0,0 +1,17 @@ +"""Constants that define defaults.""" + +EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' +IGNORE = 'E121,E123,E126,E226,E24,E704,W503,W504' +SELECT = 'E,F,W,C' +MAX_LINE_LENGTH = 79 + +TRUTHY_VALUES = set(['true', '1', 't']) + +# Other constants +WHITESPACE = frozenset(' \t') + +STATISTIC_NAMES = ( + 'logical lines', + 'physical lines', + 'tokens', +) diff --git a/src/flake8/exceptions.py b/src/flake8/exceptions.py new file mode 100644 index 0000000..5ff55a2 --- /dev/null +++ b/src/flake8/exceptions.py @@ -0,0 +1,96 @@ +"""Exception classes for all of Flake8.""" + + +class Flake8Exception(Exception): + """Plain Flake8 exception.""" + + pass + + +class FailedToLoadPlugin(Flake8Exception): + """Exception raised when a plugin fails to load.""" + + FORMAT = 'Flake8 failed to load plugin "%(name)s" due to %(exc)s.' + + def __init__(self, *args, **kwargs): + """Initialize our FailedToLoadPlugin exception.""" + self.plugin = kwargs.pop('plugin') + self.ep_name = self.plugin.name + self.original_exception = kwargs.pop('exception') + super(FailedToLoadPlugin, self).__init__(*args, **kwargs) + + def __str__(self): + """Return a nice string for our exception.""" + return self.FORMAT % {'name': self.ep_name, + 'exc': self.original_exception} + + +class InvalidSyntax(Flake8Exception): + """Exception raised when tokenizing a file fails.""" + + def __init__(self, *args, **kwargs): + """Initialize our InvalidSyntax exception.""" + self.original_exception = kwargs.pop('exception') + self.error_code = 'E902' + self.line_number = 1 + self.column_number = 0 + try: + self.error_message = self.original_exception.message + except AttributeError: + # On Python 3, the IOError is an OSError which has a + # strerror attribute instead of a message attribute + self.error_message = self.original_exception.strerror + super(InvalidSyntax, self).__init__(*args, **kwargs) + + +class HookInstallationError(Flake8Exception): + """Parent exception for all hooks errors.""" + + pass + + +class GitHookAlreadyExists(HookInstallationError): + """Exception raised when the git pre-commit hook file already exists.""" + + def __init__(self, *args, **kwargs): + """Initialize the path attribute.""" + self.path = kwargs.pop('path') + super(GitHookAlreadyExists, self).__init__(*args, **kwargs) + + def __str__(self): + """Provide a nice message regarding the exception.""" + msg = ('The Git pre-commit hook ({0}) already exists. To convince ' + 'Flake8 to install the hook, please remove the existing ' + 'hook.') + return msg.format(self.path) + + +class MercurialHookAlreadyExists(HookInstallationError): + """Exception raised when a mercurial hook is already configured.""" + + hook_name = None + + def __init__(self, *args, **kwargs): + """Initialize the relevant attributes.""" + self.path = kwargs.pop('path') + self.value = kwargs.pop('value') + super(MercurialHookAlreadyExists, self).__init__(*args, **kwargs) + + def __str__(self): + """Return a nicely formatted string for these errors.""" + msg = ('The Mercurial {0} hook already exists with "{1}" in {2}. ' + 'To convince Flake8 to install the hook, please remove the ' + '{0} configuration from the [hooks] section of your hgrc.') + return msg.format(self.hook_name, self.value, self.path) + + +class MercurialCommitHookAlreadyExists(MercurialHookAlreadyExists): + """Exception raised when the hg commit hook is already configured.""" + + hook_name = 'commit' + + +class MercurialQRefreshHookAlreadyExists(MercurialHookAlreadyExists): + """Exception raised when the hg commit hook is already configured.""" + + hook_name = 'qrefresh' diff --git a/src/flake8/formatting/__init__.py b/src/flake8/formatting/__init__.py new file mode 100644 index 0000000..bf44801 --- /dev/null +++ b/src/flake8/formatting/__init__.py @@ -0,0 +1 @@ +"""Submodule containing the default formatters for Flake8.""" diff --git a/src/flake8/formatting/base.py b/src/flake8/formatting/base.py new file mode 100644 index 0000000..4fda6f4 --- /dev/null +++ b/src/flake8/formatting/base.py @@ -0,0 +1,161 @@ +"""The base class and interface for all formatting plugins.""" +from __future__ import print_function + + +class BaseFormatter(object): + """Class defining the formatter interface. + + .. attribute:: options + + The options parsed from both configuration files and the command-line. + + .. attribute:: filename + + If specified by the user, the path to store the results of the run. + + .. attribute:: output_fd + + Initialized when the :meth:`start` is called. This will be a file + object opened for writing. + + .. attribute:: newline + + The string to add to the end of a line. This is only used when the + output filename has been specified. + """ + + def __init__(self, options): + """Initialize with the options parsed from config and cli. + + This also calls a hook, :meth:`after_init`, so subclasses do not need + to call super to call this method. + + :param optparse.Values options: + User specified configuration parsed from both configuration files + and the command-line interface. + """ + self.options = options + self.filename = options.output_file + self.output_fd = None + self.newline = '\n' + self.after_init() + + def after_init(self): + """Initialize the formatter further.""" + pass + + def start(self): + """Prepare the formatter to receive input. + + This defaults to initializing :attr:`output_fd` if :attr:`filename` + """ + if self.filename: + self.output_fd = open(self.filename, 'w') + + def handle(self, error): + """Handle an error reported by Flake8. + + This defaults to calling :meth:`format`, :meth:`show_source`, and + then :meth:`write`. To extend how errors are handled, override this + method. + + :param error: + This will be an instance of :class:`~flake8.style_guide.Error`. + :type error: + flake8.style_guide.Error + """ + line = self.format(error) + source = self.show_source(error) + self.write(line, source) + + def format(self, error): + """Format an error reported by Flake8. + + This method **must** be implemented by subclasses. + + :param error: + This will be an instance of :class:`~flake8.style_guide.Error`. + :type error: + flake8.style_guide.Error + :returns: + The formatted error string. + :rtype: + str + """ + raise NotImplementedError('Subclass of BaseFormatter did not implement' + ' format.') + + def show_benchmarks(self, benchmarks): + """Format and print the benchmarks.""" + # NOTE(sigmavirus24): The format strings are a little confusing, even + # to me, so here's a quick explanation: + # We specify the named value first followed by a ':' to indicate we're + # formatting the value. + # Next we use '<' to indicate we want the value left aligned. + # Then '10' is the width of the area. + # For floats, finally, we only want only want at most 3 digits after + # the decimal point to be displayed. This is the precision and it + # can not be specified for integers which is why we need two separate + # format strings. + float_format = '{value:<10.3} {statistic}'.format + int_format = '{value:<10} {statistic}'.format + for statistic, value in benchmarks: + if isinstance(value, int): + benchmark = int_format(statistic=statistic, value=value) + else: + benchmark = float_format(statistic=statistic, value=value) + self._write(benchmark) + + def show_source(self, error): + """Show the physical line generating the error. + + This also adds an indicator for the particular part of the line that + is reported as generating the problem. + + :param error: + This will be an instance of :class:`~flake8.style_guide.Error`. + :type error: + flake8.style_guide.Error + :returns: + The formatted error string if the user wants to show the source. + If the user does not want to show the source, this will return + ``None``. + :rtype: + str + """ + if not self.options.show_source: + return None + pointer = (' ' * error.column_number) + '^' + # Physical lines have a newline at the end, no need to add an extra + # one + return error.physical_line + pointer + + def _write(self, output): + """Handle logic of whether to use an output file or print().""" + if self.output_fd is not None: + self.output_fd.write(output + self.newline) + else: + print(output) + + def write(self, line, source): + """Write the line either to the output file or stdout. + + This handles deciding whether to write to a file or print to standard + out for subclasses. Override this if you want behaviour that differs + from the default. + + :param str line: + The formatted string to print or write. + :param str source: + The source code that has been formatted and associated with the + line of output. + """ + self._write(line) + if source: + self._write(source) + + def stop(self): + """Clean up after reporting is finished.""" + if self.output_fd is not None: + self.output_fd.close() + self.output_fd = None diff --git a/src/flake8/formatting/default.py b/src/flake8/formatting/default.py new file mode 100644 index 0000000..bef8c88 --- /dev/null +++ b/src/flake8/formatting/default.py @@ -0,0 +1,56 @@ +"""Default formatting class for Flake8.""" +from flake8.formatting import base + + +class SimpleFormatter(base.BaseFormatter): + """Simple abstraction for Default and Pylint formatter commonality. + + Sub-classes of this need to define an ``error_format`` attribute in order + to succeed. The ``format`` method relies on that attribute and expects the + ``error_format`` string to use the old-style formatting strings with named + parameters: + + * code + * text + * path + * row + * col + + """ + + error_format = None + + def format(self, error): + """Format and write error out. + + If an output filename is specified, write formatted errors to that + file. Otherwise, print the formatted error to standard out. + """ + return self.error_format % { + "code": error.code, + "text": error.text, + "path": error.filename, + "row": error.line_number, + "col": error.column_number, + } + + +class Default(SimpleFormatter): + """Default formatter for Flake8. + + This also handles backwards compatibility for people specifying a custom + format string. + """ + + error_format = '%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + + def after_init(self): + """Check for a custom format string.""" + if self.options.format.lower() != 'default': + self.error_format = self.options.format + + +class Pylint(SimpleFormatter): + """Pylint formatter for Flake8.""" + + error_format = '%(path)s:%(row)d: [%(code)s] %(text)s' diff --git a/src/flake8/main/__init__.py b/src/flake8/main/__init__.py new file mode 100644 index 0000000..d3aa1de --- /dev/null +++ b/src/flake8/main/__init__.py @@ -0,0 +1 @@ +"""Module containing the logic for the Flake8 entry-points.""" diff --git a/src/flake8/main/application.py b/src/flake8/main/application.py new file mode 100644 index 0000000..225c701 --- /dev/null +++ b/src/flake8/main/application.py @@ -0,0 +1,296 @@ +"""Module containing the application logic for Flake8.""" +from __future__ import print_function + +import logging +import sys +import time + +import flake8 +from flake8 import checker +from flake8 import defaults +from flake8 import style_guide +from flake8 import utils +from flake8.main import options +from flake8.options import aggregator +from flake8.options import manager +from flake8.plugins import manager as plugin_manager + +LOG = logging.getLogger(__name__) + + +class Application(object): + """Abstract our application into a class.""" + + def __init__(self, program='flake8', version=flake8.__version__): + # type: (str, str) -> NoneType + """Initialize our application. + + :param str program: + The name of the program/application that we're executing. + :param str version: + The version of the program/application we're executing. + """ + #: The timestamp when the Application instance was instantiated. + self.start_time = time.time() + #: The timestamp when the Application finished reported errors. + self.end_time = None + #: The name of the program being run + self.program = program + #: The version of the program being run + self.version = version + #: The instance of :class:`flake8.options.manager.OptionManager` used + #: to parse and handle the options and arguments passed by the user + self.option_manager = manager.OptionManager( + prog='flake8', version=flake8.__version__ + ) + options.register_default_options(self.option_manager) + + # We haven't found or registered our plugins yet, so let's defer + # printing the version until we aggregate options from config files + # and the command-line. First, let's clone our arguments on the CLI, + # then we'll attempt to remove ``--version`` so that we can avoid + # triggering the "version" action in optparse. If it's not there, we + # do not need to worry and we can continue. If it is, we successfully + # defer printing the version until just a little bit later. + # Similarly we have to defer printing the help text until later. + args = sys.argv[:] + try: + args.remove('--version') + except ValueError: + pass + try: + args.remove('--help') + except ValueError: + pass + try: + args.remove('-h') + except ValueError: + pass + + preliminary_opts, _ = self.option_manager.parse_args(args) + # Set the verbosity of the program + flake8.configure_logging(preliminary_opts.verbose, + preliminary_opts.output_file) + + #: The instance of :class:`flake8.plugins.manager.Checkers` + self.check_plugins = None + #: The instance of :class:`flake8.plugins.manager.Listeners` + self.listening_plugins = None + #: The instance of :class:`flake8.plugins.manager.ReportFormatters` + self.formatting_plugins = None + #: The user-selected formatter from :attr:`formatting_plugins` + self.formatter = None + #: The :class:`flake8.plugins.notifier.Notifier` for listening plugins + self.listener_trie = None + #: The :class:`flake8.style_guide.StyleGuide` built from the user's + #: options + self.guide = None + #: The :class:`flake8.checker.Manager` that will handle running all of + #: the checks selected by the user. + self.file_checker_manager = None + + #: The user-supplied options parsed into an instance of + #: :class:`optparse.Values` + self.options = None + #: The left over arguments that were not parsed by + #: :attr:`option_manager` + self.args = None + #: The number of errors, warnings, and other messages after running + #: flake8 and taking into account ignored errors and lines. + self.result_count = 0 + #: The total number of errors before accounting for ignored errors and + #: lines. + self.total_result_count = 0 + + #: Whether the program is processing a diff or not + self.running_against_diff = False + #: The parsed diff information + self.parsed_diff = {} + + def exit(self): + # type: () -> NoneType + """Handle finalization and exiting the program. + + This should be the last thing called on the application instance. It + will check certain options and exit appropriately. + """ + if self.options.count: + print(self.result_count) + + if not self.options.exit_zero: + raise SystemExit(self.result_count > 0) + + def find_plugins(self): + # type: () -> NoneType + """Find and load the plugins for this application. + + If :attr:`check_plugins`, :attr:`listening_plugins`, or + :attr:`formatting_plugins` are ``None`` then this method will update + them with the appropriate plugin manager instance. Given the expense + of finding plugins (via :mod:`pkg_resources`) we want this to be + idempotent and so only update those attributes if they are ``None``. + """ + if self.check_plugins is None: + self.check_plugins = plugin_manager.Checkers() + + if self.listening_plugins is None: + self.listening_plugins = plugin_manager.Listeners() + + if self.formatting_plugins is None: + self.formatting_plugins = plugin_manager.ReportFormatters() + + self.check_plugins.load_plugins() + self.listening_plugins.load_plugins() + self.formatting_plugins.load_plugins() + + def register_plugin_options(self): + # type: () -> NoneType + """Register options provided by plugins to our option manager.""" + self.check_plugins.register_options(self.option_manager) + self.check_plugins.register_plugin_versions(self.option_manager) + self.listening_plugins.register_options(self.option_manager) + self.formatting_plugins.register_options(self.option_manager) + + def parse_configuration_and_cli(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Parse configuration files and the CLI options. + + :param list argv: + Command-line arguments passed in directly. + """ + if self.options is None and self.args is None: + self.options, self.args = aggregator.aggregate_options( + self.option_manager, argv + ) + + self.running_against_diff = self.options.diff + if self.running_against_diff: + self.parsed_diff = utils.parse_unified_diff() + + self.check_plugins.provide_options(self.option_manager, self.options, + self.args) + self.listening_plugins.provide_options(self.option_manager, + self.options, + self.args) + self.formatting_plugins.provide_options(self.option_manager, + self.options, + self.args) + + def make_formatter(self): + # type: () -> NoneType + """Initialize a formatter based on the parsed options.""" + if self.formatter is None: + self.formatter = self.formatting_plugins.get( + self.options.format, self.formatting_plugins['default'] + ).execute(self.options) + + def make_notifier(self): + # type: () -> NoneType + """Initialize our listener Notifier.""" + if self.listener_trie is None: + self.listener_trie = self.listening_plugins.build_notifier() + + def make_guide(self): + # type: () -> NoneType + """Initialize our StyleGuide.""" + if self.guide is None: + self.guide = style_guide.StyleGuide( + self.options, self.listener_trie, self.formatter + ) + + if self.running_against_diff: + self.guide.add_diff_ranges(self.parsed_diff) + + def make_file_checker_manager(self): + # type: () -> NoneType + """Initialize our FileChecker Manager.""" + if self.file_checker_manager is None: + self.file_checker_manager = checker.Manager( + style_guide=self.guide, + arguments=self.args, + checker_plugins=self.check_plugins, + ) + + def run_checks(self): + # type: () -> NoneType + """Run the actual checks with the FileChecker Manager. + + This method encapsulates the logic to make a + :class:`~flake8.checker.Manger` instance run the checks it is + managing. + """ + files = None + if self.running_against_diff: + files = list(sorted(self.parsed_diff.keys())) + self.file_checker_manager.start(files) + self.file_checker_manager.run() + LOG.info('Finished running') + self.file_checker_manager.stop() + self.end_time = time.time() + + def report_benchmarks(self): + """Aggregate, calculate, and report benchmarks for this run.""" + if not self.options.benchmark: + return + + time_elapsed = self.end_time - self.start_time + statistics = [('seconds elapsed', time_elapsed)] + add_statistic = statistics.append + for statistic in (defaults.STATISTIC_NAMES + ('files',)): + value = self.file_checker_manager.statistics[statistic] + total_description = 'total ' + statistic + ' processed' + add_statistic((total_description, value)) + per_second_description = statistic + ' processed per second' + add_statistic((per_second_description, int(value / time_elapsed))) + + self.formatter.show_benchmarks(statistics) + + def report_errors(self): + # type: () -> NoneType + """Report all the errors found by flake8 3.0. + + This also updates the :attr:`result_count` attribute with the total + number of errors, warnings, and other messages found. + """ + LOG.info('Reporting errors') + results = self.file_checker_manager.report() + self.total_result_count, self.result_count = results + LOG.info('Found a total of %d results and reported %d', + self.total_result_count, self.result_count) + + def initialize(self, argv): + # type: () -> NoneType + """Initialize the application to be run. + + This finds the plugins, registers their options, and parses the + command-line arguments. + """ + self.find_plugins() + self.register_plugin_options() + self.parse_configuration_and_cli(argv) + self.make_formatter() + self.make_notifier() + self.make_guide() + self.make_file_checker_manager() + + def _run(self, argv): + # type: (Union[NoneType, List[str]]) -> NoneType + self.initialize(argv) + self.run_checks() + self.report_errors() + self.report_benchmarks() + + def run(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Run our application. + + This method will also handle KeyboardInterrupt exceptions for the + entirety of the flake8 application. If it sees a KeyboardInterrupt it + will forcibly clean up the :class:`~flake8.checker.Manager`. + """ + try: + self._run(argv) + except KeyboardInterrupt as exc: + LOG.critical('Caught keyboard interrupt from user') + LOG.exception(exc) + self.file_checker_manager._force_cleanup() diff --git a/src/flake8/main/cli.py b/src/flake8/main/cli.py new file mode 100644 index 0000000..29bd159 --- /dev/null +++ b/src/flake8/main/cli.py @@ -0,0 +1,17 @@ +"""Command-line implementation of flake8.""" +from flake8.main import application + + +def main(argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Main entry-point for the flake8 command-line tool. + + This handles the creation of an instance of :class:`Application`, runs it, + and then exits the application. + + :param list argv: + The arguments to be passed to the application for parsing. + """ + app = application.Application() + app.run(argv) + app.exit() diff --git a/src/flake8/main/git.py b/src/flake8/main/git.py new file mode 100644 index 0000000..bae0233 --- /dev/null +++ b/src/flake8/main/git.py @@ -0,0 +1,207 @@ +"""Module containing the main git hook interface and helpers. + +.. autofunction:: hook +.. autofunction:: install + +""" +import contextlib +import os +import shutil +import stat +import subprocess +import sys +import tempfile + +from flake8 import defaults +from flake8 import exceptions + +__all__ = ('hook', 'install') + + +def hook(lazy=False, strict=False): + """Execute Flake8 on the files in git's index. + + Determine which files are about to be committed and run Flake8 over them + to check for violations. + + :param bool lazy: + Find files not added to the index prior to committing. This is useful + if you frequently use ``git commit -a`` for example. This defaults to + False since it will otherwise include files not in the index. + :param bool strict: + If True, return the total number of errors/violations found by Flake8. + This will cause the hook to fail. + :returns: + Total number of errors found during the run. + :rtype: + int + """ + # NOTE(sigmavirus24): Delay import of application until we need it. + from flake8.main import application + app = application.Application() + with make_temporary_directory() as tempdir: + filepaths = list(copy_indexed_files_to(tempdir, lazy)) + app.initialize(filepaths) + app.run_checks() + + app.report_errors() + if strict: + return app.result_count + return 0 + + +def install(): + """Install the git hook script. + + This searches for the ``.git`` directory and will install an executable + pre-commit python script in the hooks sub-directory if one does not + already exist. + + :returns: + True if successful, False if the git directory doesn't exist. + :rtype: + bool + :raises: + flake8.exceptions.GitHookAlreadyExists + """ + git_directory = find_git_directory() + if git_directory is None or not os.path.exists(git_directory): + return False + + hooks_directory = os.path.join(git_directory, 'hooks') + if not os.path.exists(hooks_directory): + os.mkdir(hooks_directory) + + pre_commit_file = os.path.abspath( + os.path.join(hooks_directory, 'pre-commit') + ) + if os.path.exists(pre_commit_file): + raise exceptions.GitHookAlreadyExists( + 'File already exists', + path=pre_commit_file, + ) + + executable = get_executable() + + with open(pre_commit_file, 'w') as fd: + fd.write(_HOOK_TEMPLATE.format(executable=executable)) + + # NOTE(sigmavirus24): The following sets: + # - read, write, and execute permissions for the owner + # - read permissions for people in the group + # - read permissions for other people + # The owner needs the file to be readable, writable, and executable + # so that git can actually execute it as a hook. + pre_commit_permissions = stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH + os.chmod(pre_commit_file, pre_commit_permissions) + return True + + +def get_executable(): + if sys.executable is not None: + return sys.executable + return '/usr/bin/env python' + + +def find_git_directory(): + rev_parse = piped_process(['git', 'rev-parse', '--git-dir']) + + (stdout, _) = rev_parse.communicate() + stdout = to_text(stdout) + + if rev_parse.returncode == 0: + return stdout.strip() + return None + + +def copy_indexed_files_to(temporary_directory, lazy): + modified_files = find_modified_files(lazy) + for filename in modified_files: + contents = get_staged_contents_from(filename) + yield copy_file_to(temporary_directory, filename, contents) + + +def copy_file_to(destination_directory, filepath, contents): + directory, filename = os.path.split(os.path.abspath(filepath)) + temporary_directory = make_temporary_directory_from(destination_directory, + directory) + if not os.path.exists(temporary_directory): + os.makedirs(temporary_directory) + temporary_filepath = os.path.join(temporary_directory, filename) + with open(temporary_filepath, 'wb') as fd: + fd.write(contents) + return temporary_filepath + + +def make_temporary_directory_from(destination, directory): + prefix = os.path.commonprefix([directory, destination]) + common_directory_path = os.path.relpath(directory, start=prefix) + return os.path.join(destination, common_directory_path) + + +def find_modified_files(lazy): + diff_index = piped_process( + ['git', 'diff-index', '--cached', '--name-only', + '--diff-filter=ACMRTUXB', 'HEAD'], + ) + + (stdout, _) = diff_index.communicate() + stdout = to_text(stdout) + return stdout.splitlines() + + +def get_staged_contents_from(filename): + git_show = piped_process(['git', 'show', ':{0}'.format(filename)]) + (stdout, _) = git_show.communicate() + return stdout + + +@contextlib.contextmanager +def make_temporary_directory(): + temporary_directory = tempfile.mkdtemp() + yield temporary_directory + shutil.rmtree(temporary_directory, ignore_errors=True) + + +def to_text(string): + """Ensure that the string is text.""" + if callable(getattr(string, 'decode', None)): + return string.decode('utf-8') + return string + + +def piped_process(command): + return subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def git_config_for(parameter): + config = piped_process(['git', 'config', '--get', '--bool', parameter]) + (stdout, _) = config.communicate() + return to_text(stdout) + + +def config_for(parameter): + environment_variable = 'flake8_{0}'.format(parameter).upper() + git_variable = 'flake8.{0}'.format(parameter) + value = os.environ.get(environment_variable, git_config_for(git_variable)) + return value.lower() in defaults.TRUTHY_VALUES + + +_HOOK_TEMPLATE = """#!{executable} +import os +import sys + +from flake8.main import git + +if __name__ == '__main__': + sys.exit( + git.hook( + strict=git.config_for('strict'), + lazy=git.config_for('lazy'), + ) + ) +""" diff --git a/src/flake8/main/mercurial.py b/src/flake8/main/mercurial.py new file mode 100644 index 0000000..d067612 --- /dev/null +++ b/src/flake8/main/mercurial.py @@ -0,0 +1,128 @@ +"""Module containing the main mecurial hook interface and helpers. + +.. autofunction:: hook +.. autofunction:: install + +""" +import configparser +import os +import subprocess + +from flake8 import exceptions as exc + +__all__ = ('hook', 'install') + + +def hook(ui, repo, **kwargs): + """Execute Flake8 on the repository provided by Mercurial. + + To understand the parameters read more of the Mercurial documentation + around Hooks: https://www.mercurial-scm.org/wiki/Hook. + + We avoid using the ``ui`` attribute because it can cause issues with + the GPL license tha Mercurial is under. We don't import it, but we + avoid using it all the same. + """ + from flake8.main import application + hgrc = find_hgrc(create_if_missing=False) + if hgrc is None: + print('Cannot locate your root mercurial repository.') + raise SystemExit(True) + + hgconfig = configparser_for(hgrc) + strict = hgconfig.get('flake8', 'strict', fallback=True) + + filenames = list(get_filenames_from(repo, kwargs)) + + app = application.Application() + app.run(filenames) + + if strict: + return app.result_count + return 0 + + +def install(): + """Ensure that the mercurial hooks are installed.""" + hgrc = find_hgrc(create_if_missing=True) + if hgrc is None: + return False + + hgconfig = configparser_for(hgrc) + + if not hgconfig.has_section('hooks'): + hgconfig.add_section('hooks') + + if hgconfig.has_option('hooks', 'commit'): + raise exc.MercurialCommitHookAlreadyExists( + path=hgrc, + value=hgconfig.get('hooks', 'commit'), + ) + + if hgconfig.has_option('hooks', 'qrefresh'): + raise exc.MercurialQRefreshHookAlreadyExists( + path=hgrc, + value=hgconfig.get('hooks', 'qrefresh'), + ) + + hgconfig.set('hooks', 'commit', 'python:flake8.main.mercurial.hook') + hgconfig.set('hooks', 'qrefresh', 'python:flake8.main.mercurial.hook') + + if not hgconfig.has_section('flake8'): + hgconfig.add_section('flake8') + + if not hgconfig.has_option('flake8', 'strict'): + hgconfig.set('flake8', 'strict', False) + + with open(hgrc, 'w') as fd: + hgconfig.write(fd) + + return True + + +def get_filenames_from(repository, kwargs): + seen_filenames = set() + node = kwargs['node'] + for revision in range(repository[node], len(repository)): + for filename in repository[revision].files(): + full_filename = os.path.join(repository.root, filename) + have_seen_filename = full_filename in seen_filenames + filename_does_not_exist = not os.path.exists(full_filename) + if have_seen_filename or filename_does_not_exist: + continue + + seen_filenames.add(full_filename) + if full_filename.endswith('.py'): + yield full_filename + + +def find_hgrc(create_if_missing=False): + root = subprocess.Popen( + ['hg', 'root'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + (hg_directory, _) = root.communicate() + if callable(getattr(hg_directory, 'decode', None)): + hg_directory = hg_directory.decode('utf-8') + + if not os.path.isdir(hg_directory): + return None + + hgrc = os.path.abspath( + os.path.join(hg_directory, '.hg', 'hgrc') + ) + if not os.path.exists(hgrc): + if create_if_missing: + open(hgrc, 'w').close() + else: + return None + + return hgrc + + +def configparser_for(path): + parser = configparser.ConfigParser(interpolation=None) + parser.read(path) + return parser diff --git a/src/flake8/main/options.py b/src/flake8/main/options.py new file mode 100644 index 0000000..c725c38 --- /dev/null +++ b/src/flake8/main/options.py @@ -0,0 +1,201 @@ +"""Contains the logic for all of the default options for Flake8.""" +from flake8 import defaults +from flake8.main import vcs + + +def register_default_options(option_manager): + """Register the default options on our OptionManager. + + The default options include: + + - ``-v``/``--verbose`` + - ``-q``/``--quiet`` + - ``--count`` + - ``--diff`` + - ``--exclude`` + - ``--filename`` + - ``--format`` + - ``--hang-closing`` + - ``--ignore`` + - ``--max-line-length`` + - ``--select`` + - ``--disable-noqa`` + - ``--show-source`` + - ``--statistics`` + - ``--enable-extensions`` + - ``--exit-zero`` + - ``-j``/``--jobs`` + - ``--output-file`` + - ``--append-config`` + - ``--config`` + - ``--isolated`` + """ + add_option = option_manager.add_option + + # pep8 options + add_option( + '-v', '--verbose', default=0, action='count', + parse_from_config=True, + help='Print more information about what is happening in flake8.' + ' This option is repeatable and will increase verbosity each ' + 'time it is repeated.', + ) + add_option( + '-q', '--quiet', default=0, action='count', + parse_from_config=True, + help='Report only file names, or nothing. This option is repeatable.', + ) + + add_option( + '--count', action='store_true', parse_from_config=True, + help='Print total number of errors and warnings to standard error and' + ' set the exit code to 1 if total is not empty.', + ) + + add_option( + '--diff', action='store_true', + help='Report changes only within line number ranges in the unified ' + 'diff provided on standard in by the user.', + ) + + add_option( + '--exclude', metavar='patterns', default=defaults.EXCLUDE, + comma_separated_list=True, parse_from_config=True, + normalize_paths=True, + help='Comma-separated list of files or directories to exclude.' + ' (Default: %default)', + ) + + add_option( + '--filename', metavar='patterns', default='*.py', + parse_from_config=True, comma_separated_list=True, + help='Only check for filenames matching the patterns in this comma-' + 'separated list. (Default: %default)', + ) + + add_option( + '--stdin-display-name', default='stdin', + help='The name used when reporting errors from code passed via stdin.' + ' This is useful for editors piping the file contents to flake8.' + ' (Default: %default)', + ) + + # TODO(sigmavirus24): Figure out --first/--repeat + + # NOTE(sigmavirus24): We can't use choices for this option since users can + # freely provide a format string and that will break if we restrict their + # choices. + add_option( + '--format', metavar='format', default='default', + parse_from_config=True, + help='Format errors according to the chosen formatter.', + ) + + add_option( + '--hang-closing', action='store_true', parse_from_config=True, + help='Hang closing bracket instead of matching indentation of opening' + " bracket's line.", + ) + + add_option( + '--ignore', metavar='errors', default=defaults.IGNORE, + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to ignore (or skip).' + ' For example, ``--ignore=E4,E51,W234``. (Default: %default)', + ) + + add_option( + '--max-line-length', type='int', metavar='n', + default=defaults.MAX_LINE_LENGTH, parse_from_config=True, + help='Maximum allowed line length for the entirety of this run. ' + '(Default: %default)', + ) + + add_option( + '--select', metavar='errors', default=defaults.SELECT, + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to enable.' + ' For example, ``--select=E4,E51,W234``. (Default: %default)', + ) + + add_option( + '--disable-noqa', default=False, parse_from_config=True, + action='store_true', + help='Disable the effect of "# noqa". This will report errors on ' + 'lines with "# noqa" at the end.' + ) + + # TODO(sigmavirus24): Decide what to do about --show-pep8 + + add_option( + '--show-source', action='store_true', parse_from_config=True, + help='Show the source generate each error or warning.', + ) + + add_option( + '--statistics', action='store_true', parse_from_config=True, + help='Count errors and warnings.', + ) + + # Flake8 options + add_option( + '--enable-extensions', default='', parse_from_config=True, + comma_separated_list=True, type='string', + help='Enable plugins and extensions that are otherwise disabled ' + 'by default', + ) + + add_option( + '--exit-zero', action='store_true', + help='Exit with status code "0" even if there are errors.', + ) + + add_option( + '--install-hook', action='callback', type='choice', + choices=vcs.choices(), callback=vcs.install, + help='Install a hook that is run prior to a commit for the supported ' + 'version control systema.' + ) + + add_option( + '-j', '--jobs', type='string', default='auto', parse_from_config=True, + help='Number of subprocesses to use to run checks in parallel. ' + 'This is ignored on Windows. The default, "auto", will ' + 'auto-detect the number of processors available to use.' + ' (Default: %default)', + ) + + add_option( + '--output-file', default=None, type='string', parse_from_config=True, + # callback=callbacks.redirect_stdout, + help='Redirect report to a file.', + ) + + # Config file options + + add_option( + '--append-config', action='append', + help='Provide extra config files to parse in addition to the files ' + 'found by Flake8 by default. These files are the last ones read ' + 'and so they take the highest precedence when multiple files ' + 'provide the same option.', + ) + + add_option( + '--config', default=None, + help='Path to the config file that will be the authoritative config ' + 'source. This will cause Flake8 to ignore all other ' + 'configuration files.' + ) + + add_option( + '--isolated', default=False, action='store_true', + help='Ignore all found configuration files.', + ) + + # Benchmarking + + add_option( + '--benchmark', default=False, action='store_true', + help='Print benchmark information about this run of Flake8', + ) diff --git a/src/flake8/main/setuptools_command.py b/src/flake8/main/setuptools_command.py new file mode 100644 index 0000000..1c27bf6 --- /dev/null +++ b/src/flake8/main/setuptools_command.py @@ -0,0 +1,77 @@ +"""The logic for Flake8's integration with setuptools.""" +import os + +import setuptools + +from flake8.main import application as app + + +class Flake8(setuptools.Command): + """Run Flake8 via setuptools/distutils for registered modules.""" + + description = 'Run Flake8 on modules registered in setup.py' + # NOTE(sigmavirus24): If we populated this with a list of tuples, users + # could do something like ``python setup.py flake8 --ignore=E123,E234`` + # but we would have to redefine it and we can't define it dynamically. + # Since I refuse to copy-and-paste the options here or maintain two lists + # of options, and since this will break when users use plugins that + # provide command-line options, we are leaving this empty. If users want + # to configure this command, they can do so through config files. + user_options = [] + + def initialize_options(self): + """Override this method to initialize our application.""" + pass + + def finalize_options(self): + """Override this to parse the parameters.""" + pass + + def package_files(self): + """Collect the files/dirs included in the registered modules.""" + seen_package_directories = () + directories = self.distribution.package_dir or {} + empty_directory_exists = '' in directories + packages = self.distribution.packages or [] + for package in packages: + package_directory = package + if package in directories: + package_directory = directories[package] + elif empty_directory_exists: + package_directory = os.path.join(directories[''], + package_directory) + + # NOTE(sigmavirus24): Do not collect submodules, e.g., + # if we have: + # - flake8/ + # - flake8/plugins/ + # Flake8 only needs ``flake8/`` to be provided. It will + # recurse on its own. + if package_directory.startswith(seen_package_directories): + continue + + seen_package_directories += (package_directory,) + yield package_directory + + def module_files(self): + """Collect the files listed as py_modules.""" + modules = self.distribution.py_modules or [] + filename_from = '{0}.py'.format + for module in modules: + yield filename_from(module) + + def distribution_files(self): + """Collect package and module files.""" + for package in self.package_files(): + yield package + + for module in self.module_files(): + yield module + + yield 'setup.py' + + def run(self): + """Run the Flake8 application.""" + flake8 = app.Application() + flake8.run(list(self.distribution_files())) + flake8.exit() diff --git a/src/flake8/main/vcs.py b/src/flake8/main/vcs.py new file mode 100644 index 0000000..6f7499e --- /dev/null +++ b/src/flake8/main/vcs.py @@ -0,0 +1,39 @@ +"""Module containing some of the logic for our VCS installation logic.""" +from flake8 import exceptions as exc +from flake8.main import git +from flake8.main import mercurial + + +# NOTE(sigmavirus24): In the future, we may allow for VCS hooks to be defined +# as plugins, e.g., adding a flake8.vcs entry-point. In that case, this +# dictionary should disappear, and this module might contain more code for +# managing those bits (in conjuntion with flake8.plugins.manager). +_INSTALLERS = { + 'git': git.install, + 'mercurial': mercurial.install, +} + + +def install(option, option_string, value, parser): + """Determine which version control hook to install. + + For more information about the callback signature, see: + https://docs.python.org/2/library/optparse.html#optparse-option-callbacks + """ + installer = _INSTALLERS.get(value) + errored = False + successful = False + try: + successful = installer() + except exc.HookInstallationError as hook_error: + print(str(hook_error)) + errored = True + + if not successful: + print('Could not find the {0} directory'.format(value)) + raise SystemExit(not successful and errored) + + +def choices(): + """Return the list of VCS choices.""" + return list(_INSTALLERS.keys()) diff --git a/src/flake8/options/__init__.py b/src/flake8/options/__init__.py new file mode 100644 index 0000000..cc20daa --- /dev/null +++ b/src/flake8/options/__init__.py @@ -0,0 +1,12 @@ +"""Package containing the option manager and config management logic. + +- :mod:`flake8.options.config` contains the logic for finding, parsing, and + merging configuration files. + +- :mod:`flake8.options.manager` contains the logic for managing customized + Flake8 command-line and configuration options. + +- :mod:`flake8.options.aggregator` uses objects from both of the above modules + to aggregate configuration into one object used by plugins and Flake8. + +""" diff --git a/src/flake8/options/aggregator.py b/src/flake8/options/aggregator.py new file mode 100644 index 0000000..99d0cfe --- /dev/null +++ b/src/flake8/options/aggregator.py @@ -0,0 +1,74 @@ +"""Aggregation function for CLI specified options and config file options. + +This holds the logic that uses the collected and merged config files and +applies the user-specified command-line configuration on top of it. +""" +import logging + +from flake8 import utils +from flake8.options import config + +LOG = logging.getLogger(__name__) + + +def aggregate_options(manager, arglist=None, values=None): + """Aggregate and merge CLI and config file options. + + :param flake8.option.manager.OptionManager manager: + The instance of the OptionManager that we're presently using. + :param list arglist: + The list of arguments to pass to ``manager.parse_args``. In most cases + this will be None so ``parse_args`` uses ``sys.argv``. This is mostly + available to make testing easier. + :param optparse.Values values: + Previously parsed set of parsed options. + :returns: + Tuple of the parsed options and extra arguments returned by + ``manager.parse_args``. + :rtype: + tuple(optparse.Values, list) + """ + # Get defaults from the option parser + default_values, _ = manager.parse_args([], values=values) + # Get original CLI values so we can find additional config file paths and + # see if --config was specified. + original_values, original_args = manager.parse_args(arglist) + extra_config_files = utils.normalize_paths(original_values.append_config) + + # Make our new configuration file mergerator + config_parser = config.MergedConfigParser( + option_manager=manager, + extra_config_files=extra_config_files, + args=original_args, + ) + + # Get the parsed config + parsed_config = config_parser.parse(original_values.config, + original_values.isolated) + + # Extend the default ignore value with the extended default ignore list, + # registered by plugins. + extended_default_ignore = manager.extended_default_ignore.copy() + LOG.debug('Extended default ignore list: %s', + list(extended_default_ignore)) + extended_default_ignore.update(default_values.ignore) + default_values.ignore = list(extended_default_ignore) + LOG.debug('Merged default ignore list: %s', default_values.ignore) + + # Merge values parsed from config onto the default values returned + for config_name, value in parsed_config.items(): + dest_name = config_name + # If the config name is somehow different from the destination name, + # fetch the destination name from our Option + if not hasattr(default_values, config_name): + dest_name = config_parser.config_options[config_name].dest + + LOG.debug('Overriding default value of (%s) for "%s" with (%s)', + getattr(default_values, dest_name, None), + dest_name, + value) + # Override the default values with the config values + setattr(default_values, dest_name, value) + + # Finally parse the command-line options + return manager.parse_args(arglist, default_values) diff --git a/src/flake8/options/config.py b/src/flake8/options/config.py new file mode 100644 index 0000000..48719a8 --- /dev/null +++ b/src/flake8/options/config.py @@ -0,0 +1,279 @@ +"""Config handling logic for Flake8.""" +import configparser +import logging +import os.path +import sys + +LOG = logging.getLogger(__name__) + +__all__ = ('ConfigFileFinder', 'MergedConfigParser') + + +class ConfigFileFinder(object): + """Encapsulate the logic for finding and reading config files.""" + + PROJECT_FILENAMES = ('setup.cfg', 'tox.ini') + + def __init__(self, program_name, args, extra_config_files): + """Initialize object to find config files. + + :param str program_name: + Name of the current program (e.g., flake8). + :param list args: + The extra arguments passed on the command-line. + :param list extra_config_files: + Extra configuration files specified by the user to read. + """ + # The values of --append-config from the CLI + extra_config_files = extra_config_files or [] + self.extra_config_files = [ + # Ensure the paths are absolute paths for local_config_files + os.path.abspath(f) for f in extra_config_files + ] + + # Platform specific settings + self.is_windows = sys.platform == 'win32' + self.xdg_home = os.environ.get('XDG_CONFIG_HOME', + os.path.expanduser('~/.config')) + + # Look for '.' files + self.program_config = '.' + program_name + self.program_name = program_name + + # List of filenames to find in the local/project directory + self.project_filenames = ('setup.cfg', 'tox.ini', self.program_config) + + self.local_directory = os.path.abspath(os.curdir) + + if not args: + args = ['.'] + self.parent = self.tail = os.path.abspath(os.path.commonprefix(args)) + + @staticmethod + def _read_config(files): + config = configparser.RawConfigParser() + try: + found_files = config.read(files) + except configparser.ParsingError: + LOG.exception("There was an error trying to parse a config " + "file. The files we were attempting to parse " + "were: %r", files) + found_files = [] + return (config, found_files) + + def cli_config(self, files): + """Read and parse the config file specified on the command-line.""" + config, found_files = self._read_config(files) + if found_files: + LOG.debug('Found cli configuration files: %s', found_files) + return config + + def generate_possible_local_files(self): + """Find and generate all local config files.""" + tail = self.tail + parent = self.parent + local_dir = self.local_directory + while tail: + for project_filename in self.project_filenames: + filename = os.path.abspath(os.path.join(parent, + project_filename)) + yield filename + if parent == local_dir: + break + (parent, tail) = os.path.split(parent) + + def local_config_files(self): + """Find all local config files which actually exist. + + Filter results from + :meth:`~ConfigFileFinder.generate_possible_local_files` based + on whether the filename exists or not. + + :returns: + List of files that exist that are local project config files with + extra config files appended to that list (which also exist). + :rtype: + [str] + """ + exists = os.path.exists + return [ + filename + for filename in self.generate_possible_local_files() + if os.path.exists(filename) + ] + [f for f in self.extra_config_files if exists(f)] + + def local_configs(self): + """Parse all local config files into one config object.""" + config, found_files = self._read_config(self.local_config_files()) + if found_files: + LOG.debug('Found local configuration files: %s', found_files) + return config + + def user_config_file(self): + """Find the user-level config file.""" + if self.is_windows: + return os.path.expanduser('~\\' + self.program_config) + return os.path.join(self.xdg_home, self.program_name) + + def user_config(self): + """Parse the user config file into a config object.""" + config, found_files = self._read_config(self.user_config_file()) + if found_files: + LOG.debug('Found user configuration files: %s', found_files) + return config + + +class MergedConfigParser(object): + """Encapsulate merging different types of configuration files. + + This parses out the options registered that were specified in the + configuration files, handles extra configuration files, and returns + dictionaries with the parsed values. + """ + + #: Set of types that should use the + #: :meth:`~configparser.RawConfigParser.getint` method. + GETINT_TYPES = set(['int', 'count']) + #: Set of actions that should use the + #: :meth:`~configparser.RawConfigParser.getbool` method. + GETBOOL_ACTIONS = set(['store_true', 'store_false']) + + def __init__(self, option_manager, extra_config_files=None, args=None): + """Initialize the MergedConfigParser instance. + + :param flake8.option.manager.OptionManager option_manager: + Initialized OptionManager. + :param list extra_config_files: + List of extra config files to parse. + :params list args: + The extra parsed arguments from the command-line. + """ + #: Our instance of flake8.options.manager.OptionManager + self.option_manager = option_manager + #: The prog value for the cli parser + self.program_name = option_manager.program_name + #: Parsed extra arguments + self.args = args + #: Mapping of configuration option names to + #: :class:`~flake8.options.manager.Option` instances + self.config_options = option_manager.config_options_dict + #: List of extra config files + self.extra_config_files = extra_config_files or [] + #: Our instance of our :class:`~ConfigFileFinder` + self.config_finder = ConfigFileFinder(self.program_name, self.args, + self.extra_config_files) + + @staticmethod + def _normalize_value(option, value): + final_value = option.normalize(value) + LOG.debug('%r has been normalized to %r for option "%s"', + value, final_value, option.config_name) + return final_value + + def _parse_config(self, config_parser): + config_dict = {} + for option_name in config_parser.options(self.program_name): + if option_name not in self.config_options: + LOG.debug('Option "%s" is not registered. Ignoring.', + option_name) + continue + option = self.config_options[option_name] + + # Use the appropriate method to parse the config value + method = config_parser.get + if option.type in self.GETINT_TYPES: + method = config_parser.getint + elif option.action in self.GETBOOL_ACTIONS: + method = config_parser.getboolean + + value = method(self.program_name, option_name) + LOG.debug('Option "%s" returned value: %r', option_name, value) + + final_value = self._normalize_value(option, value) + config_dict[option_name] = final_value + + return config_dict + + def is_configured_by(self, config): + """Check if the specified config parser has an appropriate section.""" + return config.has_section(self.program_name) + + def parse_local_config(self): + """Parse and return the local configuration files.""" + config = self.config_finder.local_configs() + if not self.is_configured_by(config): + LOG.debug('Local configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing local configuration files.') + return self._parse_config(config) + + def parse_user_config(self): + """Parse and return the user configuration files.""" + config = self.config_finder.user_config() + if not self.is_configured_by(config): + LOG.debug('User configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing user configuration files.') + return self._parse_config(config) + + def parse_cli_config(self, config_path): + """Parse and return the file specified by --config.""" + config = self.config_finder.cli_config(config_path) + if not self.is_configured_by(config): + LOG.debug('CLI configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing CLI configuration files.') + return self._parse_config(config) + + def merge_user_and_local_config(self): + """Merge the parsed user and local configuration files. + + :returns: + Dictionary of the parsed and merged configuration options. + :rtype: + dict + """ + user_config = self.parse_user_config() + config = self.parse_local_config() + + for option, value in user_config.items(): + config.setdefault(option, value) + + return config + + def parse(self, cli_config=None, isolated=False): + """Parse and return the local and user config files. + + First this copies over the parsed local configuration and then + iterates over the options in the user configuration and sets them if + they were not set by the local configuration file. + + :param str cli_config: + Value of --config when specified at the command-line. Overrides + all other config files. + :param bool isolated: + Determines if we should parse configuration files at all or not. + If running in isolated mode, we ignore all configuration files + :returns: + Dictionary of parsed configuration options + :rtype: + dict + """ + if isolated: + LOG.debug('Refusing to parse configuration files due to user-' + 'requested isolation') + return {} + + if cli_config: + LOG.debug('Ignoring user and locally found configuration files. ' + 'Reading only configuration from "%s" specified via ' + '--config by the user', cli_config) + return self.parse_cli_config(cli_config) + + return self.merge_user_and_local_config() diff --git a/src/flake8/options/manager.py b/src/flake8/options/manager.py new file mode 100644 index 0000000..439cba2 --- /dev/null +++ b/src/flake8/options/manager.py @@ -0,0 +1,256 @@ +"""Option handling and Option management logic.""" +import logging +import optparse # pylint: disable=deprecated-module + +from flake8 import utils + +LOG = logging.getLogger(__name__) + + +class Option(object): + """Our wrapper around an optparse.Option object to add features.""" + + def __init__(self, short_option_name=None, long_option_name=None, + # Options below here are taken from the optparse.Option class + action=None, default=None, type=None, dest=None, + nargs=None, const=None, choices=None, callback=None, + callback_args=None, callback_kwargs=None, help=None, + metavar=None, + # Options below here are specific to Flake8 + parse_from_config=False, comma_separated_list=False, + normalize_paths=False): + """Initialize an Option instance wrapping optparse.Option. + + The following are all passed directly through to optparse. + + :param str short_option_name: + The short name of the option (e.g., ``-x``). This will be the + first argument passed to :class:`~optparse.Option`. + :param str long_option_name: + The long name of the option (e.g., ``--xtra-long-option``). This + will be the second argument passed to :class:`~optparse.Option`. + :param str action: + Any action allowed by :mod:`optparse`. + :param default: + Default value of the option. + :param type: + Any type allowed by :mod:`optparse`. + :param dest: + Attribute name to store parsed option value as. + :param nargs: + Number of arguments to parse for this option. + :param const: + Constant value to store on a common destination. Usually used in + conjuntion with ``action="store_const"``. + :param iterable choices: + Possible values for the option. + :param callable callback: + Callback used if the action is ``"callback"``. + :param iterable callback_args: + Additional positional arguments to the callback callable. + :param dictionary callback_kwargs: + Keyword arguments to the callback callable. + :param str help: + Help text displayed in the usage information. + :param str metavar: + Name to use instead of the long option name for help text. + + The following parameters are for Flake8's option handling alone. + + :param bool parse_from_config: + Whether or not this option should be parsed out of config files. + :param bool comma_separated_list: + Whether the option is a comma separated list when parsing from a + config file. + :param bool normalize_paths: + Whether the option is expecting a path or list of paths and should + attempt to normalize the paths to absolute paths. + """ + self.short_option_name = short_option_name + self.long_option_name = long_option_name + self.option_args = [ + x for x in (short_option_name, long_option_name) if x is not None + ] + self.option_kwargs = { + 'action': action, + 'default': default, + 'type': type, + 'dest': self._make_dest(dest), + 'nargs': nargs, + 'const': const, + 'choices': choices, + 'callback': callback, + 'callback_args': callback_args, + 'callback_kwargs': callback_kwargs, + 'help': help, + 'metavar': metavar, + } + # Set attributes for our option arguments + for key, value in self.option_kwargs.items(): + setattr(self, key, value) + + # Set our custom attributes + self.parse_from_config = parse_from_config + self.comma_separated_list = comma_separated_list + self.normalize_paths = normalize_paths + + self.config_name = None + if parse_from_config: + if not long_option_name: + raise ValueError('When specifying parse_from_config=True, ' + 'a long_option_name must also be specified.') + self.config_name = long_option_name[2:].replace('-', '_') + + self._opt = None + + def __repr__(self): + """Simple representation of an Option class.""" + return ( + 'Option({0}, {1}, action={action}, default={default}, ' + 'dest={dest}, type={type}, callback={callback}, help={help},' + ' callback={callback}, callback_args={callback_args}, ' + 'callback_kwargs={callback_kwargs}, metavar={metavar})' + ).format(self.short_option_name, self.long_option_name, + **self.option_kwargs) + + def _make_dest(self, dest): + if dest: + return dest + + if self.long_option_name: + return self.long_option_name[2:].replace('-', '_') + return self.short_option_name[1] + + def normalize(self, value): + """Normalize the value based on the option configuration.""" + if self.normalize_paths: + # Decide whether to parse a list of paths or a single path + normalize = utils.normalize_path + if self.comma_separated_list: + normalize = utils.normalize_paths + return normalize(value) + elif self.comma_separated_list: + return utils.parse_comma_separated_list(value) + return value + + def to_optparse(self): + """Convert a Flake8 Option to an optparse Option.""" + if self._opt is None: + self._opt = optparse.Option(*self.option_args, + **self.option_kwargs) + return self._opt + + +class OptionManager(object): + """Manage Options and OptionParser while adding post-processing.""" + + def __init__(self, prog=None, version=None, + usage='%prog [options] file file ...'): + """Initialize an instance of an OptionManager. + + :param str prog: + Name of the actual program (e.g., flake8). + :param str version: + Version string for the program. + :param str usage: + Basic usage string used by the OptionParser. + """ + self.parser = optparse.OptionParser(prog=prog, version=version, + usage=usage) + self.config_options_dict = {} + self.options = [] + self.program_name = prog + self.version = version + self.registered_plugins = set() + self.extended_default_ignore = set() + + @staticmethod + def format_plugin(plugin_tuple): + """Convert a plugin tuple into a dictionary mapping name to value.""" + return dict(zip(["name", "version"], plugin_tuple)) + + def add_option(self, *args, **kwargs): + """Create and register a new option. + + See parameters for :class:`~flake8.options.manager.Option` for + acceptable arguments to this method. + + .. note:: + + ``short_option_name`` and ``long_option_name`` may be specified + positionally as they are with optparse normally. + """ + if len(args) == 1 and args[0].startswith('--'): + args = (None, args[0]) + option = Option(*args, **kwargs) + self.parser.add_option(option.to_optparse()) + self.options.append(option) + if option.parse_from_config: + self.config_options_dict[option.config_name] = option + LOG.debug('Registered option "%s".', option) + + def remove_from_default_ignore(self, error_codes): + """Remove specified error codes from the default ignore list. + + :param list error_codes: + List of strings that are the error/warning codes to attempt to + remove from the extended default ignore list. + """ + LOG.debug('Removing %r from the default ignore list', error_codes) + for error_code in error_codes: + try: + self.extend_default_ignore.remove(error_code) + except ValueError: + LOG.debug('Attempted to remove %s from default ignore' + ' but it was not a member of the list.', error_code) + + def extend_default_ignore(self, error_codes): + """Extend the default ignore list with the error codes provided. + + :param list error_codes: + List of strings that are the error/warning codes with which to + extend the default ignore list. + """ + LOG.debug('Extending default ignore list with %r', error_codes) + self.extended_default_ignore.update(error_codes) + + def generate_versions(self, format_str='%(name)s: %(version)s'): + """Generate a comma-separated list of versions of plugins.""" + return ', '.join( + format_str % self.format_plugin(plugin) + for plugin in self.registered_plugins + ) + + def update_version_string(self): + """Update the flake8 version string.""" + self.parser.version = (self.version + ' (' + + self.generate_versions() + ')') + + def generate_epilog(self): + """Create an epilog with the version and name of each of plugin.""" + plugin_version_format = '%(name)s: %(version)s' + self.parser.epilog = 'Installed plugins: ' + self.generate_versions( + plugin_version_format + ) + + def parse_args(self, args=None, values=None): + """Simple proxy to calling the OptionParser's parse_args method.""" + self.generate_epilog() + self.update_version_string() + options, xargs = self.parser.parse_args(args, values) + for option in self.options: + old_value = getattr(options, option.dest) + setattr(options, option.dest, option.normalize(old_value)) + + return options, xargs + + def register_plugin(self, name, version): + """Register a plugin relying on the OptionManager. + + :param str name: + The name of the checker itself. This will be the ``name`` + attribute of the class or function loaded from the entry-point. + :param str version: + The version of the checker that we're using. + """ + self.registered_plugins.add((name, version)) diff --git a/src/flake8/plugins/__init__.py b/src/flake8/plugins/__init__.py new file mode 100644 index 0000000..fda6a44 --- /dev/null +++ b/src/flake8/plugins/__init__.py @@ -0,0 +1 @@ +"""Submodule of built-in plugins and plugin managers.""" diff --git a/src/flake8/plugins/_trie.py b/src/flake8/plugins/_trie.py new file mode 100644 index 0000000..4871abb --- /dev/null +++ b/src/flake8/plugins/_trie.py @@ -0,0 +1,97 @@ +"""Independent implementation of a Trie tree.""" + +__all__ = ('Trie', 'TrieNode') + + +def _iterate_stringlike_objects(string): + for i in range(len(string)): + yield string[i:i + 1] + + +class Trie(object): + """The object that manages the trie nodes.""" + + def __init__(self): + """Initialize an empty trie.""" + self.root = TrieNode(None, None) + + def add(self, path, node_data): + """Add the node data to the path described.""" + node = self.root + for prefix in _iterate_stringlike_objects(path): + child = node.find_prefix(prefix) + if child is None: + child = node.add_child(prefix, []) + node = child + node.data.append(node_data) + + def find(self, path): + """Find a node based on the path provided.""" + node = self.root + for prefix in _iterate_stringlike_objects(path): + child = node.find_prefix(prefix) + if child is None: + return None + node = child + return node + + def traverse(self): + """Traverse this tree. + + This performs a depth-first pre-order traversal of children in this + tree. It returns the results consistently by first sorting the + children based on their prefix and then traversing them in + alphabetical order. + """ + return self.root.traverse() + + +class TrieNode(object): + """The majority of the implementation details of a Trie.""" + + def __init__(self, prefix, data, children=None): + """Initialize a TrieNode with data and children.""" + self.children = children or {} + self.data = data + self.prefix = prefix + + def __repr__(self): + """Generate an easy to read representation of the node.""" + return 'TrieNode(prefix={0}, data={1})'.format( + self.prefix, self.data + ) + + def find_prefix(self, prefix): + """Find the prefix in the children of this node. + + :returns: A child matching the prefix or None. + :rtype: :class:`~TrieNode` or None + """ + return self.children.get(prefix, None) + + def add_child(self, prefix, data, children=None): + """Create and add a new child node. + + :returns: The newly created node + :rtype: :class:`~TrieNode` + """ + new_node = TrieNode(prefix, data, children) + self.children[prefix] = new_node + return new_node + + def traverse(self): + """Traverse children of this node. + + This performs a depth-first pre-order traversal of the remaining + children in this sub-tree. It returns the results consistently by + first sorting the children based on their prefix and then traversing + them in alphabetical order. + """ + if not self.children: + return + + for prefix in sorted(self.children.keys()): + child = self.children[prefix] + yield child + for child in child.traverse(): + yield child diff --git a/src/flake8/plugins/manager.py b/src/flake8/plugins/manager.py new file mode 100644 index 0000000..d08d542 --- /dev/null +++ b/src/flake8/plugins/manager.py @@ -0,0 +1,458 @@ +"""Plugin loading and management logic and classes.""" +import collections +import logging + +import pkg_resources + +from flake8 import exceptions +from flake8 import utils +from flake8.plugins import notifier + +LOG = logging.getLogger(__name__) + +__all__ = ( + 'Checkers', + 'Listeners', + 'Plugin', + 'PluginManager', + 'ReportFormatters', +) + +NO_GROUP_FOUND = object() + + +class Plugin(object): + """Wrap an EntryPoint from setuptools and other logic.""" + + def __init__(self, name, entry_point): + """"Initialize our Plugin. + + :param str name: + Name of the entry-point as it was registered with setuptools. + :param entry_point: + EntryPoint returned by setuptools. + :type entry_point: + setuptools.EntryPoint + """ + self.name = name + self.entry_point = entry_point + self._plugin = None + self._parameters = None + self._group = None + self._plugin_name = None + self._version = None + + def __repr__(self): + """Provide an easy to read description of the current plugin.""" + return 'Plugin(name="{0}", entry_point="{1}")'.format( + self.name, self.entry_point + ) + + def is_in_a_group(self): + """Determine if this plugin is in a group. + + :returns: + True if the plugin is in a group, otherwise False. + :rtype: + bool + """ + return self.group() is not None + + def group(self): + """Find and parse the group the plugin is in.""" + if self._group is None: + name = self.name.split('.', 1) + if len(name) > 1: + self._group = name[0] + else: + self._group = NO_GROUP_FOUND + if self._group is NO_GROUP_FOUND: + return None + return self._group + + @property + def parameters(self): + """List of arguments that need to be passed to the plugin.""" + if self._parameters is None: + self._parameters = utils.parameters_for(self) + return self._parameters + + @property + def plugin(self): + """The loaded (and cached) plugin associated with the entry-point. + + This property implicitly loads the plugin and then caches it. + """ + self.load_plugin() + return self._plugin + + @property + def version(self): + """Return the version of the plugin.""" + if self._version is None: + if self.is_in_a_group(): + self._version = version_for(self) + else: + self._version = self.plugin.version + + return self._version + + @property + def plugin_name(self): + """Return the name of the plugin.""" + if self._plugin_name is None: + if self.is_in_a_group(): + self._plugin_name = self.group() + else: + self._plugin_name = self.plugin.name + + return self._plugin_name + + @property + def off_by_default(self): + """Return whether the plugin is ignored by default.""" + return getattr(self.plugin, 'off_by_default', False) + + def execute(self, *args, **kwargs): + r"""Call the plugin with \*args and \*\*kwargs.""" + return self.plugin(*args, **kwargs) # pylint: disable=not-callable + + def _load(self, verify_requirements): + # Avoid relying on hasattr() here. + resolve = getattr(self.entry_point, 'resolve', None) + require = getattr(self.entry_point, 'require', None) + if resolve and require: + if verify_requirements: + LOG.debug('Verifying plugin "%s"\'s requirements.', + self.name) + require() + self._plugin = resolve() + else: + self._plugin = self.entry_point.load( + require=verify_requirements + ) + + def load_plugin(self, verify_requirements=False): + """Retrieve the plugin for this entry-point. + + This loads the plugin, stores it on the instance and then returns it. + It does not reload it after the first time, it merely returns the + cached plugin. + + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + :returns: + Nothing + """ + if self._plugin is None: + LOG.info('Loading plugin "%s" from entry-point.', self.name) + try: + self._load(verify_requirements) + except Exception as load_exception: + LOG.exception(load_exception, exc_info=True) + failed_to_load = exceptions.FailedToLoadPlugin( + plugin=self, + exception=load_exception, + ) + LOG.critical(str(failed_to_load)) + raise failed_to_load + + def enable(self, optmanager): + """Remove plugin name from the default ignore list.""" + optmanager.remove_from_default_ignore([self.name]) + + def disable(self, optmanager): + """Add the plugin name to the default ignore list.""" + optmanager.extend_default_ignore([self.name]) + + def provide_options(self, optmanager, options, extra_args): + """Pass the parsed options and extra arguments to the plugin.""" + parse_options = getattr(self.plugin, 'parse_options', None) + if parse_options is not None: + LOG.debug('Providing options to plugin "%s".', self.name) + try: + parse_options(optmanager, options, extra_args) + except TypeError: + parse_options(options) + + if self.name in options.enable_extensions: + self.enable(optmanager) + + def register_options(self, optmanager): + """Register the plugin's command-line options on the OptionManager. + + :param optmanager: + Instantiated OptionManager to register options on. + :type optmanager: + flake8.options.manager.OptionManager + :returns: + Nothing + """ + add_options = getattr(self.plugin, 'add_options', None) + if add_options is not None: + LOG.debug( + 'Registering options from plugin "%s" on OptionManager %r', + self.name, optmanager + ) + add_options(optmanager) + + if self.off_by_default: + self.disable(optmanager) + + +class PluginManager(object): # pylint: disable=too-few-public-methods + """Find and manage plugins consistently.""" + + def __init__(self, namespace, verify_requirements=False): + """Initialize the manager. + + :param str namespace: + Namespace of the plugins to manage, e.g., 'flake8.extension'. + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + """ + self.namespace = namespace + self.verify_requirements = verify_requirements + self.plugins = {} + self.names = [] + self._load_all_plugins() + + def _load_all_plugins(self): + LOG.info('Loading entry-points for "%s".', self.namespace) + for entry_point in pkg_resources.iter_entry_points(self.namespace): + name = entry_point.name + self.plugins[name] = Plugin(name, entry_point) + self.names.append(name) + LOG.debug('Loaded %r for plugin "%s".', self.plugins[name], name) + + def map(self, func, *args, **kwargs): + r"""Call ``func`` with the plugin and \*args and \**kwargs after. + + This yields the return value from ``func`` for each plugin. + + :param collections.Callable func: + Function to call with each plugin. Signature should at least be: + + .. code-block:: python + + def myfunc(plugin): + pass + + Any extra positional or keyword arguments specified with map will + be passed along to this function after the plugin. The plugin + passed is a :class:`~flake8.plugins.manager.Plugin`. + :param args: + Positional arguments to pass to ``func`` after each plugin. + :param kwargs: + Keyword arguments to pass to ``func`` after each plugin. + """ + for name in self.names: + yield func(self.plugins[name], *args, **kwargs) + + def versions(self): + # () -> (str, str) + """Generate the versions of plugins. + + :returns: + Tuples of the plugin_name and version + :rtype: + tuple + """ + plugins_seen = set() + for entry_point_name in self.names: + plugin = self.plugins[entry_point_name] + plugin_name = plugin.plugin_name + if plugin.plugin_name in plugins_seen: + continue + plugins_seen.add(plugin_name) + yield (plugin_name, plugin.version) + + +def version_for(plugin): + # (Plugin) -> Union[str, NoneType] + """Determine the version of a plugin by it's module. + + :param plugin: + The loaded plugin + :type plugin: + Plugin + :returns: + version string for the module + :rtype: + str + """ + module_name = plugin.plugin.__module__ + try: + module = __import__(module_name) + except ImportError: + return None + + return getattr(module, '__version__', None) + + +class PluginTypeManager(object): + """Parent class for most of the specific plugin types.""" + + namespace = None + + def __init__(self): + """Initialize the plugin type's manager.""" + self.manager = PluginManager(self.namespace) + self.plugins_loaded = False + + def __contains__(self, name): + """Check if the entry-point name is in this plugin type manager.""" + LOG.debug('Checking for "%s" in plugin type manager.', name) + return name in self.plugins + + def __getitem__(self, name): + """Retrieve a plugin by its name.""" + LOG.debug('Retrieving plugin for "%s".', name) + return self.plugins[name] + + def get(self, name, default=None): + """Retrieve the plugin referred to by ``name`` or return the default. + + :param str name: + Name of the plugin to retrieve. + :param default: + Default value to return. + :returns: + Plugin object referred to by name, if it exists. + :rtype: + :class:`Plugin` + """ + if name in self: + return self[name] + return default + + @property + def names(self): + """Proxy attribute to underlying manager.""" + return self.manager.names + + @property + def plugins(self): + """Proxy attribute to underlying manager.""" + return self.manager.plugins + + @staticmethod + def _generate_call_function(method_name, optmanager, *args, **kwargs): + def generated_function(plugin): + """Function that attempts to call a specific method on a plugin.""" + method = getattr(plugin, method_name, None) + if (method is not None and + isinstance(method, collections.Callable)): + return method(optmanager, *args, **kwargs) + return generated_function + + def load_plugins(self): + """Load all plugins of this type that are managed by this manager.""" + if self.plugins_loaded: + return + + def load_plugin(plugin): + """Call each plugin's load_plugin method.""" + return plugin.load_plugin() + + plugins = list(self.manager.map(load_plugin)) + # Do not set plugins_loaded if we run into an exception + self.plugins_loaded = True + return plugins + + def register_plugin_versions(self, optmanager): + """Register the plugins and their versions with the OptionManager.""" + self.load_plugins() + for (plugin_name, version) in self.manager.versions(): + optmanager.register_plugin(name=plugin_name, version=version) + + def register_options(self, optmanager): + """Register all of the checkers' options to the OptionManager.""" + self.load_plugins() + call_register_options = self._generate_call_function( + 'register_options', optmanager, + ) + + list(self.manager.map(call_register_options)) + + def provide_options(self, optmanager, options, extra_args): + """Provide parsed options and extra arguments to the plugins.""" + call_provide_options = self._generate_call_function( + 'provide_options', optmanager, options, extra_args, + ) + + list(self.manager.map(call_provide_options)) + + +class NotifierBuilderMixin(object): # pylint: disable=too-few-public-methods + """Mixin class that builds a Notifier from a PluginManager.""" + + def build_notifier(self): + """Build a Notifier for our Listeners. + + :returns: + Object to notify our listeners of certain error codes and + warnings. + :rtype: + :class:`~flake8.notifier.Notifier` + """ + notifier_trie = notifier.Notifier() + for name in self.names: + notifier_trie.register_listener(name, self.manager[name]) + return notifier_trie + + +class Checkers(PluginTypeManager): + """All of the checkers registered through entry-ponits.""" + + namespace = 'flake8.extension' + + def checks_expecting(self, argument_name): + """Retrieve checks that expect an argument with the specified name. + + Find all checker plugins that are expecting a specific argument. + """ + for plugin in self.plugins.values(): + if argument_name == plugin.parameters[0]: + yield plugin + + @property + def ast_plugins(self): + """List of plugins that expect the AST tree.""" + plugins = getattr(self, '_ast_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('tree')) + self._ast_plugins = plugins + return plugins + + @property + def logical_line_plugins(self): + """List of plugins that expect the logical lines.""" + plugins = getattr(self, '_logical_line_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('logical_line')) + self._logical_line_plugins = plugins + return plugins + + @property + def physical_line_plugins(self): + """List of plugins that expect the physical lines.""" + plugins = getattr(self, '_physical_line_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('physical_line')) + self._physical_line_plugins = plugins + return plugins + + +class Listeners(PluginTypeManager, NotifierBuilderMixin): + """All of the listeners registered through entry-points.""" + + namespace = 'flake8.listen' + + +class ReportFormatters(PluginTypeManager): + """All of the report formatters registered through entry-points.""" + + namespace = 'flake8.report' diff --git a/src/flake8/plugins/notifier.py b/src/flake8/plugins/notifier.py new file mode 100644 index 0000000..dc255c4 --- /dev/null +++ b/src/flake8/plugins/notifier.py @@ -0,0 +1,46 @@ +"""Implementation of the class that registers and notifies listeners.""" +from flake8.plugins import _trie + + +class Notifier(object): + """Object that tracks and notifies listener objects.""" + + def __init__(self): + """Initialize an empty notifier object.""" + self.listeners = _trie.Trie() + + def listeners_for(self, error_code): + """Retrieve listeners for an error_code. + + There may be listeners registered for E1, E100, E101, E110, E112, and + E126. To get all the listeners for one of E100, E101, E110, E112, or + E126 you would also need to incorporate the listeners for E1 (since + they're all in the same class). + + Example usage: + + .. code-block:: python + + from flake8 import notifier + + n = notifier.Notifier() + # register listeners + for listener in n.listeners_for('W102'): + listener.notify(...) + """ + path = error_code + while path: + node = self.listeners.find(path) + listeners = getattr(node, 'data', []) + for listener in listeners: + yield listener + path = path[:-1] + + def notify(self, error_code, *args, **kwargs): + """Notify all listeners for the specified error code.""" + for listener in self.listeners_for(error_code): + listener.notify(error_code, *args, **kwargs) + + def register_listener(self, error_code, listener): + """Register a listener for a specific error_code.""" + self.listeners.add(error_code, listener) diff --git a/flake8/_pyflakes.py b/src/flake8/plugins/pyflakes.py similarity index 53% rename from flake8/_pyflakes.py rename to src/flake8/plugins/pyflakes.py index e4fded6..158d5a2 100644 --- a/flake8/_pyflakes.py +++ b/src/flake8/plugins/pyflakes.py @@ -1,6 +1,9 @@ +"""Plugin built-in to Flake8 to treat pyflakes as a plugin.""" # -*- coding: utf-8 -*- +from __future__ import absolute_import + try: - # The 'demandimport' breaks pyflakes and flake8._pyflakes + # The 'demandimport' breaks pyflakes and flake8.plugins.pyflakes from mercurial import demandimport except ImportError: pass @@ -8,10 +11,11 @@ else: demandimport.disable() import os -import pycodestyle as pep8 import pyflakes import pyflakes.checker +from flake8 import utils + def patch_pyflakes(): """Add error codes to Pyflakes messages.""" @@ -21,7 +25,7 @@ def patch_pyflakes(): 'F403 ImportStarUsed', 'F404 LateFutureImport', 'F405 ImportStarUsage', - 'F810 Redefined', # XXX Obsolete? + 'F810 Redefined', 'F811 RedefinedWhileUnused', 'F812 RedefinedInListComp', 'F821 UndefinedName', @@ -39,76 +43,88 @@ patch_pyflakes() class FlakesChecker(pyflakes.checker.Checker): """Subclass the Pyflakes checker to conform with the flake8 API.""" + name = 'pyflakes' version = pyflakes.__version__ def __init__(self, tree, filename): - filename = pep8.normalize_paths(filename)[0] - withDoctest = self.withDoctest + """Initialize the PyFlakes plugin with an AST tree and filename.""" + filename = utils.normalize_paths(filename)[0] + with_doctest = self.with_doctest included_by = [include for include in self.include_in_doctest if include != '' and filename.startswith(include)] if included_by: - withDoctest = True + with_doctest = True for exclude in self.exclude_from_doctest: if exclude != '' and filename.startswith(exclude): - withDoctest = False + with_doctest = False overlaped_by = [include for include in included_by if include.startswith(exclude)] if overlaped_by: - withDoctest = True + with_doctest = True super(FlakesChecker, self).__init__(tree, filename, - withDoctest=withDoctest) + withDoctest=with_doctest) @classmethod def add_options(cls, parser): - parser.add_option('--builtins', - help="define more built-ins, comma separated") - parser.add_option('--doctests', default=False, action='store_true', - help="check syntax of the doctests") - parser.add_option('--include-in-doctest', default='', - dest='include_in_doctest', - help='Run doctests only on these files', - type='string') - parser.add_option('--exclude-from-doctest', default='', - dest='exclude_from_doctest', - help='Skip these files when running doctests', - type='string') - parser.config_options.extend(['builtins', 'doctests', - 'include-in-doctest', - 'exclude-from-doctest']) + """Register options for PyFlakes on the Flake8 OptionManager.""" + parser.add_option( + '--builtins', parse_from_config=True, comma_separated_list=True, + help="define more built-ins, comma separated", + ) + parser.add_option( + '--doctests', default=False, action='store_true', + parse_from_config=True, + help="check syntax of the doctests", + ) + parser.add_option( + '--include-in-doctest', default='', + dest='include_in_doctest', parse_from_config=True, + comma_separated_list=True, normalize_paths=True, + help='Run doctests only on these files', + type='string', + ) + parser.add_option( + '--exclude-from-doctest', default='', + dest='exclude_from_doctest', parse_from_config=True, + comma_separated_list=True, normalize_paths=True, + help='Skip these files when running doctests', + type='string', + ) @classmethod def parse_options(cls, options): + """Parse option values from Flake8's OptionManager.""" if options.builtins: - cls.builtIns = cls.builtIns.union(options.builtins.split(',')) - cls.withDoctest = options.doctests + cls.builtIns = cls.builtIns.union(options.builtins) + cls.with_doctest = options.doctests included_files = [] - for included_file in options.include_in_doctest.split(','): + for included_file in options.include_in_doctest: if included_file == '': continue if not included_file.startswith((os.sep, './', '~/')): included_files.append('./' + included_file) else: included_files.append(included_file) - cls.include_in_doctest = pep8.normalize_paths(','.join(included_files)) + cls.include_in_doctest = utils.normalize_paths(included_files) excluded_files = [] - for excluded_file in options.exclude_from_doctest.split(','): + for excluded_file in options.exclude_from_doctest: if excluded_file == '': continue if not excluded_file.startswith((os.sep, './', '~/')): excluded_files.append('./' + excluded_file) else: excluded_files.append(excluded_file) - cls.exclude_from_doctest = pep8.normalize_paths( - ','.join(excluded_files)) + cls.exclude_from_doctest = utils.normalize_paths(excluded_files) inc_exc = set(cls.include_in_doctest).intersection( - set(cls.exclude_from_doctest)) + cls.exclude_from_doctest + ) if inc_exc: raise ValueError('"%s" was specified in both the ' 'include-in-doctest and exclude-from-doctest ' @@ -116,6 +132,10 @@ class FlakesChecker(pyflakes.checker.Checker): 'both for doctesting.' % inc_exc) def run(self): - for m in self.messages: - col = getattr(m, 'col', 0) - yield m.lineno, col, (m.flake8_msg % m.message_args), m.__class__ + """Run the plugin.""" + for message in self.messages: + col = getattr(message, 'col', 0) + yield (message.lineno, + col, + (message.flake8_msg % message.message_args), + message.__class__) diff --git a/src/flake8/processor.py b/src/flake8/processor.py new file mode 100644 index 0000000..0c33cc2 --- /dev/null +++ b/src/flake8/processor.py @@ -0,0 +1,430 @@ +"""Module containing our file processor that tokenizes a file for checks.""" +import contextlib +import io +import logging +import re +import sys +import tokenize + +import flake8 +from flake8 import defaults +from flake8 import exceptions +from flake8 import utils + +LOG = logging.getLogger(__name__) +PyCF_ONLY_AST = 1024 +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + +SKIP_TOKENS = frozenset([tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, + tokenize.DEDENT]) + + +class FileProcessor(object): + """Processes a file and holdes state. + + This processes a file by generating tokens, logical and physical lines, + and AST trees. This also provides a way of passing state about the file + to checks expecting that state. Any public attribute on this object can + be requested by a plugin. The known public attributes are: + + - :attr:`blank_before` + - :attr:`blank_lines` + - :attr:`checker_state` + - :attr:`indect_char` + - :attr:`indent_level` + - :attr:`line_number` + - :attr:`logical_line` + - :attr:`max_line_length` + - :attr:`multiline` + - :attr:`noqa` + - :attr:`previous_indent_level` + - :attr:`previous_logical` + - :attr:`tokens` + - :attr:`total_lines` + - :attr:`verbose` + """ + + NOQA_FILE = re.compile(r'\s*# flake8[:=]\s*noqa', re.I) + + def __init__(self, filename, options, lines=None): + """Initialice our file processor. + + :param str filename: + Name of the file to process + """ + self.filename = filename + self.lines = lines + if lines is None: + self.lines = self.read_lines() + self.strip_utf_bom() + self.options = options + + # Defaults for public attributes + #: Number of preceding blank lines + self.blank_before = 0 + #: Number of blank lines + self.blank_lines = 0 + #: Checker states for each plugin? + self._checker_states = {} + #: Current checker state + self.checker_state = None + #: User provided option for hang closing + self.hang_closing = options.hang_closing + #: Character used for indentation + self.indent_char = None + #: Current level of indentation + self.indent_level = 0 + #: Line number in the file + self.line_number = 0 + #: Current logical line + self.logical_line = '' + #: Maximum line length as configured by the user + self.max_line_length = options.max_line_length + #: Whether the current physical line is multiline + self.multiline = False + #: Whether or not we're observing NoQA + self.noqa = False + #: Previous level of indentation + self.previous_indent_level = 0 + #: Previous logical line + self.previous_logical = '' + #: Current set of tokens + self.tokens = [] + #: Total number of lines in the file + self.total_lines = len(self.lines) + #: Verbosity level of Flake8 + self.verbose = options.verbose + #: Statistics dictionary + self.statistics = { + 'logical lines': 0, + } + + @contextlib.contextmanager + def inside_multiline(self, line_number): + """Context-manager to toggle the multiline attribute.""" + self.line_number = line_number + self.multiline = True + yield + self.multiline = False + + def reset_blank_before(self): + """Reset the blank_before attribute to zero.""" + self.blank_before = 0 + + def delete_first_token(self): + """Delete the first token in the list of tokens.""" + del self.tokens[0] + + def visited_new_blank_line(self): + """Note that we visited a new blank line.""" + self.blank_lines += 1 + + def update_state(self, mapping): + """Update the indent level based on the logical line mapping.""" + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + + def update_checker_state_for(self, plugin): + """Update the checker_state attribute for the plugin.""" + if 'checker_state' in plugin.parameters: + self.checker_state = self._checker_states.setdefault( + plugin.name, {} + ) + + def next_logical_line(self): + """Record the previous logical line. + + This also resets the tokens list and the blank_lines count. + """ + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + self.blank_lines = 0 + self.tokens = [] + + def build_logical_line_tokens(self): + """Build the mapping, comments, and logical line lists.""" + logical = [] + comments = [] + length = 0 + previous_row = previous_column = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = mutate_string(text) + if previous_row: + (start_row, start_column) = start + if previous_row != start_row: + row_index = previous_row - 1 + column_index = previous_column - 1 + previous_text = self.lines[row_index][column_index] + if (previous_text == ',' or + (previous_text not in '{[(' and + text not in '}])')): + text = ' ' + text + elif previous_column != start_column: + text = line[previous_column:start_column] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (previous_row, previous_column) = end + return comments, logical, mapping + + def build_ast(self): + """Build an abstract syntax tree from the list of lines.""" + return compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + + def build_logical_line(self): + """Build a logical line from the current tokens list.""" + comments, logical, mapping_list = self.build_logical_line_tokens() + self.logical_line = ''.join(logical) + self.statistics['logical lines'] += 1 + return ''.join(comments), self.logical_line, mapping_list + + def split_line(self, token): + """Split a physical line's line based on new-lines. + + This also auto-increments the line number for the caller. + """ + for line in token[1].split('\n')[:-1]: + yield line + self.line_number += 1 + + def keyword_arguments_for(self, parameters, arguments=None): + """Generate the keyword arguments for a list of parameters.""" + if arguments is None: + arguments = {} + for param in parameters: + if param in arguments: + continue + try: + arguments[param] = getattr(self, param) + except AttributeError as exc: + LOG.exception(exc) + raise + return arguments + + def check_physical_error(self, error_code, line): + """Update attributes based on error code and line.""" + if error_code == 'E101': + self.indent_char = line[0] + + def generate_tokens(self): + """Tokenize the file and yield the tokens. + + :raises flake8.exceptions.InvalidSyntax: + If a :class:`tokenize.TokenError` is raised while generating + tokens. + """ + try: + for token in tokenize.generate_tokens(self.next_line): + if token[2][0] > self.total_lines: + break + self.tokens.append(token) + yield token + # NOTE(sigmavirus24): pycodestyle was catching both a SyntaxError + # and a tokenize.TokenError. In looking a the source on Python 2 and + # Python 3, the SyntaxError should never arise from generate_tokens. + # If we were using tokenize.tokenize, we would have to catch that. Of + # course, I'm going to be unsurprised to be proven wrong at a later + # date. + except tokenize.TokenError as exc: + raise exceptions.InvalidSyntax(exc.message, exception=exc) + + def line_for(self, line_number): + """Retrieve the physical line at the specified line number.""" + return self.lines[line_number - 1] + + def next_line(self): + """Get the next line from the list.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in defaults.WHITESPACE: + self.indent_char = line[0] + return line + + def read_lines(self): + # type: () -> List[str] + """Read the lines for this file checker.""" + if self.filename is None or self.filename == '-': + self.filename = 'stdin' + return self.read_lines_from_stdin() + return self.read_lines_from_filename() + + def _readlines_py2(self): + # type: () -> List[str] + with open(self.filename, 'rU') as fd: + return fd.readlines() + + def _readlines_py3(self): + # type: () -> List[str] + try: + with open(self.filename, 'rb') as fd: + (coding, lines) = tokenize.detect_encoding(fd.readline) + textfd = io.TextIOWrapper(fd, coding, line_buffering=True) + return ([l.decode(coding) for l in lines] + + textfd.readlines()) + except (LookupError, SyntaxError, UnicodeError): + # If we can't detect the codec with tokenize.detect_encoding, or + # the detected encoding is incorrect, just fallback to latin-1. + with open(self.filename, encoding='latin-1') as fd: + return fd.readlines() + + def read_lines_from_filename(self): + # type: () -> List[str] + """Read the lines for a file.""" + if (2, 6) <= sys.version_info < (3, 0): + readlines = self._readlines_py2 + elif (3, 0) <= sys.version_info < (4, 0): + readlines = self._readlines_py3 + return readlines() + + def read_lines_from_stdin(self): + # type: () -> List[str] + """Read the lines from standard in.""" + return utils.stdin_get_value().splitlines(True) + + def should_ignore_file(self): + # type: () -> bool + """Check if ``# flake8: noqa`` is in the file to be ignored. + + :returns: + True if a line matches :attr:`FileProcessor.NOQA_FILE`, + otherwise False + :rtype: + bool + """ + ignore_file = self.NOQA_FILE.search + return any(ignore_file(line) for line in self.lines) + + def strip_utf_bom(self): + # type: () -> NoneType + """Strip the UTF bom from the lines of the file.""" + if not self.lines: + # If we have nothing to analyze quit early + return + + first_byte = ord(self.lines[0][0]) + if first_byte not in (0xEF, 0xFEFF): + return + + # If the first byte of the file is a UTF-8 BOM, strip it + if first_byte == 0xFEFF: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xEF\xBB\xBF': + self.lines[0] = self.lines[0][3:] + + +def is_eol_token(token): + """Check if the token is an end-of-line token.""" + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' + +if COMMENT_WITH_NL: # If on Python 2.6 + def is_eol_token(token, _is_eol_token=is_eol_token): + """Check if the token is an end-of-line token.""" + return (_is_eol_token(token) or + (token[0] == tokenize.COMMENT and token[1] == token[4])) + + +def is_multiline_string(token): + """Check if this is a multiline string.""" + return token[0] == tokenize.STRING and '\n' in token[1] + + +def token_is_newline(token): + """Check if the token type is a newline token type.""" + return token[0] in NEWLINE + + +def token_is_comment(token): + """Check if the token type is a comment.""" + return COMMENT_WITH_NL and token[0] == tokenize.COMMENT + + +def count_parentheses(current_parentheses_count, token_text): + """Count the number of parentheses.""" + current_parentheses_count = current_parentheses_count or 0 + if token_text in '([{': + return current_parentheses_count + 1 + elif token_text in '}])': + return current_parentheses_count - 1 + return current_parentheses_count + + +def log_token(log, token): + """Log a token to a provided logging object.""" + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + log.log(flake8._EXTRA_VERBOSE, 'l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], + token[1])) + + +# NOTE(sigmavirus24): This was taken wholesale from +# https://github.com/PyCQA/pycodestyle +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +# NOTE(sigmavirus24): This was taken wholesale from +# https://github.com/PyCQA/pycodestyle. The in-line comments were edited to be +# more descriptive. +def mutate_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # NOTE(sigmavirus24): If there are string modifiers (e.g., b, u, r) + # use the last "character" to determine if we're using single or double + # quotes and then find the first instance of it + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Check for triple-quoted strings + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] diff --git a/src/flake8/style_guide.py b/src/flake8/style_guide.py new file mode 100644 index 0000000..89890ba --- /dev/null +++ b/src/flake8/style_guide.py @@ -0,0 +1,283 @@ +"""Implementation of the StyleGuide used by Flake8.""" +import collections +import enum +import linecache +import logging +import re + +from flake8 import utils + +__all__ = ( + 'StyleGuide', +) + +LOG = logging.getLogger(__name__) + + +# TODO(sigmavirus24): Determine if we need to use enum/enum34 +class Selected(enum.Enum): + """Enum representing an explicitly or implicitly selected code.""" + + Explicitly = 'explicitly selected' + Implicitly = 'implicitly selected' + + +class Ignored(enum.Enum): + """Enum representing an explicitly or implicitly ignored code.""" + + Explicitly = 'explicitly ignored' + Implicitly = 'implicitly ignored' + + +class Decision(enum.Enum): + """Enum representing whether a code should be ignored or selected.""" + + Ignored = 'ignored error' + Selected = 'selected error' + + +Error = collections.namedtuple( + 'Error', + [ + 'code', + 'filename', + 'line_number', + 'column_number', + 'text', + 'physical_line', + ], +) + + +class StyleGuide(object): + """Manage a Flake8 user's style guide.""" + + NOQA_INLINE_REGEXP = re.compile( + # We're looking for items that look like this: + # ``# noqa`` + # ``# noqa: E123`` + # ``# noqa: E123,W451,F921`` + # ``# NoQA: E123,W451,F921`` + # ``# NOQA: E123,W451,F921`` + # We do not care about the ``: `` that follows ``noqa`` + # We do not care about the casing of ``noqa`` + # We want a comma-separated list of errors + '# noqa(?:: )?(?P[A-Z0-9,]+)?$', + re.IGNORECASE + ) + + def __init__(self, options, listener_trie, formatter): + """Initialize our StyleGuide. + + .. todo:: Add parameter documentation. + """ + self.options = options + self.listener = listener_trie + self.formatter = formatter + self._selected = tuple(options.select) + self._ignored = tuple(options.ignore) + self._decision_cache = {} + self._parsed_diff = {} + + def is_user_selected(self, code): + # type: (str) -> Union[Selected, Ignored] + """Determine if the code has been selected by the user. + + :param str code: + The code for the check that has been run. + :returns: + Selected.Implicitly if the selected list is empty, + Selected.Explicitly if the selected list is not empty and a match + was found, + Ignored.Implicitly if the selected list is not empty but no match + was found. + """ + if not self._selected: + return Selected.Implicitly + + if code.startswith(self._selected): + return Selected.Explicitly + + return Ignored.Implicitly + + def is_user_ignored(self, code): + # type: (str) -> Union[Selected, Ignored] + """Determine if the code has been ignored by the user. + + :param str code: + The code for the check that has been run. + :returns: + Selected.Implicitly if the ignored list is empty, + Ignored.Explicitly if the ignored list is not empty and a match was + found, + Selected.Implicitly if the ignored list is not empty but no match + was found. + """ + if self._ignored and code.startswith(self._ignored): + return Ignored.Explicitly + + return Selected.Implicitly + + def _decision_for(self, code): + # type: (Error) -> Decision + startswith = code.startswith + selected = sorted([s for s in self._selected if startswith(s)])[0] + ignored = sorted([i for i in self._ignored if startswith(i)])[0] + + if selected.startswith(ignored): + return Decision.Selected + return Decision.Ignored + + def should_report_error(self, code): + # type: (str) -> Decision + """Determine if the error code should be reported or ignored. + + This method only cares about the select and ignore rules as specified + by the user in their configuration files and command-line flags. + + This method does not look at whether the specific line is being + ignored in the file itself. + + :param str code: + The code for the check that has been run. + """ + decision = self._decision_cache.get(code) + if decision is None: + LOG.debug('Deciding if "%s" should be reported', code) + selected = self.is_user_selected(code) + ignored = self.is_user_ignored(code) + LOG.debug('The user configured "%s" to be "%s", "%s"', + code, selected, ignored) + + if ((selected is Selected.Explicitly or + selected is Selected.Implicitly) and + ignored is Selected.Implicitly): + decision = Decision.Selected + elif (selected is Selected.Explicitly and + ignored is Ignored.Explicitly): + decision = self._decision_for(code) + elif (selected is Ignored.Implicitly or + ignored is Ignored.Explicitly): + decision = Decision.Ignored # pylint: disable=R0204 + + self._decision_cache[code] = decision + LOG.debug('"%s" will be "%s"', code, decision) + return decision + + def is_inline_ignored(self, error): + # type: (Error) -> bool + """Determine if an comment has been added to ignore this line.""" + physical_line = error.physical_line + # TODO(sigmavirus24): Determine how to handle stdin with linecache + if self.options.disable_noqa: + return False + + if physical_line is None: + physical_line = linecache.getline(error.filename, + error.line_number) + noqa_match = self.NOQA_INLINE_REGEXP.search(physical_line) + if noqa_match is None: + LOG.debug('%r is not inline ignored', error) + return False + + codes_str = noqa_match.groupdict()['codes'] + if codes_str is None: + LOG.debug('%r is ignored by a blanket ``# noqa``', error) + return True + + codes = set(utils.parse_comma_separated_list(codes_str)) + if error.code in codes or error.code.startswith(tuple(codes)): + LOG.debug('%r is ignored specifically inline with ``# noqa: %s``', + error, codes_str) + return True + + LOG.debug('%r is not ignored inline with ``# noqa: %s``', + error, codes_str) + return False + + def is_in_diff(self, error): + # type: (Error) -> bool + """Determine if an error is included in a diff's line ranges. + + This function relies on the parsed data added via + :meth:`~StyleGuide.add_diff_ranges`. If that has not been called and + we are not evaluating files in a diff, then this will always return + True. If there are diff ranges, then this will return True if the + line number in the error falls inside one of the ranges for the file + (and assuming the file is part of the diff data). If there are diff + ranges, this will return False if the file is not part of the diff + data or the line number of the error is not in any of the ranges of + the diff. + + :returns: + True if there is no diff or if the error is in the diff's line + number ranges. False if the error's line number falls outside + the diff's line number ranges. + :rtype: + bool + """ + if not self._parsed_diff: + return True + + # NOTE(sigmavirus24): The parsed diff will be a defaultdict with + # a set as the default value (if we have received it from + # flake8.utils.parse_unified_diff). In that case ranges below + # could be an empty set (which is False-y) or if someone else + # is using this API, it could be None. If we could guarantee one + # or the other, we would check for it more explicitly. + line_numbers = self._parsed_diff.get(error.filename) + if not line_numbers: + return False + + return error.line_number in line_numbers + + def handle_error(self, code, filename, line_number, column_number, text, + physical_line=None): + # type: (str, str, int, int, str) -> int + """Handle an error reported by a check. + + :param str code: + The error code found, e.g., E123. + :param str filename: + The file in which the error was found. + :param int line_number: + The line number (where counting starts at 1) at which the error + occurs. + :param int column_number: + The column number (where counting starts at 1) at which the error + occurs. + :param str text: + The text of the error message. + :param str physical_line: + The actual physical line causing the error. + :returns: + 1 if the error was reported. 0 if it was ignored. This is to allow + for counting of the number of errors found that were not ignored. + :rtype: + int + """ + error = Error(code, filename, line_number, column_number, text, + physical_line) + if error.filename is None or error.filename == '-': + error = error._replace(filename=self.options.stdin_display_name) + error_is_selected = (self.should_report_error(error.code) is + Decision.Selected) + is_not_inline_ignored = self.is_inline_ignored(error) is False + is_included_in_diff = self.is_in_diff(error) + if (error_is_selected and is_not_inline_ignored and + is_included_in_diff): + self.formatter.handle(error) + self.listener.notify(error.code, error) + return 1 + return 0 + + def add_diff_ranges(self, diffinfo): + """Update the StyleGuide to filter out information not in the diff. + + This provides information to the StyleGuide so that only the errors + in the line number ranges are reported. + + :param dict diffinfo: + Dictionary mapping filenames to sets of line number ranges. + """ + self._parsed_diff = diffinfo diff --git a/src/flake8/utils.py b/src/flake8/utils.py new file mode 100644 index 0000000..597dea6 --- /dev/null +++ b/src/flake8/utils.py @@ -0,0 +1,279 @@ +"""Utility methods for flake8.""" +import collections +import fnmatch as _fnmatch +import inspect +import io +import os +import re +import sys + +DIFF_HUNK_REGEXP = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') + + +def parse_comma_separated_list(value): + # type: (Union[Sequence[str], str]) -> List[str] + """Parse a comma-separated list. + + :param value: + String or list of strings to be parsed and normalized. + :returns: + List of values with whitespace stripped. + :rtype: + list + """ + if not value: + return [] + + if not isinstance(value, (list, tuple)): + value = value.split(',') + + return [item.strip() for item in value] + + +def normalize_paths(paths, parent=os.curdir): + # type: (Union[Sequence[str], str], str) -> List[str] + """Parse a comma-separated list of paths. + + :returns: + The normalized paths. + :rtype: + [str] + """ + return [normalize_path(p, parent) + for p in parse_comma_separated_list(paths)] + + +def normalize_path(path, parent=os.curdir): + # type: (str, str) -> str + """Normalize a single-path. + + :returns: + The normalized path. + :rtype: + str + """ + # NOTE(sigmavirus24): Using os.path.sep allows for Windows paths to + # be specified and work appropriately. + separator = os.path.sep + if separator in path: + path = os.path.abspath(os.path.join(parent, path)) + return path.rstrip(separator) + + +def stdin_get_value(): + # type: () -> str + """Get and cache it so plugins can use it.""" + cached_value = getattr(stdin_get_value, 'cached_stdin', None) + if cached_value is None: + stdin_value = sys.stdin.read() + if sys.version_info < (3, 0): + cached_type = io.BytesIO + else: + cached_type = io.StringIO + stdin_get_value.cached_stdin = cached_type(stdin_value) + cached_value = stdin_get_value.cached_stdin + return cached_value.getvalue() + + +def parse_unified_diff(diff=None): + # type: (str) -> List[str] + """Parse the unified diff passed on stdin. + + :returns: + dictionary mapping file names to sets of line numbers + :rtype: + dict + """ + # Allow us to not have to patch out stdin_get_value + if diff is None: + diff = stdin_get_value() + + number_of_rows = None + current_path = None + parsed_paths = collections.defaultdict(set) + for line in diff.splitlines(): + if number_of_rows: + # NOTE(sigmavirus24): Below we use a slice because stdin may be + # bytes instead of text on Python 3. + if line[:1] != '-': + number_of_rows -= 1 + # We're in the part of the diff that has lines starting with +, -, + # and ' ' to show context and the changes made. We skip these + # because the information we care about is the filename and the + # range within it. + # When number_of_rows reaches 0, we will once again start + # searching for filenames and ranges. + continue + + # NOTE(sigmavirus24): Diffs that we support look roughly like: + # diff a/file.py b/file.py + # ... + # --- a/file.py + # +++ b/file.py + # Below we're looking for that last line. Every diff tool that + # gives us this output may have additional information after + # ``b/file.py`` which it will separate with a \t, e.g., + # +++ b/file.py\t100644 + # Which is an example that has the new file permissions/mode. + # In this case we only care about the file name. + if line[:3] == '+++': + current_path = line[4:].split('\t', 1)[0] + # NOTE(sigmavirus24): This check is for diff output from git. + if current_path[:2] == 'b/': + current_path = current_path[2:] + # We don't need to do anything else. We have set up our local + # ``current_path`` variable. We can skip the rest of this loop. + # The next line we will see will give us the hung information + # which is in the next section of logic. + continue + + hunk_match = DIFF_HUNK_REGEXP.match(line) + # NOTE(sigmavirus24): pep8/pycodestyle check for: + # line[:3] == '@@ ' + # But the DIFF_HUNK_REGEXP enforces that the line start with that + # So we can more simply check for a match instead of slicing and + # comparing. + if hunk_match: + (row, number_of_rows) = [ + 1 if not group else int(group) + for group in hunk_match.groups() + ] + parsed_paths[current_path].update( + range(row, row + number_of_rows) + ) + + # We have now parsed our diff into a dictionary that looks like: + # {'file.py': set(range(10, 16), range(18, 20)), ...} + return parsed_paths + + +def is_windows(): + # type: () -> bool + """Determine if we're running on Windows. + + :returns: + True if running on Windows, otherwise False + :rtype: + bool + """ + return os.name == 'nt' + + +def can_run_multiprocessing_on_windows(): + # type: () -> bool + """Determine if we can use multiprocessing on Windows. + + :returns: + True if the version of Python is modern enough, otherwise False + :rtype: + bool + """ + is_new_enough_python27 = sys.version_info >= (2, 7, 11) + is_new_enough_python3 = sys.version_info > (3, 2) + return is_new_enough_python27 or is_new_enough_python3 + + +def is_using_stdin(paths): + # type: (List[str]) -> bool + """Determine if we're going to read from stdin. + + :param list paths: + The paths that we're going to check. + :returns: + True if stdin (-) is in the path, otherwise False + :rtype: + bool + """ + return '-' in paths + + +def _default_predicate(*args): + return False + + +def filenames_from(arg, predicate=None): + # type: (str, callable) -> Generator + """Generate filenames from an argument. + + :param str arg: + Parameter from the command-line. + :param callable predicate: + Predicate to use to filter out filenames. If the predicate + returns ``True`` we will exclude the filename, otherwise we + will yield it. By default, we include every filename + generated. + :returns: + Generator of paths + """ + if predicate is None: + predicate = _default_predicate + if os.path.isdir(arg): + for root, sub_directories, files in os.walk(arg): + for filename in files: + joined = os.path.join(root, filename) + if predicate(joined): + continue + yield joined + # NOTE(sigmavirus24): os.walk() will skip a directory if you + # remove it from the list of sub-directories. + for directory in sub_directories: + if predicate(directory): + sub_directories.remove(directory) + else: + yield arg + + +def fnmatch(filename, patterns, default=True): + # type: (str, List[str], bool) -> bool + """Wrap :func:`fnmatch.fnmatch` to add some functionality. + + :param str filename: + Name of the file we're trying to match. + :param list patterns: + Patterns we're using to try to match the filename. + :param bool default: + The default value if patterns is empty + :returns: + True if a pattern matches the filename, False if it doesn't. + ``default`` if patterns is empty. + """ + if not patterns: + return default + return any(_fnmatch.fnmatch(filename, pattern) for pattern in patterns) + + +def parameters_for(plugin): + # type: (flake8.plugins.manager.Plugin) -> List[str] + """Return the parameters for the plugin. + + This will inspect the plugin and return either the function parameters + if the plugin is a function or the parameters for ``__init__`` after + ``self`` if the plugin is a class. + + :param plugin: + The internal plugin object. + :type plugin: + flake8.plugins.manager.Plugin + :returns: + Parameters to the plugin. + :rtype: + list(str) + """ + func = plugin.plugin + is_class = not inspect.isfunction(func) + if is_class: # The plugin is a class + func = plugin.plugin.__init__ + + if sys.version_info < (3, 3): + parameters = inspect.getargspec(func)[0] + else: + parameters = [ + parameter.name + for parameter in inspect.signature(func).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD + ] + + if is_class: + parameters.remove('self') + + return parameters diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9bf4f95 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,6 @@ +"""Test configuration for py.test.""" +import sys + +import flake8 + +flake8.configure_logging(2, 'test-logs-%s.%s.log' % sys.version_info[0:2]) diff --git a/tests/fixtures/config_files/README.rst b/tests/fixtures/config_files/README.rst new file mode 100644 index 0000000..b00adad --- /dev/null +++ b/tests/fixtures/config_files/README.rst @@ -0,0 +1,38 @@ +About this directory +==================== + +The files in this directory are test fixtures for unit and integration tests. +Their purpose is described below. Please note the list of file names that can +not be created as they are already used by tests. + +New fixtures are preferred over updating existing features unless existing +tests will fail. + +Files that should not be created +-------------------------------- + +- ``tests/fixtures/config_files/missing.ini`` + +Purposes of existing fixtures +----------------------------- + +``tests/fixtures/config_files/cli-specified.ini`` + + This should only be used when providing config file(s) specified by the + user on the command-line. + +``tests/fixtures/config_files/local-config.ini`` + + This should be used when providing config files that would have been found + by looking for config files in the current working project directory. + + +``tests/fixtures/config_files/no-flake8-section.ini`` + + This should be used when parsing an ini file without a ``[flake8]`` + section. + +``tests/fixtures/config_files/user-config.ini`` + + This is an example configuration file that would be found in the user's + home directory (or XDG Configuration Directory). diff --git a/tests/fixtures/config_files/broken.ini b/tests/fixtures/config_files/broken.ini new file mode 100644 index 0000000..33986ae --- /dev/null +++ b/tests/fixtures/config_files/broken.ini @@ -0,0 +1,9 @@ +[flake8] +exclude = +<<<<<<< 642f88cb1b6027e184d9a662b255f7fea4d9eacc + tests/fixtures/, +======= + tests/, +>>>>>>> HEAD + docs/ +ignore = D203 diff --git a/tests/fixtures/config_files/cli-specified-with-inline-comments.ini b/tests/fixtures/config_files/cli-specified-with-inline-comments.ini new file mode 100644 index 0000000..4d57e85 --- /dev/null +++ b/tests/fixtures/config_files/cli-specified-with-inline-comments.ini @@ -0,0 +1,16 @@ +[flake8] +# This is a flake8 config, there are many like it, but this is mine +ignore = + # Disable E123 + E123, + # Disable W234 + W234, + # Also disable E111 + E111 +exclude = + # Exclude foo/ + foo/, + # Exclude bar/ while we're at it + bar/, + # Exclude bogus/ + bogus/ diff --git a/tests/fixtures/config_files/cli-specified-without-inline-comments.ini b/tests/fixtures/config_files/cli-specified-without-inline-comments.ini new file mode 100644 index 0000000..f50ba75 --- /dev/null +++ b/tests/fixtures/config_files/cli-specified-without-inline-comments.ini @@ -0,0 +1,16 @@ +[flake8] +# This is a flake8 config, there are many like it, but this is mine +# Disable E123 +# Disable W234 +# Also disable E111 +ignore = + E123, + W234, + E111 +# Exclude foo/ +# Exclude bar/ while we're at it +# Exclude bogus/ +exclude = + foo/, + bar/, + bogus/ diff --git a/tests/fixtures/config_files/cli-specified.ini b/tests/fixtures/config_files/cli-specified.ini new file mode 100644 index 0000000..753604a --- /dev/null +++ b/tests/fixtures/config_files/cli-specified.ini @@ -0,0 +1,9 @@ +[flake8] +ignore = + E123, + W234, + E111 +exclude = + foo/, + bar/, + bogus/ diff --git a/tests/fixtures/config_files/local-config.ini b/tests/fixtures/config_files/local-config.ini new file mode 100644 index 0000000..348751a --- /dev/null +++ b/tests/fixtures/config_files/local-config.ini @@ -0,0 +1,3 @@ +[flake8] +exclude = docs/ +select = E,W,F diff --git a/tests/fixtures/config_files/no-flake8-section.ini b/tests/fixtures/config_files/no-flake8-section.ini new file mode 100644 index 0000000..a85b709 --- /dev/null +++ b/tests/fixtures/config_files/no-flake8-section.ini @@ -0,0 +1,20 @@ +[tox] +minversion=2.3.1 +envlist = py26,py27,py32,py33,py34,py35,flake8 + +[testenv] +deps = + mock + pytest +commands = + py.test {posargs} + +[testenv:flake8] +skipsdist = true +skip_install = true +use_develop = false +deps = + flake8 + flake8-docstrings +commands = + flake8 diff --git a/tests/fixtures/config_files/user-config.ini b/tests/fixtures/config_files/user-config.ini new file mode 100644 index 0000000..b06c24f --- /dev/null +++ b/tests/fixtures/config_files/user-config.ini @@ -0,0 +1,5 @@ +[flake8] +exclude = + tests/fixtures/, + docs/ +ignore = D203 diff --git a/tests/fixtures/diffs/multi_file_diff b/tests/fixtures/diffs/multi_file_diff new file mode 100644 index 0000000..de86209 --- /dev/null +++ b/tests/fixtures/diffs/multi_file_diff @@ -0,0 +1,130 @@ +diff --git a/flake8/utils.py b/flake8/utils.py +index f6ce384..7cd12b0 100644 +--- a/flake8/utils.py ++++ b/flake8/utils.py +@@ -75,8 +75,8 @@ def stdin_get_value(): + return cached_value.getvalue() + + +-def parse_unified_diff(): +- # type: () -> List[str] ++def parse_unified_diff(diff=None): ++ # type: (str) -> List[str] + """Parse the unified diff passed on stdin. + + :returns: +@@ -84,7 +84,10 @@ def parse_unified_diff(): + :rtype: + dict + """ +- diff = stdin_get_value() ++ # Allow us to not have to patch out stdin_get_value ++ if diff is None: ++ diff = stdin_get_value() ++ + number_of_rows = None + current_path = None + parsed_paths = collections.defaultdict(set) +diff --git a/tests/fixtures/diffs/single_file_diff b/tests/fixtures/diffs/single_file_diff +new file mode 100644 +index 0000000..77ca534 +--- /dev/null ++++ b/tests/fixtures/diffs/single_file_diff +@@ -0,0 +1,27 @@ ++diff --git a/flake8/utils.py b/flake8/utils.py ++index f6ce384..7cd12b0 100644 ++--- a/flake8/utils.py +++++ b/flake8/utils.py ++@@ -75,8 +75,8 @@ def stdin_get_value(): ++ return cached_value.getvalue() ++ ++ ++-def parse_unified_diff(): ++- # type: () -> List[str] +++def parse_unified_diff(diff=None): +++ # type: (str) -> List[str] ++ """Parse the unified diff passed on stdin. ++ ++ :returns: ++@@ -84,7 +84,10 @@ def parse_unified_diff(): ++ :rtype: ++ dict ++ """ ++- diff = stdin_get_value() +++ # Allow us to not have to patch out stdin_get_value +++ if diff is None: +++ diff = stdin_get_value() +++ ++ number_of_rows = None ++ current_path = None ++ parsed_paths = collections.defaultdict(set) +diff --git a/tests/fixtures/diffs/two_file_diff b/tests/fixtures/diffs/two_file_diff +new file mode 100644 +index 0000000..5bd35cd +--- /dev/null ++++ b/tests/fixtures/diffs/two_file_diff +@@ -0,0 +1,45 @@ ++diff --git a/flake8/utils.py b/flake8/utils.py ++index f6ce384..7cd12b0 100644 ++--- a/flake8/utils.py +++++ b/flake8/utils.py ++@@ -75,8 +75,8 @@ def stdin_get_value(): ++ return cached_value.getvalue() ++ ++ ++-def parse_unified_diff(): ++- # type: () -> List[str] +++def parse_unified_diff(diff=None): +++ # type: (str) -> List[str] ++ """Parse the unified diff passed on stdin. ++ ++ :returns: ++@@ -84,7 +84,10 @@ def parse_unified_diff(): ++ :rtype: ++ dict ++ """ ++- diff = stdin_get_value() +++ # Allow us to not have to patch out stdin_get_value +++ if diff is None: +++ diff = stdin_get_value() +++ ++ number_of_rows = None ++ current_path = None ++ parsed_paths = collections.defaultdict(set) ++diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py ++index d69d939..21482ce 100644 ++--- a/tests/unit/test_utils.py +++++ b/tests/unit/test_utils.py ++@@ -115,3 +115,13 @@ def test_parameters_for_function_plugin(): ++ plugin = plugin_manager.Plugin('plugin-name', object()) ++ plugin._plugin = fake_plugin ++ assert utils.parameters_for(plugin) == ['physical_line', 'self', 'tree'] +++ +++ +++def read_diff_file(filename): +++ """Read the diff file in its entirety.""" +++ with open(filename, 'r') as fd: +++ content = fd.read() +++ return content +++ +++ +++SINGLE_FILE_DIFF = read_diff_file('tests/fixtures/diffs/single_file_diff') +diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py +index d69d939..1461369 100644 +--- a/tests/unit/test_utils.py ++++ b/tests/unit/test_utils.py +@@ -115,3 +115,14 @@ def test_parameters_for_function_plugin(): + plugin = plugin_manager.Plugin('plugin-name', object()) + plugin._plugin = fake_plugin + assert utils.parameters_for(plugin) == ['physical_line', 'self', 'tree'] ++ ++ ++def read_diff_file(filename): ++ """Read the diff file in its entirety.""" ++ with open(filename, 'r') as fd: ++ content = fd.read() ++ return content ++ ++ ++SINGLE_FILE_DIFF = read_diff_file('tests/fixtures/diffs/single_file_diff') ++TWO_FILE_DIFF = read_diff_file('tests/fixtures/diffs/two_file_diff') diff --git a/tests/fixtures/diffs/single_file_diff b/tests/fixtures/diffs/single_file_diff new file mode 100644 index 0000000..77ca534 --- /dev/null +++ b/tests/fixtures/diffs/single_file_diff @@ -0,0 +1,27 @@ +diff --git a/flake8/utils.py b/flake8/utils.py +index f6ce384..7cd12b0 100644 +--- a/flake8/utils.py ++++ b/flake8/utils.py +@@ -75,8 +75,8 @@ def stdin_get_value(): + return cached_value.getvalue() + + +-def parse_unified_diff(): +- # type: () -> List[str] ++def parse_unified_diff(diff=None): ++ # type: (str) -> List[str] + """Parse the unified diff passed on stdin. + + :returns: +@@ -84,7 +84,10 @@ def parse_unified_diff(): + :rtype: + dict + """ +- diff = stdin_get_value() ++ # Allow us to not have to patch out stdin_get_value ++ if diff is None: ++ diff = stdin_get_value() ++ + number_of_rows = None + current_path = None + parsed_paths = collections.defaultdict(set) diff --git a/tests/fixtures/diffs/two_file_diff b/tests/fixtures/diffs/two_file_diff new file mode 100644 index 0000000..5bd35cd --- /dev/null +++ b/tests/fixtures/diffs/two_file_diff @@ -0,0 +1,45 @@ +diff --git a/flake8/utils.py b/flake8/utils.py +index f6ce384..7cd12b0 100644 +--- a/flake8/utils.py ++++ b/flake8/utils.py +@@ -75,8 +75,8 @@ def stdin_get_value(): + return cached_value.getvalue() + + +-def parse_unified_diff(): +- # type: () -> List[str] ++def parse_unified_diff(diff=None): ++ # type: (str) -> List[str] + """Parse the unified diff passed on stdin. + + :returns: +@@ -84,7 +84,10 @@ def parse_unified_diff(): + :rtype: + dict + """ +- diff = stdin_get_value() ++ # Allow us to not have to patch out stdin_get_value ++ if diff is None: ++ diff = stdin_get_value() ++ + number_of_rows = None + current_path = None + parsed_paths = collections.defaultdict(set) +diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py +index d69d939..21482ce 100644 +--- a/tests/unit/test_utils.py ++++ b/tests/unit/test_utils.py +@@ -115,3 +115,13 @@ def test_parameters_for_function_plugin(): + plugin = plugin_manager.Plugin('plugin-name', object()) + plugin._plugin = fake_plugin + assert utils.parameters_for(plugin) == ['physical_line', 'self', 'tree'] ++ ++ ++def read_diff_file(filename): ++ """Read the diff file in its entirety.""" ++ with open(filename, 'r') as fd: ++ content = fd.read() ++ return content ++ ++ ++SINGLE_FILE_DIFF = read_diff_file('tests/fixtures/diffs/single_file_diff') diff --git a/tests/fixtures/example-code/inline-ignores/E501.py b/tests/fixtures/example-code/inline-ignores/E501.py new file mode 100644 index 0000000..62e5c0c --- /dev/null +++ b/tests/fixtures/example-code/inline-ignores/E501.py @@ -0,0 +1,3 @@ +from some.module.that.has.nested.sub.modules import ClassWithVeryVeryVeryVeryLongName # noqa: E501,F401 + +# ClassWithVeryVeryVeryVeryLongName() diff --git a/tests/fixtures/example-code/inline-ignores/E731.py b/tests/fixtures/example-code/inline-ignores/E731.py new file mode 100644 index 0000000..866c79e --- /dev/null +++ b/tests/fixtures/example-code/inline-ignores/E731.py @@ -0,0 +1 @@ +example = lambda: 'example' # noqa: E731 diff --git a/tests/integration/test_aggregator.py b/tests/integration/test_aggregator.py new file mode 100644 index 0000000..929bdbf --- /dev/null +++ b/tests/integration/test_aggregator.py @@ -0,0 +1,48 @@ +"""Test aggregation of config files and command-line options.""" +import os + +import pytest + +from flake8.main import options +from flake8.options import aggregator +from flake8.options import manager + +CLI_SPECIFIED_CONFIG = 'tests/fixtures/config_files/cli-specified.ini' + + +@pytest.fixture +def optmanager(): + """Create a new OptionManager.""" + option_manager = manager.OptionManager( + prog='flake8', + version='3.0.0', + ) + options.register_default_options(option_manager) + return option_manager + + +def test_aggregate_options_with_config(optmanager): + """Verify we aggregate options and config values appropriately.""" + arguments = ['flake8', '--config', CLI_SPECIFIED_CONFIG, '--select', + 'E11,E34,E402,W,F', '--exclude', 'tests/*'] + options, args = aggregator.aggregate_options(optmanager, arguments) + + assert options.config == CLI_SPECIFIED_CONFIG + assert options.select == ['E11', 'E34', 'E402', 'W', 'F'] + assert options.ignore == ['E123', 'W234', 'E111'] + assert options.exclude == [os.path.abspath('tests/*')] + + +def test_aggregate_options_when_isolated(optmanager): + """Verify we aggregate options and config values appropriately.""" + arguments = ['flake8', '--isolated', '--select', 'E11,E34,E402,W,F', + '--exclude', 'tests/*'] + optmanager.extend_default_ignore(['E8']) + options, args = aggregator.aggregate_options(optmanager, arguments) + + assert options.isolated is True + assert options.select == ['E11', 'E34', 'E402', 'W', 'F'] + assert sorted(options.ignore) == [ + 'E121', 'E123', 'E126', 'E226', 'E24', 'E704', 'E8', 'W503', 'W504', + ] + assert options.exclude == [os.path.abspath('tests/*')] diff --git a/tests/unit/test_base_formatter.py b/tests/unit/test_base_formatter.py new file mode 100644 index 0000000..dc4a95c --- /dev/null +++ b/tests/unit/test_base_formatter.py @@ -0,0 +1,143 @@ +"""Tests for the BaseFormatter object.""" +import optparse + +import mock +import pytest + +from flake8 import style_guide +from flake8.formatting import base + + +def options(**kwargs): + """Create an optparse.Values instance.""" + kwargs.setdefault('output_file', None) + return optparse.Values(kwargs) + + +@pytest.mark.parametrize('filename', [None, 'out.txt']) +def test_start(filename): + """Verify we open a new file in the start method.""" + mock_open = mock.mock_open() + formatter = base.BaseFormatter(options(output_file=filename)) + with mock.patch('flake8.formatting.base.open', mock_open): + formatter.start() + + if filename is None: + assert mock_open.called is False + else: + mock_open.assert_called_once_with(filename, 'w') + + +def test_stop(): + """Verify we close open file objects.""" + filemock = mock.Mock() + formatter = base.BaseFormatter(options()) + formatter.output_fd = filemock + formatter.stop() + + filemock.close.assert_called_once_with() + assert formatter.output_fd is None + + +def test_format_needs_to_be_implemented(): + """Ensure BaseFormatter#format raises a NotImplementedError.""" + formatter = base.BaseFormatter(options()) + with pytest.raises(NotImplementedError): + formatter.format('foo') + + +def test_show_source_returns_nothing_when_not_showing_source(): + """Ensure we return nothing when users want nothing.""" + formatter = base.BaseFormatter(options(show_source=False)) + assert formatter.show_source( + style_guide.Error('A000', 'file.py', 1, 1, 'error text', 'line') + ) is None + + +@pytest.mark.parametrize('line, column', [ + ('x=1\n', 2), + (' x=(1\n +2)\n', 5), + # TODO(sigmavirus24): Add more examples +]) +def test_show_source_updates_physical_line_appropriately(line, column): + """Ensure the error column is appropriately indicated.""" + formatter = base.BaseFormatter(options(show_source=True)) + error = style_guide.Error('A000', 'file.py', 1, column, 'error', line) + output = formatter.show_source(error) + _, pointer = output.rsplit('\n', 1) + assert pointer.count(' ') == column + + +def test_write_uses_an_output_file(): + """Verify that we use the output file when it's present.""" + line = 'Something to write' + source = 'source' + filemock = mock.Mock() + + formatter = base.BaseFormatter(options()) + formatter.output_fd = filemock + formatter.write(line, source) + + assert filemock.write.called is True + assert filemock.write.call_count == 2 + assert filemock.write.mock_calls == [ + mock.call(line + formatter.newline), + mock.call(source + formatter.newline), + ] + + +@mock.patch('flake8.formatting.base.print') +def test_write_uses_print(print_function): + """Verify that we use the print function without an output file.""" + line = 'Something to write' + source = 'source' + + formatter = base.BaseFormatter(options()) + formatter.write(line, source) + + assert print_function.called is True + assert print_function.call_count == 2 + assert print_function.mock_calls == [ + mock.call(line), + mock.call(source), + ] + + +class AfterInitFormatter(base.BaseFormatter): + """Subclass for testing after_init.""" + + def after_init(self): + """Define method to verify operation.""" + self.post_initialized = True + + +def test_after_init_is_always_called(): + """Verify after_init is called.""" + formatter = AfterInitFormatter(options()) + assert getattr(formatter, 'post_initialized') is True + + +class FormatFormatter(base.BaseFormatter): + """Subclass for testing format.""" + + def format(self, error): + """Define method to verify operation.""" + return repr(error) + + +def test_handle_formats_the_error(): + """Verify that a formatter will call format from handle.""" + formatter = FormatFormatter(options(show_source=False)) + filemock = formatter.output_fd = mock.Mock() + error = style_guide.Error( + code='A001', + filename='example.py', + line_number=1, + column_number=1, + text='Fake error', + physical_line='a = 1', + ) + + formatter.handle(error) + + filemock.write.assert_called_once_with(repr(error) + '\n') diff --git a/tests/unit/test_checker_manager.py b/tests/unit/test_checker_manager.py new file mode 100644 index 0000000..8240723 --- /dev/null +++ b/tests/unit/test_checker_manager.py @@ -0,0 +1,59 @@ +"""Tests for the Manager object for FileCheckers.""" +import errno + +import mock +import pytest + +from flake8 import checker + + +def style_guide_mock(**kwargs): + """Create a mock StyleGuide object.""" + kwargs.setdefault('diff', False) + kwargs.setdefault('jobs', '4') + style_guide = mock.Mock() + style_guide.options = mock.Mock(**kwargs) + return style_guide + + +def test_oserrors_cause_serial_fall_back(): + """Verify that OSErrors will cause the Manager to fallback to serial.""" + err = OSError(errno.ENOSPC, 'Ominous message about spaceeeeee') + style_guide = style_guide_mock() + with mock.patch('multiprocessing.Queue', side_effect=err): + manager = checker.Manager(style_guide, [], []) + assert manager.using_multiprocessing is False + + +def test_oserrors_are_reraised(): + """Verify that OSErrors will cause the Manager to fallback to serial.""" + err = OSError(errno.EAGAIN, 'Ominous message') + style_guide = style_guide_mock() + with mock.patch('multiprocessing.Queue', side_effect=err): + with pytest.raises(OSError): + checker.Manager(style_guide, [], []) + + +def test_multiprocessing_is_disabled(): + """Verify not being able to import multiprocessing forces jobs to 0.""" + style_guide = style_guide_mock() + with mock.patch('flake8.checker.multiprocessing', None): + manager = checker.Manager(style_guide, [], []) + assert manager.jobs == 0 + + +def test_make_checkers(): + """Verify that we create a list of FileChecker instances.""" + style_guide = style_guide_mock() + files = ['file1', 'file2'] + with mock.patch('flake8.checker.multiprocessing', None): + manager = checker.Manager(style_guide, files, []) + + with mock.patch('flake8.utils.filenames_from') as filenames_from: + filenames_from.side_effect = [['file1'], ['file2']] + with mock.patch('flake8.utils.fnmatch', return_value=True): + with mock.patch('flake8.processor.FileProcessor'): + manager.make_checkers() + + for file_checker in manager.checkers: + assert file_checker.filename in files diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py new file mode 100644 index 0000000..2e1a1e5 --- /dev/null +++ b/tests/unit/test_config_file_finder.py @@ -0,0 +1,126 @@ +"""Tests for the ConfigFileFinder.""" +import configparser +import os +import sys + +import mock +import pytest + +from flake8.options import config + +CLI_SPECIFIED_FILEPATH = 'tests/fixtures/config_files/cli-specified.ini' +BROKEN_CONFIG_PATH = 'tests/fixtures/config_files/broken.ini' + + +def test_uses_default_args(): + """Show that we default the args value.""" + finder = config.ConfigFileFinder('flake8', None, []) + assert finder.parent == os.path.abspath('.') + + +@pytest.mark.parametrize('platform,is_windows', [ + ('win32', True), + ('linux', False), + ('darwin', False), +]) +def test_windows_detection(platform, is_windows): + """Verify we detect Windows to the best of our knowledge.""" + with mock.patch.object(sys, 'platform', platform): + finder = config.ConfigFileFinder('flake8', None, []) + assert finder.is_windows is is_windows + + +def test_cli_config(): + """Verify opening and reading the file specified via the cli.""" + cli_filepath = CLI_SPECIFIED_FILEPATH + finder = config.ConfigFileFinder('flake8', None, []) + + parsed_config = finder.cli_config(cli_filepath) + assert parsed_config.has_section('flake8') + + +@pytest.mark.parametrize('args,expected', [ + # No arguments, common prefix of abspath('.') + ([], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath('.flake8')]), + # Common prefix of "flake8/" + (['flake8/options', 'flake8/'], + [os.path.abspath('flake8/setup.cfg'), + os.path.abspath('flake8/tox.ini'), + os.path.abspath('flake8/.flake8'), + os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath('.flake8')]), + # Common prefix of "flake8/options" + (['flake8/options', 'flake8/options/sub'], + [os.path.abspath('flake8/options/setup.cfg'), + os.path.abspath('flake8/options/tox.ini'), + os.path.abspath('flake8/options/.flake8'), + os.path.abspath('flake8/setup.cfg'), + os.path.abspath('flake8/tox.ini'), + os.path.abspath('flake8/.flake8'), + os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath('.flake8')]), +]) +def test_generate_possible_local_files(args, expected): + """Verify generation of all possible config paths.""" + finder = config.ConfigFileFinder('flake8', args, []) + + assert (list(finder.generate_possible_local_files()) == + expected) + + +@pytest.mark.parametrize('args,extra_config_files,expected', [ + # No arguments, common prefix of abspath('.') + ([], + [], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini')]), + # Common prefix of "flake8/" + (['flake8/options', 'flake8/'], + [], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini')]), + # Common prefix of "flake8/options" + (['flake8/options', 'flake8/options/sub'], + [], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini')]), + # Common prefix of "flake8/" with extra config files specified + (['flake8/'], + [CLI_SPECIFIED_FILEPATH], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath(CLI_SPECIFIED_FILEPATH)]), + # Common prefix of "flake8/" with missing extra config files specified + (['flake8/'], + [CLI_SPECIFIED_FILEPATH, + 'tests/fixtures/config_files/missing.ini'], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath(CLI_SPECIFIED_FILEPATH)]), +]) +def test_local_config_files(args, extra_config_files, expected): + """Verify discovery of local config files.""" + finder = config.ConfigFileFinder('flake8', args, extra_config_files) + + assert list(finder.local_config_files()) == expected + + +def test_local_configs(): + """Verify we return a ConfigParser.""" + finder = config.ConfigFileFinder('flake8', None, []) + + assert isinstance(finder.local_configs(), configparser.RawConfigParser) + + +@pytest.mark.parametrize('files', [ + [BROKEN_CONFIG_PATH], + [CLI_SPECIFIED_FILEPATH, BROKEN_CONFIG_PATH], +]) +def test_read_config_catches_broken_config_files(files): + """Verify that we do not allow the exception to bubble up.""" + assert config.ConfigFileFinder._read_config(files)[1] == [] diff --git a/tests/unit/test_file_processor.py b/tests/unit/test_file_processor.py new file mode 100644 index 0000000..db17c2e --- /dev/null +++ b/tests/unit/test_file_processor.py @@ -0,0 +1,293 @@ +"""Tests for the FileProcessor class.""" +import ast +import optparse +import tokenize + +from flake8 import processor + +import mock +import pytest + + +def options_from(**kwargs): + """Generate a Values instances with our kwargs.""" + kwargs.setdefault('hang_closing', True) + kwargs.setdefault('max_line_length', 79) + kwargs.setdefault('verbose', False) + return optparse.Values(kwargs) + + +def test_read_lines_splits_lines(): + """Verify that read_lines splits the lines of the file.""" + file_processor = processor.FileProcessor(__file__, options_from()) + lines = file_processor.lines + assert len(lines) > 5 + assert '"""Tests for the FileProcessor class."""\n' in lines + + +@pytest.mark.parametrize('first_line', [ + '\xEF\xBB\xBF"""Module docstring."""\n', + u'\uFEFF"""Module docstring."""\n', +]) +def test_strip_utf_bom(first_line): + r"""Verify that we strip '\xEF\xBB\xBF' from the first line.""" + lines = [first_line] + file_processor = processor.FileProcessor('-', options_from(), lines[:]) + assert file_processor.lines != lines + assert file_processor.lines[0] == '"""Module docstring."""\n' + + +@pytest.mark.parametrize('lines, expected', [ + (['\xEF\xBB\xBF"""Module docstring."""\n'], False), + ([u'\uFEFF"""Module docstring."""\n'], False), + (['#!/usr/bin/python', '# flake8 is great', 'a = 1'], False), + (['#!/usr/bin/python', '# flake8: noqa', 'a = 1'], True), + (['# flake8: noqa', '#!/usr/bin/python', 'a = 1'], True), + (['#!/usr/bin/python', 'a = 1', '# flake8: noqa'], True), +]) +def test_should_ignore_file(lines, expected): + """Verify that we ignore a file if told to.""" + file_processor = processor.FileProcessor('-', options_from(), lines) + assert file_processor.should_ignore_file() is expected + + +@mock.patch('flake8.utils.stdin_get_value') +def test_read_lines_from_stdin(stdin_get_value): + """Verify that we use our own utility function to retrieve stdin.""" + stdin_value = mock.Mock() + stdin_value.splitlines.return_value = [] + stdin_get_value.return_value = stdin_value + file_processor = processor.FileProcessor('-', options_from()) + stdin_get_value.assert_called_once_with() + stdin_value.splitlines.assert_called_once_with(True) + + +@mock.patch('flake8.utils.stdin_get_value') +def test_read_lines_sets_filename_attribute(stdin_get_value): + """Verify that we update the filename attribute.""" + stdin_value = mock.Mock() + stdin_value.splitlines.return_value = [] + stdin_get_value.return_value = stdin_value + file_processor = processor.FileProcessor('-', options_from()) + assert file_processor.filename == 'stdin' + + +def test_line_for(): + """Verify we grab the correct line from the cached lines.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'Line 1', + 'Line 2', + 'Line 3', + ]) + + for i in range(1, 4): + assert file_processor.line_for(i) == 'Line {0}'.format(i) + + +def test_next_line(): + """Verify we update the file_processor state for each new line.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'Line 1', + 'Line 2', + 'Line 3', + ]) + + for i in range(1, 4): + assert file_processor.next_line() == 'Line {}'.format(i) + assert file_processor.line_number == i + + +@pytest.mark.parametrize('error_code, line, expected_indent_char', [ + ('E101', '\t\ta = 1', '\t'), + ('E101', ' a = 1', ' '), + ('W101', 'frobulate()', None), + ('F821', 'class FizBuz:', None), +]) +def test_check_physical_error(error_code, line, expected_indent_char): + """Verify we update the indet char for the appropriate error code.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'Line 1', + ]) + + file_processor.check_physical_error(error_code, line) + assert file_processor.indent_char == expected_indent_char + + +@pytest.mark.parametrize('params, args, expected_kwargs', [ + (['blank_before', 'blank_lines'], None, {'blank_before': 0, + 'blank_lines': 0}), + (['noqa', 'fake'], {'fake': 'foo'}, {'noqa': False, 'fake': 'foo'}), + (['blank_before', 'blank_lines', 'noqa'], + {'blank_before': 10, 'blank_lines': 5, 'noqa': True}, + {'blank_before': 10, 'blank_lines': 5, 'noqa': True}), + ([], {'fake': 'foo'}, {'fake': 'foo'}), +]) +def test_keyword_arguments_for(params, args, expected_kwargs): + """Verify the keyword args are generated properly.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'Line 1', + ]) + kwargs_for = file_processor.keyword_arguments_for + + assert kwargs_for(params, args) == expected_kwargs + + +def test_keyword_arguments_for_does_not_handle_attribute_errors(): + """Verify we re-raise AttributeErrors.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'Line 1', + ]) + + with pytest.raises(AttributeError): + file_processor.keyword_arguments_for(['fake']) + + +@pytest.mark.parametrize('unsplit_line, expected_lines', [ + ('line', []), + ('line 1\n', ['line 1']), + ('line 1\nline 2\n', ['line 1', 'line 2']), + ('line 1\n\nline 2\n', ['line 1', '', 'line 2']), +]) +def test_split_line(unsplit_line, expected_lines): + """Verify the token line spliting.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'Line 1', + ]) + + actual_lines = list(file_processor.split_line((1, unsplit_line))) + assert expected_lines == actual_lines + + assert len(actual_lines) == file_processor.line_number + + +def test_build_ast(): + """Verify the logic for how we build an AST for plugins.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'a = 1\n' + ]) + + module = file_processor.build_ast() + assert isinstance(module, ast.Module) + + +def test_next_logical_line_updates_the_previous_logical_line(): + """Verify that we update our tracking of the previous logical line.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'a = 1\n' + ]) + + file_processor.indent_level = 1 + file_processor.logical_line = 'a = 1' + assert file_processor.previous_logical == '' + assert file_processor.previous_indent_level is 0 + + file_processor.next_logical_line() + assert file_processor.previous_logical == 'a = 1' + assert file_processor.previous_indent_level == 1 + + +def test_visited_new_blank_line(): + """Verify we update the number of blank lines seen.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'a = 1\n' + ]) + + assert file_processor.blank_lines == 0 + file_processor.visited_new_blank_line() + assert file_processor.blank_lines == 1 + + +def test_inside_multiline(): + """Verify we update the line number and reset multiline.""" + file_processor = processor.FileProcessor('-', options_from(), lines=[ + 'a = 1\n' + ]) + + assert file_processor.multiline is False + assert file_processor.line_number == 0 + with file_processor.inside_multiline(10): + assert file_processor.multiline is True + assert file_processor.line_number == 10 + + assert file_processor.multiline is False + + +@pytest.mark.parametrize('string, expected', [ + ('""', '""'), + ("''", "''"), + ('"a"', '"x"'), + ("'a'", "'x'"), + ('"x"', '"x"'), + ("'x'", "'x'"), + ('"abcdef"', '"xxxxxx"'), + ("'abcdef'", "'xxxxxx'"), + ('""""""', '""""""'), + ("''''''", "''''''"), + ('"""a"""', '"""x"""'), + ("'''a'''", "'''x'''"), + ('"""x"""', '"""x"""'), + ("'''x'''", "'''x'''"), + ('"""abcdef"""', '"""xxxxxx"""'), + ("'''abcdef'''", "'''xxxxxx'''"), + ('"""xxxxxx"""', '"""xxxxxx"""'), + ("'''xxxxxx'''", "'''xxxxxx'''"), +]) +def test_mutate_string(string, expected): + """Verify we appropriately mutate the string to sanitize it.""" + actual = processor.mutate_string(string) + assert expected == actual + + +@pytest.mark.parametrize('string, expected', [ + (' ', 4), + (' ', 6), + ('\t', 8), + ('\t\t', 16), + (' \t', 8), + (' \t', 16), +]) +def test_expand_indent(string, expected): + """Verify we correctly measure the amount of indentation.""" + actual = processor.expand_indent(string) + assert expected == actual + + +@pytest.mark.parametrize('token, log_string', [ + [(tokenize.COMMENT, '# this is a comment', + (1, 0), # (start_row, start_column) + (1, 19), # (end_ro, end_column) + '# this is a comment',), + "l.1\t[:19]\tCOMMENT\t'# this is a comment'"], + [(tokenize.COMMENT, '# this is a comment', + (1, 5), # (start_row, start_column) + (1, 19), # (end_ro, end_column) + '# this is a comment',), + "l.1\t[5:19]\tCOMMENT\t'# this is a comment'"], + [(tokenize.COMMENT, '# this is a comment', + (1, 0), # (start_row, start_column) + (2, 19), # (end_ro, end_column) + '# this is a comment',), + "l.1\tl.2\tCOMMENT\t'# this is a comment'"], +]) +def test_log_token(token, log_string): + """Verify we use the log object passed in.""" + LOG = mock.Mock() + processor.log_token(LOG, token) + LOG.log.assert_called_once_with( + 5, # flake8._EXTRA_VERBOSE + log_string, + ) + + +@pytest.mark.parametrize('current_count, token_text, expected', [ + (None, '(', 1), + (None, '[', 1), + (None, '{', 1), + (1, ')', 0), + (1, ']', 0), + (1, '}', 0), + (10, '+', 10), +]) +def test_count_parentheses(current_count, token_text, expected): + """Verify our arithmetic is correct.""" + assert processor.count_parentheses(current_count, token_text) == expected diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py new file mode 100644 index 0000000..eb57802 --- /dev/null +++ b/tests/unit/test_merged_config_parser.py @@ -0,0 +1,206 @@ +"""Unit tests for flake8.options.config.MergedConfigParser.""" +import os + +import mock +import pytest + +from flake8.options import config +from flake8.options import manager + + +@pytest.fixture +def optmanager(): + """Generate an OptionManager with simple values.""" + return manager.OptionManager(prog='flake8', version='3.0.0a1') + + +@pytest.mark.parametrize('args,extra_config_files', [ + (None, None), + (None, []), + (None, ['foo.ini']), + ('flake8/', []), + ('flake8/', ['foo.ini']), +]) +def test_creates_its_own_config_file_finder(args, extra_config_files, + optmanager): + """Verify we create a ConfigFileFinder correctly.""" + class_path = 'flake8.options.config.ConfigFileFinder' + with mock.patch(class_path) as ConfigFileFinder: + parser = config.MergedConfigParser( + option_manager=optmanager, + extra_config_files=extra_config_files, + args=args, + ) + + assert parser.program_name == 'flake8' + ConfigFileFinder.assert_called_once_with( + 'flake8', + args, + extra_config_files or [], + ) + + +def test_parse_cli_config(optmanager): + """Parse the specified config file as a cli config file.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + + parsed_config = parser.parse_cli_config( + 'tests/fixtures/config_files/cli-specified.ini' + ) + assert parsed_config == { + 'ignore': ['E123', 'W234', 'E111'], + 'exclude': [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] + } + + +@pytest.mark.parametrize('filename,is_configured_by', [ + ('tests/fixtures/config_files/cli-specified.ini', True), + ('tests/fixtures/config_files/no-flake8-section.ini', False), +]) +def test_is_configured_by(filename, is_configured_by, optmanager): + """Verify the behaviour of the is_configured_by method.""" + parsed_config, _ = config.ConfigFileFinder._read_config(filename) + parser = config.MergedConfigParser(optmanager) + + assert parser.is_configured_by(parsed_config) is is_configured_by + + +def test_parse_user_config(optmanager): + """Verify parsing of user config files.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + + with mock.patch.object(parser.config_finder, 'user_config_file') as usercf: + usercf.return_value = 'tests/fixtures/config_files/cli-specified.ini' + parsed_config = parser.parse_user_config() + + assert parsed_config == { + 'ignore': ['E123', 'W234', 'E111'], + 'exclude': [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] + } + + +def test_parse_local_config(optmanager): + """Verify parsing of local config files.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + config_finder = parser.config_finder + + with mock.patch.object(config_finder, 'local_config_files') as localcfs: + localcfs.return_value = [ + 'tests/fixtures/config_files/cli-specified.ini' + ] + parsed_config = parser.parse_local_config() + + assert parsed_config == { + 'ignore': ['E123', 'W234', 'E111'], + 'exclude': [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] + } + + +def test_merge_user_and_local_config(optmanager): + """Verify merging of parsed user and local config files.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + optmanager.add_option('--select', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + config_finder = parser.config_finder + + with mock.patch.object(config_finder, 'local_config_files') as localcfs: + localcfs.return_value = [ + 'tests/fixtures/config_files/local-config.ini' + ] + with mock.patch.object(config_finder, + 'user_config_file') as usercf: + usercf.return_value = ('tests/fixtures/config_files/' + 'user-config.ini') + parsed_config = parser.merge_user_and_local_config() + + assert parsed_config == { + 'exclude': [ + os.path.abspath('docs/') + ], + 'ignore': ['D203'], + 'select': ['E', 'W', 'F'], + } + + +@mock.patch('flake8.options.config.ConfigFileFinder') +def test_parse_isolates_config(ConfigFileManager, optmanager): + """Verify behaviour of the parse method with isolated=True.""" + parser = config.MergedConfigParser(optmanager) + + assert parser.parse(isolated=True) == {} + assert parser.config_finder.local_configs.called is False + assert parser.config_finder.user_config.called is False + + +@mock.patch('flake8.options.config.ConfigFileFinder') +def test_parse_uses_cli_config(ConfigFileManager, optmanager): + """Verify behaviour of the parse method with a specified config.""" + parser = config.MergedConfigParser(optmanager) + + parser.parse(cli_config='foo.ini') + parser.config_finder.cli_config.assert_called_once_with('foo.ini') + + +@pytest.mark.parametrize('config_fixture_path', [ + 'tests/fixtures/config_files/cli-specified.ini', + 'tests/fixtures/config_files/cli-specified-with-inline-comments.ini', + 'tests/fixtures/config_files/cli-specified-without-inline-comments.ini', +]) +def test_parsed_configs_are_equivalent(optmanager, config_fixture_path): + """Verify the each file matches the expected parsed output. + + This is used to ensure our documented behaviour does not regress. + """ + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + config_finder = parser.config_finder + + with mock.patch.object(config_finder, 'local_config_files') as localcfs: + localcfs.return_value = [config_fixture_path] + with mock.patch.object(config_finder, + 'user_config_file') as usercf: + usercf.return_value = [] + parsed_config = parser.merge_user_and_local_config() + + assert parsed_config['ignore'] == ['E123', 'W234', 'E111'] + assert parsed_config['exclude'] == [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] diff --git a/tests/unit/test_notifier.py b/tests/unit/test_notifier.py new file mode 100644 index 0000000..6a162cf --- /dev/null +++ b/tests/unit/test_notifier.py @@ -0,0 +1,54 @@ +"""Unit tests for the Notifier object.""" +import pytest + +from flake8.plugins import notifier + + +class _Listener(object): + def __init__(self, error_code): + self.error_code = error_code + self.was_notified = False + + def notify(self, error_code, *args, **kwargs): + assert error_code.startswith(self.error_code) + self.was_notified = True + + +class TestNotifier(object): + """Notifier unit tests.""" + + @pytest.fixture(autouse=True) + def setup(self): + """Set up each TestNotifier instance.""" + self.notifier = notifier.Notifier() + self.listener_map = {} + + def add_listener(error_code): + listener = _Listener(error_code) + self.listener_map[error_code] = listener + self.notifier.register_listener(error_code, listener) + + for i in range(10): + add_listener('E{0}'.format(i)) + for j in range(30): + add_listener('E{0}{1:02d}'.format(i, j)) + + def test_notify(self): + """Show that we notify a specific error code.""" + self.notifier.notify('E111', 'extra', 'args') + assert self.listener_map['E111'].was_notified is True + assert self.listener_map['E1'].was_notified is True + + @pytest.mark.parametrize('code', ['W123', 'W12', 'W1', 'W']) + def test_no_listeners_for(self, code): + """Show that we return an empty list of listeners.""" + assert list(self.notifier.listeners_for(code)) == [] + + @pytest.mark.parametrize('code,expected', [ + ('E101', ['E101', 'E1']), + ('E211', ['E211', 'E2']), + ]) + def test_listeners_for(self, code, expected): + """Verify that we retrieve the correct listeners.""" + assert ([l.error_code for l in self.notifier.listeners_for(code)] == + expected) diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py new file mode 100644 index 0000000..67e2255 --- /dev/null +++ b/tests/unit/test_option.py @@ -0,0 +1,68 @@ +"""Unit tests for flake8.options.manager.Option.""" +import mock +import pytest + +from flake8.options import manager + + +def test_to_optparse(): + """Test conversion to an optparse.Option class.""" + opt = manager.Option( + short_option_name='-t', + long_option_name='--test', + action='count', + parse_from_config=True, + normalize_paths=True, + ) + assert opt.normalize_paths is True + assert opt.parse_from_config is True + + optparse_opt = opt.to_optparse() + assert not hasattr(optparse_opt, 'parse_from_config') + assert not hasattr(optparse_opt, 'normalize_paths') + assert optparse_opt.action == 'count' + + +@mock.patch('optparse.Option') +def test_to_optparse_creates_an_option_as_we_expect(Option): + """Show that we pass all keyword args to optparse.Option.""" + opt = manager.Option('-t', '--test', action='count') + opt.to_optparse() + option_kwargs = { + 'action': 'count', + 'default': None, + 'type': None, + 'dest': 'test', + 'nargs': None, + 'const': None, + 'choices': None, + 'callback': None, + 'callback_args': None, + 'callback_kwargs': None, + 'help': None, + 'metavar': None, + } + + Option.assert_called_once_with( + '-t', '--test', **option_kwargs + ) + + +def test_config_name_generation(): + """Show that we generate the config name deterministically.""" + opt = manager.Option(long_option_name='--some-very-long-option-name', + parse_from_config=True) + + assert opt.config_name == 'some_very_long_option_name' + + +def test_config_name_needs_long_option_name(): + """Show that we error out if the Option should be parsed from config.""" + with pytest.raises(ValueError): + manager.Option('-s', parse_from_config=True) + + +def test_dest_is_not_overridden(): + """Show that we do not override custom destinations.""" + opt = manager.Option('-s', '--short', dest='something_not_short') + assert opt.dest == 'something_not_short' diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py new file mode 100644 index 0000000..53e8bf1 --- /dev/null +++ b/tests/unit/test_option_manager.py @@ -0,0 +1,194 @@ +"""Unit tests for flake.options.manager.OptionManager.""" +import optparse +import os + +import pytest + +from flake8.options import manager + +TEST_VERSION = '3.0.0b1' + + +@pytest.fixture +def optmanager(): + """Generate a simple OptionManager with default test arguments.""" + return manager.OptionManager(prog='flake8', version=TEST_VERSION) + + +def test_option_manager_creates_option_parser(optmanager): + """Verify that a new manager creates a new parser.""" + assert optmanager.parser is not None + assert isinstance(optmanager.parser, optparse.OptionParser) is True + + +def test_add_option_short_option_only(optmanager): + """Verify the behaviour of adding a short-option only.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('-s', help='Test short opt') + assert optmanager.options[0].short_option_name == '-s' + + +def test_add_option_long_option_only(optmanager): + """Verify the behaviour of adding a long-option only.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--long', help='Test long opt') + assert optmanager.options[0].short_option_name is None + assert optmanager.options[0].long_option_name == '--long' + + +def test_add_short_and_long_option_names(optmanager): + """Verify the behaviour of using both short and long option names.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('-b', '--both', help='Test both opts') + assert optmanager.options[0].short_option_name == '-b' + assert optmanager.options[0].long_option_name == '--both' + + +def test_add_option_with_custom_args(optmanager): + """Verify that add_option handles custom Flake8 parameters.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--parse', parse_from_config=True) + optmanager.add_option('--commas', comma_separated_list=True) + optmanager.add_option('--files', normalize_paths=True) + + attrs = ['parse_from_config', 'comma_separated_list', 'normalize_paths'] + for option, attr in zip(optmanager.options, attrs): + assert getattr(option, attr) is True + + +def test_parse_args_normalize_path(optmanager): + """Show that parse_args handles path normalization.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--config', normalize_paths=True) + + options, args = optmanager.parse_args(['--config', '../config.ini']) + assert options.config == os.path.abspath('../config.ini') + + +def test_parse_args_handles_comma_separated_defaults(optmanager): + """Show that parse_args handles defaults that are comma-separated.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--exclude', default='E123,W234', + comma_separated_list=True) + + options, args = optmanager.parse_args([]) + assert options.exclude == ['E123', 'W234'] + + +def test_parse_args_handles_comma_separated_lists(optmanager): + """Show that parse_args handles user-specified comma-separated lists.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--exclude', default='E123,W234', + comma_separated_list=True) + + options, args = optmanager.parse_args(['--exclude', 'E201,W111,F280']) + assert options.exclude == ['E201', 'W111', 'F280'] + + +def test_parse_args_normalize_paths(optmanager): + """Verify parse_args normalizes a comma-separated list of paths.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--extra-config', normalize_paths=True, + comma_separated_list=True) + + options, args = optmanager.parse_args([ + '--extra-config', '../config.ini,tox.ini,flake8/some-other.cfg' + ]) + assert options.extra_config == [ + os.path.abspath('../config.ini'), + 'tox.ini', + os.path.abspath('flake8/some-other.cfg'), + ] + + +def test_format_plugin(): + """Verify that format_plugin turns a tuple into a dictionary.""" + plugin = manager.OptionManager.format_plugin(('Testing', '0.0.0')) + assert plugin['name'] == 'Testing' + assert plugin['version'] == '0.0.0' + + +def test_generate_versions(optmanager): + """Verify a comma-separated string is generated of registered plugins.""" + optmanager.registered_plugins = [ + ('Testing 100', '0.0.0'), + ('Testing 101', '0.0.0'), + ('Testing 300', '0.0.0'), + ] + assert (optmanager.generate_versions() == + 'Testing 100: 0.0.0, Testing 101: 0.0.0, Testing 300: 0.0.0') + + +def test_generate_versions_with_format_string(optmanager): + """Verify a comma-separated string is generated of registered plugins.""" + optmanager.registered_plugins.update([ + ('Testing', '0.0.0'), + ('Testing', '0.0.0'), + ('Testing', '0.0.0'), + ]) + assert ( + optmanager.generate_versions() == 'Testing: 0.0.0' + ) + + +def test_update_version_string(optmanager): + """Verify we update the version string idempotently.""" + assert optmanager.version == TEST_VERSION + assert optmanager.parser.version == TEST_VERSION + + optmanager.registered_plugins = [ + ('Testing 100', '0.0.0'), + ('Testing 101', '0.0.0'), + ('Testing 300', '0.0.0'), + ] + + optmanager.update_version_string() + + assert optmanager.version == TEST_VERSION + assert (optmanager.parser.version == TEST_VERSION + ' (' + 'Testing 100: 0.0.0, Testing 101: 0.0.0, Testing 300: 0.0.0)') + + +def test_generate_epilog(optmanager): + """Verify how we generate the epilog for help text.""" + assert optmanager.parser.epilog is None + + optmanager.registered_plugins = [ + ('Testing 100', '0.0.0'), + ('Testing 101', '0.0.0'), + ('Testing 300', '0.0.0'), + ] + + expected_value = ( + 'Installed plugins: Testing 100: 0.0.0, Testing 101: 0.0.0, Testing' + ' 300: 0.0.0' + ) + + optmanager.generate_epilog() + assert optmanager.parser.epilog == expected_value + + +def test_extend_default_ignore(optmanager): + """Verify that we update the extended default ignore list.""" + assert optmanager.extended_default_ignore == set() + + optmanager.extend_default_ignore(['T100', 'T101', 'T102']) + assert optmanager.extended_default_ignore == set(['T100', + 'T101', + 'T102']) diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py new file mode 100644 index 0000000..f69bc05 --- /dev/null +++ b/tests/unit/test_plugin.py @@ -0,0 +1,153 @@ +"""Tests for flake8.plugins.manager.Plugin.""" +import optparse + +import mock +import pytest + +from flake8 import exceptions +from flake8.plugins import manager + + +def test_load_plugin_fallsback_on_old_setuptools(): + """Verify we fallback gracefully to on old versions of setuptools.""" + entry_point = mock.Mock(spec=['load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin() + entry_point.load.assert_called_once_with(require=False) + + +def test_load_plugin_avoids_deprecated_entry_point_methods(): + """Verify we use the preferred methods on new versions of setuptools.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin(verify_requirements=True) + assert entry_point.load.called is False + entry_point.require.assert_called_once_with() + entry_point.resolve.assert_called_once_with() + + +def test_load_plugin_is_idempotent(): + """Verify we use the preferred methods on new versions of setuptools.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin(verify_requirements=True) + plugin.load_plugin(verify_requirements=True) + plugin.load_plugin() + assert entry_point.load.called is False + entry_point.require.assert_called_once_with() + entry_point.resolve.assert_called_once_with() + + +def test_load_plugin_only_calls_require_when_verifying_requirements(): + """Verify we do not call require when verify_requirements is False.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin() + assert entry_point.load.called is False + assert entry_point.require.called is False + entry_point.resolve.assert_called_once_with() + + +def test_load_plugin_catches_and_reraises_exceptions(): + """Verify we raise our own FailedToLoadPlugin.""" + entry_point = mock.Mock(spec=['require', 'resolve']) + entry_point.resolve.side_effect = ValueError('Test failure') + plugin = manager.Plugin('T000', entry_point) + + with pytest.raises(exceptions.FailedToLoadPlugin): + plugin.load_plugin() + + +def test_plugin_property_loads_plugin_on_first_use(): + """Verify that we load our plugin when we first try to use it.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + assert plugin.plugin is not None + entry_point.resolve.assert_called_once_with() + + +def test_execute_calls_plugin_with_passed_arguments(): + """Verify that we pass arguments directly to the plugin.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock() + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + plugin.execute('arg1', 'arg2', kwarg1='value1', kwarg2='value2') + plugin_obj.assert_called_once_with( + 'arg1', 'arg2', kwarg1='value1', kwarg2='value2' + ) + + # Extra assertions + assert entry_point.load.called is False + assert entry_point.require.called is False + assert entry_point.resolve.called is False + + +def test_version_proxies_to_the_plugin(): + """Verify that we pass arguments directly to the plugin.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['version']) + plugin_obj.version = 'a.b.c' + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + assert plugin.version == 'a.b.c' + + +def test_register_options(): + """Verify we call add_options on the plugin only if it exists.""" + # Set up our mocks and Plugin object + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['name', 'version', 'add_options', + 'parse_options']) + option_manager = mock.Mock(spec=['register_plugin']) + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + # Call the method we're testing. + plugin.register_options(option_manager) + + # Assert that we call add_options + plugin_obj.add_options.assert_called_once_with(option_manager) + + +def test_register_options_checks_plugin_for_method(): + """Verify we call add_options on the plugin only if it exists.""" + # Set up our mocks and Plugin object + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['name', 'version', 'parse_options']) + option_manager = mock.Mock(spec=['register_plugin']) + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + # Call the method we're testing. + plugin.register_options(option_manager) + + # Assert that we register the plugin + assert option_manager.register_plugin.called is False + + +def test_provide_options(): + """Verify we call add_options on the plugin only if it exists.""" + # Set up our mocks and Plugin object + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['name', 'version', 'add_options', + 'parse_options']) + option_values = optparse.Values({'enable_extensions': []}) + option_manager = mock.Mock() + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + # Call the method we're testing. + plugin.provide_options(option_manager, option_values, None) + + # Assert that we call add_options + plugin_obj.parse_options.assert_called_once_with( + option_manager, option_values, None + ) diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py new file mode 100644 index 0000000..8991b96 --- /dev/null +++ b/tests/unit/test_plugin_manager.py @@ -0,0 +1,50 @@ +"""Tests for flake8.plugins.manager.PluginManager.""" +import mock + +from flake8.plugins import manager + + +def create_entry_point_mock(name): + """Create a mocked EntryPoint.""" + ep = mock.Mock(spec=['name']) + ep.name = name + return ep + + +@mock.patch('pkg_resources.iter_entry_points') +def test_calls_pkg_resources_on_instantiation(iter_entry_points): + """Verify that we call iter_entry_points when we create a manager.""" + iter_entry_points.return_value = [] + manager.PluginManager(namespace='testing.pkg_resources') + + iter_entry_points.assert_called_once_with('testing.pkg_resources') + + +@mock.patch('pkg_resources.iter_entry_points') +def test_calls_pkg_resources_creates_plugins_automaticaly(iter_entry_points): + """Verify that we create Plugins on instantiation.""" + iter_entry_points.return_value = [ + create_entry_point_mock('T100'), + create_entry_point_mock('T200'), + ] + plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') + + iter_entry_points.assert_called_once_with('testing.pkg_resources') + assert 'T100' in plugin_mgr.plugins + assert 'T200' in plugin_mgr.plugins + assert isinstance(plugin_mgr.plugins['T100'], manager.Plugin) + assert isinstance(plugin_mgr.plugins['T200'], manager.Plugin) + + +@mock.patch('pkg_resources.iter_entry_points') +def test_handles_mapping_functions_across_plugins(iter_entry_points): + """Verify we can use the PluginManager call functions on all plugins.""" + entry_point_mocks = [ + create_entry_point_mock('T100'), + create_entry_point_mock('T200'), + ] + iter_entry_points.return_value = entry_point_mocks + plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') + plugins = [plugin_mgr.plugins[name] for name in plugin_mgr.names] + + assert list(plugin_mgr.map(lambda x: x)) == plugins diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py new file mode 100644 index 0000000..186c2e3 --- /dev/null +++ b/tests/unit/test_plugin_type_manager.py @@ -0,0 +1,228 @@ +"""Tests for flake8.plugins.manager.PluginTypeManager.""" +import collections + +import mock +import pytest + +from flake8 import exceptions +from flake8.plugins import manager + +TEST_NAMESPACE = "testing.plugin-type-manager" + + +def create_plugin_mock(raise_exception=False): + """Create an auto-spec'd mock of a flake8 Plugin.""" + plugin = mock.create_autospec(manager.Plugin, instance=True) + if raise_exception: + plugin.load_plugin.side_effect = exceptions.FailedToLoadPlugin( + plugin=mock.Mock(name='T101'), + exception=ValueError('Test failure'), + ) + return plugin + + +def create_mapping_manager_mock(plugins): + """Create a mock for the PluginManager.""" + # Have a function that will actually call the method underneath + def fake_map(func): + for plugin in plugins: + yield func(plugin) + + # Mock out the PluginManager instance + manager_mock = mock.Mock(spec=['map']) + # Replace the map method + manager_mock.map = fake_map + return manager_mock + + +def create_manager_with_plugins(plugins): + """Create a fake PluginManager with a plugins dictionary.""" + manager_mock = mock.create_autospec(manager.PluginManager) + manager_mock.plugins = plugins + return manager_mock + + +class FakeTestType(manager.PluginTypeManager): + """Fake PluginTypeManager.""" + + namespace = TEST_NAMESPACE + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_instantiates_a_manager(PluginManager): + """Verify we create a PluginManager on instantiation.""" + FakeTestType() + + PluginManager.assert_called_once_with(TEST_NAMESPACE) + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxies_names_to_manager(PluginManager): + """Verify we proxy the names attribute.""" + PluginManager.return_value = mock.Mock(names=[ + 'T100', 'T200', 'T300' + ]) + type_mgr = FakeTestType() + + assert type_mgr.names == ['T100', 'T200', 'T300'] + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxies_plugins_to_manager(PluginManager): + """Verify we proxy the plugins attribute.""" + PluginManager.return_value = mock.Mock(plugins=[ + 'T100', 'T200', 'T300' + ]) + type_mgr = FakeTestType() + + assert type_mgr.plugins == ['T100', 'T200', 'T300'] + + +def test_generate_call_function(): + """Verify the function we generate.""" + optmanager = object() + plugin = mock.Mock(method_name=lambda x: x) + func = manager.PluginTypeManager._generate_call_function( + 'method_name', optmanager, + ) + + assert isinstance(func, collections.Callable) + assert func(plugin) is optmanager + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_load_plugins(PluginManager): + """Verify load plugins loads *every* plugin.""" + # Create a bunch of fake plugins + plugins = [create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + # Return our PluginManager mock + PluginManager.return_value = create_mapping_manager_mock(plugins) + + type_mgr = FakeTestType() + # Load the tests (do what we're actually testing) + assert len(type_mgr.load_plugins()) == 8 + # Assert that our closure does what we think it does + for plugin in plugins: + plugin.load_plugin.assert_called_once_with() + assert type_mgr.plugins_loaded is True + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_load_plugins_fails(PluginManager): + """Verify load plugins bubbles up exceptions.""" + plugins = [create_plugin_mock(), create_plugin_mock(True), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + # Return our PluginManager mock + PluginManager.return_value = create_mapping_manager_mock(plugins) + + type_mgr = FakeTestType() + with pytest.raises(exceptions.FailedToLoadPlugin): + type_mgr.load_plugins() + + # Assert we didn't finish loading plugins + assert type_mgr.plugins_loaded is False + # Assert the first two plugins had their load_plugin method called + plugins[0].load_plugin.assert_called_once_with() + plugins[1].load_plugin.assert_called_once_with() + # Assert the rest of the plugins were not loaded + for plugin in plugins[2:]: + assert plugin.load_plugin.called is False + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_register_options(PluginManager): + """Test that we map over every plugin to register options.""" + plugins = [create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + # Return our PluginManager mock + PluginManager.return_value = create_mapping_manager_mock(plugins) + optmanager = object() + + type_mgr = FakeTestType() + type_mgr.register_options(optmanager) + + for plugin in plugins: + plugin.register_options.assert_called_with(optmanager) + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_provide_options(PluginManager): + """Test that we map over every plugin to provide parsed options.""" + plugins = [create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + # Return our PluginManager mock + PluginManager.return_value = create_mapping_manager_mock(plugins) + optmanager = object() + options = object() + extra_args = [] + + type_mgr = FakeTestType() + type_mgr.provide_options(optmanager, options, extra_args) + + for plugin in plugins: + plugin.provide_options.assert_called_with(optmanager, + options, + extra_args) + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxy_contains_to_managers_plugins_dict(PluginManager): + """Verify that we proxy __contains__ to the manager's dictionary.""" + plugins = {'T10%i' % i: create_plugin_mock() for i in range(8)} + # Return our PluginManager mock + PluginManager.return_value = create_manager_with_plugins(plugins) + + type_mgr = FakeTestType() + for i in range(8): + key = 'T10%i' % i + assert key in type_mgr + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxies_getitem_to_managers_plugins_dictionary(PluginManager): + """Verify that we can use the PluginTypeManager like a dictionary.""" + plugins = {'T10%i' % i: create_plugin_mock() for i in range(8)} + # Return our PluginManager mock + PluginManager.return_value = create_manager_with_plugins(plugins) + + type_mgr = FakeTestType() + for i in range(8): + key = 'T10%i' % i + assert type_mgr[key] is plugins[key] + + +class FakePluginTypeManager(manager.NotifierBuilderMixin): + """Provide an easy way to test the NotifierBuilderMixin.""" + + def __init__(self, manager): + """Initialize with our fake manager.""" + self.names = sorted(manager.keys()) + self.manager = manager + + +@pytest.fixture +def notifier_builder(): + """Create a fake plugin type manager.""" + return FakePluginTypeManager(manager={ + 'T100': object(), + 'T101': object(), + 'T110': object(), + }) + + +def test_build_notifier(notifier_builder): + """Verify we properly build a Notifier object.""" + notifier = notifier_builder.build_notifier() + for name in ('T100', 'T101', 'T110'): + assert list(notifier.listeners_for(name)) == [ + notifier_builder.manager[name] + ] diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py new file mode 100644 index 0000000..3d05528 --- /dev/null +++ b/tests/unit/test_style_guide.py @@ -0,0 +1,207 @@ +"""Tests for the flake8.style_guide.StyleGuide class.""" +import optparse + +import mock +import pytest + +from flake8 import style_guide +from flake8.formatting import base +from flake8.plugins import notifier + + +def create_options(**kwargs): + """Create and return an instance of optparse.Values.""" + kwargs.setdefault('select', []) + kwargs.setdefault('ignore', []) + kwargs.setdefault('disable_noqa', False) + return optparse.Values(kwargs) + + +@pytest.mark.parametrize('ignore_list,error_code', [ + (['E111', 'E121'], 'E111'), + (['E111', 'E121'], 'E121'), + (['E11', 'E12'], 'E121'), + (['E2', 'E12'], 'E121'), + (['E2', 'E12'], 'E211'), +]) +def test_is_user_ignored_ignores_errors(ignore_list, error_code): + """Verify we detect users explicitly ignoring an error.""" + guide = style_guide.StyleGuide(create_options(ignore=ignore_list), + listener_trie=None, + formatter=None) + + assert guide.is_user_ignored(error_code) is style_guide.Ignored.Explicitly + + +@pytest.mark.parametrize('ignore_list,error_code', [ + (['E111', 'E121'], 'E112'), + (['E111', 'E121'], 'E122'), + (['E11', 'E12'], 'W121'), + (['E2', 'E12'], 'E112'), + (['E2', 'E12'], 'E111'), +]) +def test_is_user_ignored_implicitly_selects_errors(ignore_list, error_code): + """Verify we detect users does not explicitly ignore an error.""" + guide = style_guide.StyleGuide(create_options(ignore=ignore_list), + listener_trie=None, + formatter=None) + + assert guide.is_user_ignored(error_code) is style_guide.Selected.Implicitly + + +@pytest.mark.parametrize('select_list,error_code', [ + (['E111', 'E121'], 'E111'), + (['E111', 'E121'], 'E121'), + (['E11', 'E12'], 'E121'), + (['E2', 'E12'], 'E121'), + (['E2', 'E12'], 'E211'), +]) +def test_is_user_selected_selects_errors(select_list, error_code): + """Verify we detect users explicitly selecting an error.""" + guide = style_guide.StyleGuide(create_options(select=select_list), + listener_trie=None, + formatter=None) + + assert (guide.is_user_selected(error_code) is + style_guide.Selected.Explicitly) + + +def test_is_user_selected_implicitly_selects_errors(): + """Verify we detect users implicitly selecting an error.""" + select_list = [] + error_code = 'E121' + guide = style_guide.StyleGuide(create_options(select=select_list), + listener_trie=None, + formatter=None) + + assert (guide.is_user_selected(error_code) is + style_guide.Selected.Implicitly) + + +@pytest.mark.parametrize('select_list,error_code', [ + (['E111', 'E121'], 'E112'), + (['E111', 'E121'], 'E122'), + (['E11', 'E12'], 'E132'), + (['E2', 'E12'], 'E321'), + (['E2', 'E12'], 'E410'), +]) +def test_is_user_selected_excludes_errors(select_list, error_code): + """Verify we detect users implicitly excludes an error.""" + guide = style_guide.StyleGuide(create_options(select=select_list), + listener_trie=None, + formatter=None) + + assert guide.is_user_selected(error_code) is style_guide.Ignored.Implicitly + + +@pytest.mark.parametrize('select_list,ignore_list,error_code,expected', [ + (['E111', 'E121'], [], 'E111', style_guide.Decision.Selected), + (['E111', 'E121'], [], 'E112', style_guide.Decision.Ignored), + (['E111', 'E121'], [], 'E121', style_guide.Decision.Selected), + (['E111', 'E121'], [], 'E122', style_guide.Decision.Ignored), + (['E11', 'E12'], [], 'E132', style_guide.Decision.Ignored), + (['E2', 'E12'], [], 'E321', style_guide.Decision.Ignored), + (['E2', 'E12'], [], 'E410', style_guide.Decision.Ignored), + (['E11', 'E121'], ['E1'], 'E112', style_guide.Decision.Selected), + (['E111', 'E121'], ['E2'], 'E122', style_guide.Decision.Ignored), + (['E11', 'E12'], ['E13'], 'E132', style_guide.Decision.Ignored), + (['E1', 'E3'], ['E32'], 'E321', style_guide.Decision.Ignored), + ([], ['E2', 'E12'], 'E410', style_guide.Decision.Selected), + (['E4'], ['E2', 'E12', 'E41'], 'E410', style_guide.Decision.Ignored), + (['E41'], ['E2', 'E12', 'E4'], 'E410', style_guide.Decision.Selected), +]) +def test_should_report_error(select_list, ignore_list, error_code, expected): + """Verify we decide when to report an error.""" + guide = style_guide.StyleGuide(create_options(select=select_list, + ignore=ignore_list), + listener_trie=None, + formatter=None) + + assert guide.should_report_error(error_code) is expected + + +@pytest.mark.parametrize('error_code,physical_line,expected_result', [ + ('E111', 'a = 1', False), + ('E121', 'a = 1 # noqa: E111', False), + ('E121', 'a = 1 # noqa: E111,W123,F821', False), + ('E111', 'a = 1 # noqa: E111,W123,F821', True), + ('W123', 'a = 1 # noqa: E111,W123,F821', True), + ('E111', 'a = 1 # noqa: E11,W123,F821', True), +]) +def test_is_inline_ignored(error_code, physical_line, expected_result): + """Verify that we detect inline usage of ``# noqa``.""" + guide = style_guide.StyleGuide(create_options(select=['E', 'W', 'F']), + listener_trie=None, + formatter=None) + error = style_guide.Error(error_code, 'filename.py', 1, 1, 'error text', + None) + # We want `None` to be passed as the physical line so we actually use our + # monkey-patched linecache.getline value. + + with mock.patch('linecache.getline', return_value=physical_line): + assert guide.is_inline_ignored(error) is expected_result + + +def test_disable_is_inline_ignored(): + """Verify that is_inline_ignored exits immediately if disabling NoQA.""" + guide = style_guide.StyleGuide(create_options(disable_noqa=True), + listener_trie=None, + formatter=None) + error = style_guide.Error('E121', 'filename.py', 1, 1, 'error text', + 'line') + + with mock.patch('linecache.getline') as getline: + assert guide.is_inline_ignored(error) is False + + assert getline.called is False + + +@pytest.mark.parametrize('select_list,ignore_list,error_code', [ + (['E111', 'E121'], [], 'E111'), + (['E111', 'E121'], [], 'E121'), + (['E11', 'E121'], ['E1'], 'E112'), + ([], ['E2', 'E12'], 'E410'), + (['E41'], ['E2', 'E12', 'E4'], 'E410'), +]) +def test_handle_error_notifies_listeners(select_list, ignore_list, error_code): + """Verify that error codes notify the listener trie appropriately.""" + listener_trie = mock.create_autospec(notifier.Notifier, instance=True) + formatter = mock.create_autospec(base.BaseFormatter, instance=True) + guide = style_guide.StyleGuide(create_options(select=select_list, + ignore=ignore_list), + listener_trie=listener_trie, + formatter=formatter) + + with mock.patch('linecache.getline', return_value=''): + guide.handle_error(error_code, 'stdin', 1, 1, 'error found') + error = style_guide.Error(error_code, 'stdin', 1, 1, 'error found', + None) + listener_trie.notify.assert_called_once_with(error_code, error) + formatter.handle.assert_called_once_with(error) + + +@pytest.mark.parametrize('select_list,ignore_list,error_code', [ + (['E111', 'E121'], [], 'E122'), + (['E11', 'E12'], [], 'E132'), + (['E2', 'E12'], [], 'E321'), + (['E2', 'E12'], [], 'E410'), + (['E111', 'E121'], ['E2'], 'E122'), + (['E11', 'E12'], ['E13'], 'E132'), + (['E1', 'E3'], ['E32'], 'E321'), + (['E4'], ['E2', 'E12', 'E41'], 'E410'), + (['E111', 'E121'], [], 'E112'), +]) +def test_handle_error_does_not_notify_listeners(select_list, ignore_list, + error_code): + """Verify that error codes notify the listener trie appropriately.""" + listener_trie = mock.create_autospec(notifier.Notifier, instance=True) + formatter = mock.create_autospec(base.BaseFormatter, instance=True) + guide = style_guide.StyleGuide(create_options(select=select_list, + ignore=ignore_list), + listener_trie=listener_trie, + formatter=formatter) + + with mock.patch('linecache.getline', return_value=''): + guide.handle_error(error_code, 'stdin', 1, 1, 'error found') + assert listener_trie.notify.called is False + assert formatter.handle.called is False diff --git a/tests/unit/test_trie.py b/tests/unit/test_trie.py new file mode 100644 index 0000000..152b5b6 --- /dev/null +++ b/tests/unit/test_trie.py @@ -0,0 +1,122 @@ +"""Unit test for the _trie module.""" +from flake8.plugins import _trie as trie + + +class TestTrie(object): + """Collection of tests for the Trie class.""" + + def test_traverse_without_data(self): + """Verify the behaviour when traversing an empty Trie.""" + tree = trie.Trie() + assert list(tree.traverse()) == [] + + def test_traverse_with_data(self): + """Verify that traversal of our Trie is depth-first and pre-order.""" + tree = trie.Trie() + tree.add('A', 'A') + tree.add('a', 'a') + tree.add('AB', 'B') + tree.add('Ab', 'b') + tree.add('AbC', 'C') + tree.add('Abc', 'c') + # The trie tree here should look something like + # + # + # / \ + # A a + # / | + # B b + # / \ + # C c + # + # And the traversal should look like: + # + # A B b C c a + expected_order = ['A', 'B', 'b', 'C', 'c', 'a'] + for expected, actual_node in zip(expected_order, tree.traverse()): + assert actual_node.prefix == expected + + def test_find(self): + """Exercise the Trie.find method.""" + tree = trie.Trie() + tree.add('A', 'A') + tree.add('a', 'a') + tree.add('AB', 'AB') + tree.add('Ab', 'Ab') + tree.add('AbC', 'AbC') + tree.add('Abc', 'Abc') + + assert tree.find('AB').data == ['AB'] + assert tree.find('AbC').data == ['AbC'] + assert tree.find('A').data == ['A'] + assert tree.find('X') is None + + +class TestTrieNode(object): + """Collection of tests for the TrieNode class.""" + + def test_add_child(self): + """Verify we add children appropriately.""" + node = trie.TrieNode('E', 'E is for Eat') + assert node.find_prefix('a') is None + added = node.add_child('a', 'a is for Apple') + assert node.find_prefix('a') is added + + def test_add_child_overrides_previous_child(self): + """Verify adding a child will replace the previous child.""" + node = trie.TrieNode('E', 'E is for Eat', children={ + 'a': trie.TrieNode('a', 'a is for Apple') + }) + previous = node.find_prefix('a') + assert previous is not None + + added = node.add_child('a', 'a is for Ascertain') + assert node.find_prefix('a') is added + + def test_find_prefix(self): + """Verify we can find a child with the specified prefix.""" + node = trie.TrieNode('E', 'E is for Eat', children={ + 'a': trie.TrieNode('a', 'a is for Apple') + }) + child = node.find_prefix('a') + assert child is not None + assert child.prefix == 'a' + assert child.data == 'a is for Apple' + + def test_find_prefix_returns_None_when_no_children_have_the_prefix(self): + """Verify we receive None from find_prefix for missing children.""" + node = trie.TrieNode('E', 'E is for Eat', children={ + 'a': trie.TrieNode('a', 'a is for Apple') + }) + assert node.find_prefix('b') is None + + def test_traverse_does_nothing_when_a_node_has_no_children(self): + """Verify traversing a node with no children does nothing.""" + node = trie.TrieNode('E', 'E is for Eat') + assert list(node.traverse()) == [] + + def test_traverse(self): + """Verify traversal is depth-first and pre-order.""" + root = trie.TrieNode(None, None) + node = root.add_child('A', 'A') + root.add_child('a', 'a') + node.add_child('B', 'B') + node = node.add_child('b', 'b') + node.add_child('C', 'C') + node.add_child('c', 'c') + # The sub-tree here should look something like + # + # + # / \ + # A a + # / | + # B b + # / \ + # C c + # + # And the traversal should look like: + # + # A B b C c a + expected_order = ['A', 'B', 'b', 'C', 'c', 'a'] + for expected, actual_node in zip(expected_order, root.traverse()): + assert actual_node.prefix == expected diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..5d31e82 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,150 @@ +"""Tests for flake8's utils module.""" +import os + +import mock +import pytest + +from flake8 import utils +from flake8.plugins import manager as plugin_manager + +RELATIVE_PATHS = ["flake8", "pep8", "pyflakes", "mccabe"] + + +@pytest.mark.parametrize("value,expected", [ + ("E123,\n\tW234,\n E206", ["E123", "W234", "E206"]), + ("E123,W234,E206", ["E123", "W234", "E206"]), + (["E123", "W234", "E206"], ["E123", "W234", "E206"]), + (["E123", "\n\tW234", "\n E206"], ["E123", "W234", "E206"]), +]) +def test_parse_comma_separated_list(value, expected): + """Verify that similar inputs produce identical outputs.""" + assert utils.parse_comma_separated_list(value) == expected + + +@pytest.mark.parametrize("value,expected", [ + ("flake8", "flake8"), + ("../flake8", os.path.abspath("../flake8")), + ("flake8/", os.path.abspath("flake8")), +]) +def test_normalize_path(value, expected): + """Verify that we normalize paths provided to the tool.""" + assert utils.normalize_path(value) == expected + + +@pytest.mark.parametrize("value,expected", [ + ("flake8,pep8,pyflakes,mccabe", ["flake8", "pep8", "pyflakes", "mccabe"]), + ("flake8,\n\tpep8,\n pyflakes,\n\n mccabe", + ["flake8", "pep8", "pyflakes", "mccabe"]), + ("../flake8,../pep8,../pyflakes,../mccabe", + [os.path.abspath("../" + p) for p in RELATIVE_PATHS]), +]) +def test_normalize_paths(value, expected): + """Verify we normalize comma-separated paths provided to the tool.""" + assert utils.normalize_paths(value) == expected + + +def test_is_windows_checks_for_nt(): + """Verify that we correctly detect Windows.""" + with mock.patch.object(os, 'name', 'nt'): + assert utils.is_windows() is True + + with mock.patch.object(os, 'name', 'posix'): + assert utils.is_windows() is False + + +@pytest.mark.parametrize('filename,patterns,expected', [ + ('foo.py', [], True), + ('foo.py', ['*.pyc'], False), + ('foo.pyc', ['*.pyc'], True), + ('foo.pyc', ['*.swp', '*.pyc', '*.py'], True), +]) +def test_fnmatch(filename, patterns, expected): + """Verify that our fnmatch wrapper works as expected.""" + assert utils.fnmatch(filename, patterns) is expected + + +def test_fnmatch_returns_the_default_with_empty_default(): + """The default parameter should be returned when no patterns are given.""" + sentinel = object() + assert utils.fnmatch('file.py', [], default=sentinel) is sentinel + + +def test_filenames_from_a_directory(): + """Verify that filenames_from walks a directory.""" + filenames = list(utils.filenames_from('src/flake8/')) + assert len(filenames) > 2 + assert 'src/flake8/__init__.py' in filenames + + +def test_filenames_from_a_directory_with_a_predicate(): + """Verify that predicates filter filenames_from.""" + filenames = list(utils.filenames_from( + arg='src/flake8/', + predicate=lambda filename: filename == 'flake8/__init__.py', + )) + assert len(filenames) > 2 + assert 'flake8/__init__.py' not in filenames + + +def test_filenames_from_a_single_file(): + """Verify that we simply yield that filename.""" + filenames = list(utils.filenames_from('flake8/__init__.py')) + + assert len(filenames) == 1 + assert ['flake8/__init__.py'] == filenames + + +def test_parameters_for_class_plugin(): + """Verify that we can retrieve the parameters for a class plugin.""" + class FakeCheck(object): + def __init__(self, tree): + pass + + plugin = plugin_manager.Plugin('plugin-name', object()) + plugin._plugin = FakeCheck + assert utils.parameters_for(plugin) == ['tree'] + + +def test_parameters_for_function_plugin(): + """Verify that we retrieve the parameters for a function plugin.""" + def fake_plugin(physical_line, self, tree): + pass + + plugin = plugin_manager.Plugin('plugin-name', object()) + plugin._plugin = fake_plugin + assert utils.parameters_for(plugin) == ['physical_line', 'self', 'tree'] + + +def read_diff_file(filename): + """Read the diff file in its entirety.""" + with open(filename, 'r') as fd: + content = fd.read() + return content + + +SINGLE_FILE_DIFF = read_diff_file('tests/fixtures/diffs/single_file_diff') +SINGLE_FILE_INFO = { + 'flake8/utils.py': set(range(75, 83)).union(set(range(84, 94))), +} +TWO_FILE_DIFF = read_diff_file('tests/fixtures/diffs/two_file_diff') +TWO_FILE_INFO = { + 'flake8/utils.py': set(range(75, 83)).union(set(range(84, 94))), + 'tests/unit/test_utils.py': set(range(115, 128)), +} +MULTI_FILE_DIFF = read_diff_file('tests/fixtures/diffs/multi_file_diff') +MULTI_FILE_INFO = { + 'flake8/utils.py': set(range(75, 83)).union(set(range(84, 94))), + 'tests/unit/test_utils.py': set(range(115, 129)), + 'tests/fixtures/diffs/single_file_diff': set(range(1, 28)), + 'tests/fixtures/diffs/two_file_diff': set(range(1, 46)), +} + + +@pytest.mark.parametrize("diff, parsed_diff", [ + (SINGLE_FILE_DIFF, SINGLE_FILE_INFO), + (TWO_FILE_DIFF, TWO_FILE_INFO), + (MULTI_FILE_DIFF, MULTI_FILE_INFO), +]) +def test_parse_unified_diff(diff, parsed_diff): + """Verify that what we parse from a diff matches expectations.""" + assert utils.parse_unified_diff(diff) == parsed_diff diff --git a/tox.ini b/tox.ini index 14e2a44..58e7072 100644 --- a/tox.ini +++ b/tox.ini @@ -1,40 +1,138 @@ [tox] -minversion = 1.6 -envlist = - py26,py27,py33,py34,py27-flake8,py34-flake8 +minversion=2.3.1 +envlist = py27,py33,py34,py35,flake8 [testenv] -usedevelop = True deps = mock - nose + pytest commands = - python setup.py test -q - python setup.py flake8 - nosetests flake8.tests._test_warnings + py.test {posargs} -[testenv:py27-flake8] -basepython = python2.7 +[testenv:venv] +deps = + . +commands = {posargs} + +# Linters +[testenv:flake8] +skipsdist = true +skip_install = true +use_develop = false deps = flake8 -commands = flake8 {posargs} flake8/ + flake8-docstrings>=0.2.7 + flake8-import-order +commands = + flake8 src/flake8/ tests/ setup.py -[testenv:py34-flake8] -basepython = python3.4 +[testenv:pylint] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false deps = - flake8 -commands = flake8 {posargs} flake8/ + pyflakes + pylint +commands = + pylint flake8 + +[testenv:doc8] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + sphinx + doc8 +commands = + doc8 docs/source/ + +[testenv:mypy] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + mypy-lang +commands = + mypy flake8 + +[testenv:bandit] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + bandit +commands = + bandit -r flake8/ -c .bandit.yml + +[testenv:linters] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + {[testenv:flake8]deps} + {[testenv:pylint]deps} + {[testenv:doc8]deps} + {[testenv:bandit]deps} +commands = + {[testenv:flake8]commands} + {[testenv:pylint]commands} + {[testenv:doc8]commands} + {[testenv:bandit]commands} + +# Documentation +[testenv:docs] +basepython = python3 +deps = + -rdocs/source/requirements.txt +commands = + sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html + +[testenv:serve-docs] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +changedir = docs/build/html +deps = +commands = + python -m http.server {posargs} + +[testenv:readme] +deps = + readme_renderer +commands = + python setup.py check -r -s [testenv:release] -basepython = python2.7 +skipsdist = true +basepython = python3 +skip_install = true +use_develop = false deps = - twine >= 1.5.0 wheel + setuptools + twine >= 1.5.0 commands = - python setup.py sdist bdist_wheel - twine upload --skip-existing {posargs} dist/* + python setup.py -q sdist bdist_wheel + twine upload --skip-existing dist/* +# Flake8 Configuration [flake8] -select = E,F,W -max_line_length = 79 -exclude = .git,.tox,dist,docs,*egg +# Ignore some flake8-docstrings errors +# NOTE(sigmavirus24): While we're still using flake8 2.x, this ignore line +# defaults to selecting all other errors so we do not need select=E,F,W,I,D +# Once Flake8 3.0 is released and in a good state, we can use both and it will +# work well \o/ +ignore = D203 +# NOTE(sigmavirus24): Once we release 3.0.0 this exclude option can be specified +# across multiple lines. Presently it cannot be specified across multiple lines. +# :-( +exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,tests/fixtures/ +max-complexity = 10 +import-order-style = google +application-import-names = flake8