Merge branch 'per_file_ignore' into 'master'

Support more syntaxes in per-file-ignores

Closes #471

See merge request pycqa/flake8!281

src/flake8/main/options.py

@@ -1,6 +1,5 @@
 """Contains the logic for all of the default options for Flake8."""
 from flake8 import defaults
-from flake8 import utils
 from flake8.main import debug
 from flake8.main import vcs
 
@@ -146,9 +145,8 @@ def register_default_options(option_manager):
 
     add_option(
         "--per-file-ignores",
+        default="",
         parse_from_config=True,
-        comma_separated_list=True,
-        separator=utils.NEWLINE_SEPARATED_LIST_RE,
         help="A pairing of filenames and violation codes that defines which "
         "violations to ignore in a particular file. The filenames can be "
         "specified in a manner similar to the ``--exclude`` option and the "

src/flake8/options/manager.py

@@ -32,7 +32,6 @@ class Option(object):
         parse_from_config=False,
         comma_separated_list=False,
         normalize_paths=False,
-        separator=None,
     ):
         """Initialize an Option instance wrapping optparse.Option.
 
@@ -80,8 +79,6 @@ class Option(object):
         :param bool normalize_paths:
             Whether the option is expecting a path or list of paths and should
             attempt to normalize the paths to absolute paths.
-        :param separator:
-            The item that separates the "comma"-separated list.
         """
         self.short_option_name = short_option_name
         self.long_option_name = long_option_name
@@ -110,7 +107,6 @@ class Option(object):
         self.parse_from_config = parse_from_config
         self.comma_separated_list = comma_separated_list
         self.normalize_paths = normalize_paths
-        self.separator = separator or utils.COMMA_SEPARATED_LIST_RE
 
         self.config_name = None
         if parse_from_config:

src/flake8/style_guide.py

@@ -357,9 +357,10 @@ class StyleGuideManager(object):
         :rtype:
             :class:`~flake8.style_guide.StyleGuide`
         """
-        for value in options.per_file_ignores:
-            filename, violations_str = value.split(":")
-            violations = utils.parse_comma_separated_list(violations_str)
+        per_file = utils.parse_files_to_codes_mapping(
+            options.per_file_ignores
+        )
+        for filename, violations in per_file:
             yield self.default_style_guide.copy(
                 filename=filename, extend_ignore_with=violations
             )

src/flake8/utils.py

@@ -11,7 +11,6 @@ import tokenize
 
 DIFF_HUNK_REGEXP = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$")
 COMMA_SEPARATED_LIST_RE = re.compile(r"[,\s]")
-NEWLINE_SEPARATED_LIST_RE = re.compile(r"[\s]")
 LOCAL_PLUGIN_LIST_RE = re.compile(r"[,\t\n\r\f\v]")
 
 
@@ -41,6 +40,100 @@ def parse_comma_separated_list(value, regexp=COMMA_SEPARATED_LIST_RE):
     return [item for item in item_gen if item]
 
 
+_Token = collections.namedtuple("Token", ("tp", "src"))
+_CODE, _FILE, _COLON, _COMMA, _WS = "code", "file", "colon", "comma", "ws"
+_EOF = "eof"
+_FILE_LIST_TOKEN_TYPES = [
+    (re.compile(r"[A-Z][0-9]*"), _CODE),
+    (re.compile(r"[^\s:,]+"), _FILE),
+    (re.compile(r"\s*:\s*"), _COLON),
+    (re.compile(r"\s*,\s*"), _COMMA),
+    (re.compile(r"\s+"), _WS),
+]
+
+
+def _tokenize_files_to_codes_mapping(value):
+    # type: (str) -> List[_Token]
+    tokens = []
+    i = 0
+    while i < len(value):
+        for token_re, token_name in _FILE_LIST_TOKEN_TYPES:
+            match = token_re.match(value, i)
+            if match:
+                tokens.append(_Token(token_name, match.group().strip()))
+                i = match.end()
+                break
+        else:
+            raise AssertionError("unreachable", value, i)
+    tokens.append(_Token(_EOF, ""))
+
+    return tokens
+
+
+def parse_files_to_codes_mapping(value):  # noqa: C901
+    # type: (Union[Sequence[str], str]) -> List[Tuple[List[str], List[str]]]
+    """Parse a files-to-codes maping.
+
+    A files-to-codes mapping a sequence of values specified as
+    `filenames list:codes list ...`.  Each of the lists may be separated by
+    either comma or whitespace tokens.
+
+    :param value: String to be parsed and normalized.
+    :type value: str
+    """
+    if isinstance(value, (list, tuple)):
+        value = "\n".join(value)
+
+    ret = []
+    if not value.strip():
+        return ret
+
+    class State:
+        seen_sep = True
+        seen_colon = False
+        filenames = []
+        codes = []
+
+    def _reset():
+        if State.codes:
+            for filename in State.filenames:
+                ret.append((filename, State.codes))
+        State.seen_sep = True
+        State.seen_colon = False
+        State.filenames = []
+        State.codes = []
+
+    for token in _tokenize_files_to_codes_mapping(value):
+        # legal in any state: separator sets the sep bit
+        if token.tp in {_COMMA, _WS}:
+            State.seen_sep = True
+        # looking for filenames
+        elif not State.seen_colon:
+            if token.tp == _COLON:
+                State.seen_colon = True
+                State.seen_sep = True
+            elif State.seen_sep and token.tp == _FILE:
+                State.filenames.append(token.src)
+                State.seen_sep = False
+            else:
+                raise ValueError("Unexpected token: {}".format(token))
+        # looking for codes
+        else:
+            if token.tp == _EOF:
+                _reset()
+            elif State.seen_sep and token.tp == _CODE:
+                State.codes.append(token.src)
+                State.seen_sep = False
+            elif State.seen_sep and token.tp == _FILE:
+                _reset()
+                State.filenames.append(token.src)
+                State.seen_sep = False
+            else:
+                raise ValueError("Unexpected token: {}".format(token))
+
+    return ret
+
+
 def normalize_paths(paths, parent=os.curdir):
     # type: (Union[Sequence[str], str], str) -> List[str]
     """Parse a comma-separated list of paths.