Merge branch 'file-tokens' of xzise/flake8

src/flake8/processor.py

@@ -42,6 +42,7 @@ class FileProcessor(object):
     - :attr:`previous_indent_level`
     - :attr:`previous_logical`
     - :attr:`tokens`
+    - :attr:`file_tokens`
     - :attr:`total_lines`
     - :attr:`verbose`
     """
@@ -98,6 +99,19 @@ class FileProcessor(object):
         self.statistics = {
             'logical lines': 0,
         }
+        self._file_tokens = None
+
+    @property
+    def file_tokens(self):
+        if self._file_tokens is None:
+            line_iter = iter(self.lines)
+            try:
+                self._file_tokens = list(tokenize.generate_tokens(
+                    lambda: next(line_iter)))
+            except tokenize.TokenError as exc:
+                raise exceptions.InvalidSyntax(exc.message, exception=exc)
+
+        return self._file_tokens[:]
 
     @contextlib.contextmanager
     def inside_multiline(self, line_number):