From 0b20e68e12684e9fa87b3fea7a0493bb90a017c2 Mon Sep 17 00:00:00 2001 From: Noortheen Raja Date: Fri, 15 Jan 2021 01:08:16 +0530 Subject: [PATCH 1/2] feat: add option to run with cache support fixes #601 --- src/flake8/cacher.py | 29 +++++++++++++++++++++++++++++ src/flake8/checker.py | 18 ++++++++++++++++++ src/flake8/main/application.py | 4 ++++ src/flake8/main/options.py | 15 +++++++++++++++ 4 files changed, 66 insertions(+) create mode 100644 src/flake8/cacher.py diff --git a/src/flake8/cacher.py b/src/flake8/cacher.py new file mode 100644 index 0000000..16ab1c8 --- /dev/null +++ b/src/flake8/cacher.py @@ -0,0 +1,29 @@ +"""Cache files manager""" +import hashlib +import os +import pickle + + +class Cacher(object): + def __init__(self, file_name, cache_path): + # type: (str, str) -> None + self.file_name = file_name + enc = hashlib.md5(file_name.encode()) + self.cache_path = os.path.join(cache_path, str(enc.hexdigest())) + + def get(self): + if not os.path.exists(self.cache_path): + return + with open(self.cache_path, "rb") as fr: + mtime, results = pickle.load(fr) + # if the mtime doesn't change then return cache. + # otherwise it is invalid + current_mtime = os.path.getmtime(self.file_name) + if mtime == current_mtime: + return results + + def save(self, results): + """saves the given result to the cache file""" + with open(self.cache_path, "wb") as fw: + current_mtime = os.path.getmtime(self.file_name) + pickle.dump((current_mtime, results), fw) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index b4ef97d..fd0f375 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -14,6 +14,7 @@ except ImportError: multiprocessing = None # type: ignore from flake8 import defaults +from flake8 import cacher from flake8 import exceptions from flake8 import processor from flake8 import utils @@ -360,6 +361,7 @@ class FileChecker(object): self.options = options self.filename = filename self.checks = checks + self.cacher = cacher.Cacher(filename, self.options.cache_location) # fmt: off self.results = [] # type: List[Tuple[str, int, int, str, Optional[str]]] # noqa: E501 # fmt: on @@ -583,6 +585,22 @@ class FileChecker(object): self.run_logical_checks() def run_checks(self): + """cached wrapper of self.run_checks""" + # handle both cases where cache is enabled/disabled or invalid + cache_available = False + if self.options.cache: + saved = self.cacher.get() + if saved is not None: + self.filename, self.results, self.statistics = saved + cache_available = True + + if cache_available is False: + result = self._run_checks() + if self.options.cache: + self.cacher.save(result) + return self.filename, self.results, self.statistics + + def _run_checks(self): """Run checks against the file.""" try: self.process_tokens() diff --git a/src/flake8/main/application.py b/src/flake8/main/application.py index 5c9e076..8adcc05 100644 --- a/src/flake8/main/application.py +++ b/src/flake8/main/application.py @@ -3,6 +3,7 @@ from __future__ import print_function import argparse import logging +import os import sys import time from typing import Dict, List, Optional, Set, Tuple @@ -336,6 +337,9 @@ class Application(object): self.make_formatter() self.make_guide() self.make_file_checker_manager() + if self.options.cache: + if not os.path.exists(self.options.cache_location): + os.makedirs(self.options.cache_location) def report(self): """Report errors, statistics, and benchmarks.""" diff --git a/src/flake8/main/options.py b/src/flake8/main/options.py index a0f066f..73b262e 100644 --- a/src/flake8/main/options.py +++ b/src/flake8/main/options.py @@ -298,6 +298,21 @@ def register_default_options(option_manager): ) # Flake8 options + add_option( + "--cache", + default=False, + parse_from_config=False, + action="store_true", + help="Cache flake8 results on consecutive runs.", + ) + + add_option( + "--cache-location", + default=".cache/flake8", + parse_from_config=True, + help="Location to store cached results.", + ) + add_option( "--enable-extensions", default="", From 88478aafc4f5049450a5eb6f3601cee3610b4534 Mon Sep 17 00:00:00 2001 From: Noortheen Raja Date: Fri, 15 Jan 2021 02:21:15 +0530 Subject: [PATCH 2/2] fix: linter errors --- src/flake8/cacher.py | 18 +++++++++++------- src/flake8/checker.py | 4 ++-- tests/unit/conftest.py | 1 + tests/unit/test_file_checker.py | 14 +++++++++++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/flake8/cacher.py b/src/flake8/cacher.py index 16ab1c8..c40cc13 100644 --- a/src/flake8/cacher.py +++ b/src/flake8/cacher.py @@ -1,21 +1,25 @@ -"""Cache files manager""" +"""Cache files manager.""" import hashlib +import marshal import os -import pickle class Cacher(object): + """Class to implement saving/loading cache for given file.""" + def __init__(self, file_name, cache_path): # type: (str, str) -> None + """Init cache class for the given file.""" self.file_name = file_name - enc = hashlib.md5(file_name.encode()) - self.cache_path = os.path.join(cache_path, str(enc.hexdigest())) + checksum = hashlib.md5(file_name.encode()) # nosec + self.cache_path = os.path.join(cache_path, str(checksum.hexdigest())) def get(self): + """Get cached result from the cache_path if available and valid.""" if not os.path.exists(self.cache_path): return with open(self.cache_path, "rb") as fr: - mtime, results = pickle.load(fr) + mtime, results = marshal.load(fr) # nosec # if the mtime doesn't change then return cache. # otherwise it is invalid current_mtime = os.path.getmtime(self.file_name) @@ -23,7 +27,7 @@ class Cacher(object): return results def save(self, results): - """saves the given result to the cache file""" + """Save the given result to the cache file.""" with open(self.cache_path, "wb") as fw: current_mtime = os.path.getmtime(self.file_name) - pickle.dump((current_mtime, results), fw) + marshal.dump((current_mtime, results), fw) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index fd0f375..3613da2 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -13,8 +13,8 @@ try: except ImportError: multiprocessing = None # type: ignore -from flake8 import defaults from flake8 import cacher +from flake8 import defaults from flake8 import exceptions from flake8 import processor from flake8 import utils @@ -585,7 +585,7 @@ class FileChecker(object): self.run_logical_checks() def run_checks(self): - """cached wrapper of self.run_checks""" + """Cache wrapper of self.run_checks.""" # handle both cases where cache is enabled/disabled or invalid cache_available = False if self.options.cache: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index eb76f98..aed14c0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,6 +12,7 @@ def options_from(**kwargs): kwargs.setdefault('verbose', False) kwargs.setdefault('stdin_display_name', 'stdin') kwargs.setdefault('disable_noqa', False) + kwargs.setdefault('cache_location', ".cache/flake8") return argparse.Namespace(**kwargs) diff --git a/tests/unit/test_file_checker.py b/tests/unit/test_file_checker.py index c4ee2bf..3c85ec7 100644 --- a/tests/unit/test_file_checker.py +++ b/tests/unit/test_file_checker.py @@ -1,4 +1,6 @@ """Unit tests for the FileChecker class.""" +import argparse + import mock import pytest @@ -6,6 +8,12 @@ import flake8 from flake8 import checker +def options(**kwargs): + """Generate argparse.Namespace for our Application.""" + kwargs.setdefault('cache_location', ".cache/flake8") + return argparse.Namespace(**kwargs) + + @mock.patch('flake8.processor.FileProcessor') def test_run_ast_checks_handles_SyntaxErrors(FileProcessor): # noqa: N802,N803 """Stress our SyntaxError handling. @@ -16,7 +24,7 @@ def test_run_ast_checks_handles_SyntaxErrors(FileProcessor): # noqa: N802,N803 FileProcessor.return_value = processor processor.build_ast.side_effect = SyntaxError('Failed to build ast', ('', 1, 5, 'foo(\n')) - file_checker = checker.FileChecker(__file__, checks={}, options=object()) + file_checker = checker.FileChecker(__file__, checks={}, options=options()) with mock.patch.object(file_checker, 'report') as report: file_checker.run_ast_checks() @@ -31,14 +39,14 @@ def test_run_ast_checks_handles_SyntaxErrors(FileProcessor): # noqa: N802,N803 def test_repr(*args): """Verify we generate a correct repr.""" file_checker = checker.FileChecker( - 'example.py', checks={}, options=object(), + 'example.py', checks={}, options=options(), ) assert repr(file_checker) == 'FileChecker for example.py' def test_nonexistent_file(): """Verify that checking non-existent file results in an error.""" - c = checker.FileChecker("foobar.py", checks={}, options=object()) + c = checker.FileChecker("foobar.py", checks={}, options=options()) assert c.processor is None assert not c.should_process