mirror of
https://github.com/pre-commit/pre-commit-hooks.git
synced 2026-04-14 14:34:45 +00:00
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot]
parent
72ad6dc953
commit
f4cd1ba0d6
813 changed files with 66015 additions and 58839 deletions
|
|
@ -1,14 +1,14 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""CacheControl import Interface.
|
||||
|
||||
Make it easy to import from cachecontrol without long namespaces.
|
||||
"""
|
||||
__author__ = "Eric Larson"
|
||||
__email__ = "eric@ionrock.org"
|
||||
__version__ = "0.12.10"
|
||||
from __future__ import annotations
|
||||
__author__ = 'Eric Larson'
|
||||
__email__ = 'eric@ionrock.org'
|
||||
__version__ = '0.12.10'
|
||||
|
||||
from .wrapper import CacheControl
|
||||
from .adapter import CacheControlAdapter
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from argparse import ArgumentParser
|
||||
|
||||
from pip._vendor import requests
|
||||
|
||||
from pip._vendor.cachecontrol.adapter import CacheControlAdapter
|
||||
from pip._vendor.cachecontrol.cache import DictCache
|
||||
from pip._vendor.cachecontrol.controller import logger
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
|
||||
def setup_logging():
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
|
@ -21,11 +20,11 @@ def setup_logging():
|
|||
|
||||
def get_session():
|
||||
adapter = CacheControlAdapter(
|
||||
DictCache(), cache_etags=True, serializer=None, heuristic=None
|
||||
DictCache(), cache_etags=True, serializer=None, heuristic=None,
|
||||
)
|
||||
sess = requests.Session()
|
||||
sess.mount("http://", adapter)
|
||||
sess.mount("https://", adapter)
|
||||
sess.mount('http://', adapter)
|
||||
sess.mount('https://', adapter)
|
||||
|
||||
sess.cache_controller = adapter.controller
|
||||
return sess
|
||||
|
|
@ -33,7 +32,7 @@ def get_session():
|
|||
|
||||
def get_args():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("url", help="The URL to try and cache")
|
||||
parser.add_argument('url', help='The URL to try and cache')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
|
@ -52,10 +51,10 @@ def main(args=None):
|
|||
|
||||
# Now try to get it
|
||||
if sess.cache_controller.cached_request(resp.request):
|
||||
print("Cached!")
|
||||
print('Cached!')
|
||||
else:
|
||||
print("Not cached :(")
|
||||
print('Not cached :(')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,20 +1,22 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
import functools
|
||||
import types
|
||||
import zlib
|
||||
|
||||
from pip._vendor.requests.adapters import HTTPAdapter
|
||||
|
||||
from .controller import CacheController, PERMANENT_REDIRECT_STATUSES
|
||||
from .cache import DictCache
|
||||
from .controller import CacheController
|
||||
from .controller import PERMANENT_REDIRECT_STATUSES
|
||||
from .filewrapper import CallbackFileWrapper
|
||||
|
||||
|
||||
class CacheControlAdapter(HTTPAdapter):
|
||||
invalidating_methods = {"PUT", "PATCH", "DELETE"}
|
||||
invalidating_methods = {'PUT', 'PATCH', 'DELETE'}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -25,16 +27,16 @@ class CacheControlAdapter(HTTPAdapter):
|
|||
heuristic=None,
|
||||
cacheable_methods=None,
|
||||
*args,
|
||||
**kw
|
||||
**kw,
|
||||
):
|
||||
super(CacheControlAdapter, self).__init__(*args, **kw)
|
||||
super().__init__(*args, **kw)
|
||||
self.cache = DictCache() if cache is None else cache
|
||||
self.heuristic = heuristic
|
||||
self.cacheable_methods = cacheable_methods or ("GET",)
|
||||
self.cacheable_methods = cacheable_methods or ('GET',)
|
||||
|
||||
controller_factory = controller_class or CacheController
|
||||
self.controller = controller_factory(
|
||||
self.cache, cache_etags=cache_etags, serializer=serializer
|
||||
self.cache, cache_etags=cache_etags, serializer=serializer,
|
||||
)
|
||||
|
||||
def send(self, request, cacheable_methods=None, **kw):
|
||||
|
|
@ -54,12 +56,12 @@ class CacheControlAdapter(HTTPAdapter):
|
|||
# check for etags and add headers if appropriate
|
||||
request.headers.update(self.controller.conditional_headers(request))
|
||||
|
||||
resp = super(CacheControlAdapter, self).send(request, **kw)
|
||||
resp = super().send(request, **kw)
|
||||
|
||||
return resp
|
||||
|
||||
def build_response(
|
||||
self, request, response, from_cache=False, cacheable_methods=None
|
||||
self, request, response, from_cache=False, cacheable_methods=None,
|
||||
):
|
||||
"""
|
||||
Build a response by making a request or using the cache.
|
||||
|
|
@ -81,7 +83,7 @@ class CacheControlAdapter(HTTPAdapter):
|
|||
# have an etag. In either case, we want to try and
|
||||
# update the cache if that is the case.
|
||||
cached_response = self.controller.update_cached_response(
|
||||
request, response
|
||||
request, response,
|
||||
)
|
||||
|
||||
if cached_response is not response:
|
||||
|
|
@ -105,7 +107,7 @@ class CacheControlAdapter(HTTPAdapter):
|
|||
response._fp = CallbackFileWrapper(
|
||||
response._fp,
|
||||
functools.partial(
|
||||
self.controller.cache_response, request, response
|
||||
self.controller.cache_response, request, response,
|
||||
),
|
||||
)
|
||||
if response.chunked:
|
||||
|
|
@ -117,10 +119,10 @@ class CacheControlAdapter(HTTPAdapter):
|
|||
self._fp._close()
|
||||
|
||||
response._update_chunk_length = types.MethodType(
|
||||
_update_chunk_length, response
|
||||
_update_chunk_length, response,
|
||||
)
|
||||
|
||||
resp = super(CacheControlAdapter, self).build_response(request, response)
|
||||
resp = super().build_response(request, response)
|
||||
|
||||
# See if we should invalidate the cache.
|
||||
if request.method in self.invalidating_methods and resp.ok:
|
||||
|
|
@ -134,4 +136,4 @@ class CacheControlAdapter(HTTPAdapter):
|
|||
|
||||
def close(self):
|
||||
self.cache.close()
|
||||
super(CacheControlAdapter, self).close()
|
||||
super().close()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
The cache object API for implementing caches. The default is a thread
|
||||
safe in-memory dictionary.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from threading import Lock
|
||||
|
||||
|
||||
class BaseCache(object):
|
||||
class BaseCache:
|
||||
|
||||
def get(self, key):
|
||||
raise NotImplementedError()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
from .file_cache import FileCache # noqa
|
||||
from .redis_cache import RedisCache # noqa
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
|
|
@ -28,18 +29,18 @@ def _secure_open_write(filename, fmode):
|
|||
|
||||
# Do not follow symlinks to prevent someone from making a symlink that
|
||||
# we follow and insecurely open a cache file.
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
if hasattr(os, 'O_NOFOLLOW'):
|
||||
flags |= os.O_NOFOLLOW
|
||||
|
||||
# On Windows we'll mark this file as binary
|
||||
if hasattr(os, "O_BINARY"):
|
||||
if hasattr(os, 'O_BINARY'):
|
||||
flags |= os.O_BINARY
|
||||
|
||||
# Before we open our file, we want to delete any existing file that is
|
||||
# there
|
||||
try:
|
||||
os.remove(filename)
|
||||
except (IOError, OSError):
|
||||
except OSError:
|
||||
# The file must not exist already, so we can just skip ahead to opening
|
||||
pass
|
||||
|
||||
|
|
@ -49,7 +50,7 @@ def _secure_open_write(filename, fmode):
|
|||
# happen if someone is attempting to attack us.
|
||||
fd = os.open(filename, flags, fmode)
|
||||
try:
|
||||
return os.fdopen(fd, "wb")
|
||||
return os.fdopen(fd, 'wb')
|
||||
|
||||
except:
|
||||
# An error occurred wrapping our FD in a file object
|
||||
|
|
@ -70,7 +71,7 @@ class FileCache(BaseCache):
|
|||
):
|
||||
|
||||
if use_dir_lock is not None and lock_class is not None:
|
||||
raise ValueError("Cannot use use_dir_lock and lock_class together")
|
||||
raise ValueError('Cannot use use_dir_lock and lock_class together')
|
||||
|
||||
try:
|
||||
from lockfile import LockFile
|
||||
|
|
@ -81,7 +82,7 @@ class FileCache(BaseCache):
|
|||
NOTE: In order to use the FileCache you must have
|
||||
lockfile installed. You can install it via pip:
|
||||
pip install lockfile
|
||||
"""
|
||||
""",
|
||||
)
|
||||
raise ImportError(notice)
|
||||
|
||||
|
|
@ -112,7 +113,7 @@ class FileCache(BaseCache):
|
|||
def get(self, key):
|
||||
name = self._fn(key)
|
||||
try:
|
||||
with open(name, "rb") as fh:
|
||||
with open(name, 'rb') as fh:
|
||||
return fh.read()
|
||||
|
||||
except FileNotFoundError:
|
||||
|
|
@ -124,7 +125,7 @@ class FileCache(BaseCache):
|
|||
# Make sure the directory exists
|
||||
try:
|
||||
os.makedirs(os.path.dirname(name), self.dirmode)
|
||||
except (IOError, OSError):
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with self.lock_class(name) as lock:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import division
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pip._vendor.cachecontrol.cache import BaseCache
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
try:
|
||||
from urllib.parse import urljoin
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
The httplib2 algorithms ported for use with requests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import logging
|
||||
import re
|
||||
import calendar
|
||||
import time
|
||||
from email.utils import parsedate_tz
|
||||
|
||||
|
|
@ -19,7 +20,7 @@ from .serialize import Serializer
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
|
||||
URI = re.compile(r'^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?')
|
||||
|
||||
PERMANENT_REDIRECT_STATUSES = (301, 308)
|
||||
|
||||
|
|
@ -33,12 +34,12 @@ def parse_uri(uri):
|
|||
return (groups[1], groups[3], groups[4], groups[6], groups[8])
|
||||
|
||||
|
||||
class CacheController(object):
|
||||
class CacheController:
|
||||
"""An interface to see if request should cached or not.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, cache=None, cache_etags=True, serializer=None, status_codes=None
|
||||
self, cache=None, cache_etags=True, serializer=None, status_codes=None,
|
||||
):
|
||||
self.cache = DictCache() if cache is None else cache
|
||||
self.cache_etags = cache_etags
|
||||
|
|
@ -50,18 +51,18 @@ class CacheController(object):
|
|||
"""Normalize the URL to create a safe key for the cache"""
|
||||
(scheme, authority, path, query, fragment) = parse_uri(uri)
|
||||
if not scheme or not authority:
|
||||
raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
|
||||
raise Exception('Only absolute URIs are allowed. uri = %s' % uri)
|
||||
|
||||
scheme = scheme.lower()
|
||||
authority = authority.lower()
|
||||
|
||||
if not path:
|
||||
path = "/"
|
||||
path = '/'
|
||||
|
||||
# Could do syntax based normalization of the URI before
|
||||
# computing the digest. See Section 6.2.2 of Std 66.
|
||||
request_uri = query and "?".join([path, query]) or path
|
||||
defrag_uri = scheme + "://" + authority + request_uri
|
||||
request_uri = query and '?'.join([path, query]) or path
|
||||
defrag_uri = scheme + '://' + authority + request_uri
|
||||
|
||||
return defrag_uri
|
||||
|
||||
|
|
@ -72,35 +73,35 @@ class CacheController(object):
|
|||
def parse_cache_control(self, headers):
|
||||
known_directives = {
|
||||
# https://tools.ietf.org/html/rfc7234#section-5.2
|
||||
"max-age": (int, True),
|
||||
"max-stale": (int, False),
|
||||
"min-fresh": (int, True),
|
||||
"no-cache": (None, False),
|
||||
"no-store": (None, False),
|
||||
"no-transform": (None, False),
|
||||
"only-if-cached": (None, False),
|
||||
"must-revalidate": (None, False),
|
||||
"public": (None, False),
|
||||
"private": (None, False),
|
||||
"proxy-revalidate": (None, False),
|
||||
"s-maxage": (int, True),
|
||||
'max-age': (int, True),
|
||||
'max-stale': (int, False),
|
||||
'min-fresh': (int, True),
|
||||
'no-cache': (None, False),
|
||||
'no-store': (None, False),
|
||||
'no-transform': (None, False),
|
||||
'only-if-cached': (None, False),
|
||||
'must-revalidate': (None, False),
|
||||
'public': (None, False),
|
||||
'private': (None, False),
|
||||
'proxy-revalidate': (None, False),
|
||||
's-maxage': (int, True),
|
||||
}
|
||||
|
||||
cc_headers = headers.get("cache-control", headers.get("Cache-Control", ""))
|
||||
cc_headers = headers.get('cache-control', headers.get('Cache-Control', ''))
|
||||
|
||||
retval = {}
|
||||
|
||||
for cc_directive in cc_headers.split(","):
|
||||
for cc_directive in cc_headers.split(','):
|
||||
if not cc_directive.strip():
|
||||
continue
|
||||
|
||||
parts = cc_directive.split("=", 1)
|
||||
parts = cc_directive.split('=', 1)
|
||||
directive = parts[0].strip()
|
||||
|
||||
try:
|
||||
typ, required = known_directives[directive]
|
||||
except KeyError:
|
||||
logger.debug("Ignoring unknown cache-control directive: %s", directive)
|
||||
logger.debug('Ignoring unknown cache-control directive: %s', directive)
|
||||
continue
|
||||
|
||||
if not typ or not required:
|
||||
|
|
@ -111,12 +112,12 @@ class CacheController(object):
|
|||
except IndexError:
|
||||
if required:
|
||||
logger.debug(
|
||||
"Missing value for cache-control " "directive: %s",
|
||||
'Missing value for cache-control ' 'directive: %s',
|
||||
directive,
|
||||
)
|
||||
except ValueError:
|
||||
logger.debug(
|
||||
"Invalid value for cache-control directive " "%s, must be %s",
|
||||
'Invalid value for cache-control directive ' '%s, must be %s',
|
||||
directive,
|
||||
typ.__name__,
|
||||
)
|
||||
|
|
@ -133,24 +134,24 @@ class CacheController(object):
|
|||
cc = self.parse_cache_control(request.headers)
|
||||
|
||||
# Bail out if the request insists on fresh data
|
||||
if "no-cache" in cc:
|
||||
if 'no-cache' in cc:
|
||||
logger.debug('Request header has "no-cache", cache bypassed')
|
||||
return False
|
||||
|
||||
if "max-age" in cc and cc["max-age"] == 0:
|
||||
if 'max-age' in cc and cc['max-age'] == 0:
|
||||
logger.debug('Request header has "max_age" as 0, cache bypassed')
|
||||
return False
|
||||
|
||||
# Request allows serving from the cache, let's see if we find something
|
||||
cache_data = self.cache.get(cache_url)
|
||||
if cache_data is None:
|
||||
logger.debug("No cache entry available")
|
||||
logger.debug('No cache entry available')
|
||||
return False
|
||||
|
||||
# Check whether it can be deserialized
|
||||
resp = self.serializer.loads(request, cache_data)
|
||||
if not resp:
|
||||
logger.warning("Cache entry deserialization failed, entry ignored")
|
||||
logger.warning('Cache entry deserialization failed, entry ignored')
|
||||
return False
|
||||
|
||||
# If we have a cached permanent redirect, return it immediately. We
|
||||
|
|
@ -164,26 +165,26 @@ class CacheController(object):
|
|||
# with cache busting headers as usual (ie no-cache).
|
||||
if int(resp.status) in PERMANENT_REDIRECT_STATUSES:
|
||||
msg = (
|
||||
"Returning cached permanent redirect response "
|
||||
"(ignoring date and etag information)"
|
||||
'Returning cached permanent redirect response '
|
||||
'(ignoring date and etag information)'
|
||||
)
|
||||
logger.debug(msg)
|
||||
return resp
|
||||
|
||||
headers = CaseInsensitiveDict(resp.headers)
|
||||
if not headers or "date" not in headers:
|
||||
if "etag" not in headers:
|
||||
if not headers or 'date' not in headers:
|
||||
if 'etag' not in headers:
|
||||
# Without date or etag, the cached response can never be used
|
||||
# and should be deleted.
|
||||
logger.debug("Purging cached response: no date or etag")
|
||||
logger.debug('Purging cached response: no date or etag')
|
||||
self.cache.delete(cache_url)
|
||||
logger.debug("Ignoring cached response: no date")
|
||||
logger.debug('Ignoring cached response: no date')
|
||||
return False
|
||||
|
||||
now = time.time()
|
||||
date = calendar.timegm(parsedate_tz(headers["date"]))
|
||||
date = calendar.timegm(parsedate_tz(headers['date']))
|
||||
current_age = max(0, now - date)
|
||||
logger.debug("Current age based on date: %i", current_age)
|
||||
logger.debug('Current age based on date: %i', current_age)
|
||||
|
||||
# TODO: There is an assumption that the result will be a
|
||||
# urllib3 response object. This may not be best since we
|
||||
|
|
@ -195,40 +196,40 @@ class CacheController(object):
|
|||
freshness_lifetime = 0
|
||||
|
||||
# Check the max-age pragma in the cache control header
|
||||
if "max-age" in resp_cc:
|
||||
freshness_lifetime = resp_cc["max-age"]
|
||||
logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime)
|
||||
if 'max-age' in resp_cc:
|
||||
freshness_lifetime = resp_cc['max-age']
|
||||
logger.debug('Freshness lifetime from max-age: %i', freshness_lifetime)
|
||||
|
||||
# If there isn't a max-age, check for an expires header
|
||||
elif "expires" in headers:
|
||||
expires = parsedate_tz(headers["expires"])
|
||||
elif 'expires' in headers:
|
||||
expires = parsedate_tz(headers['expires'])
|
||||
if expires is not None:
|
||||
expire_time = calendar.timegm(expires) - date
|
||||
freshness_lifetime = max(0, expire_time)
|
||||
logger.debug("Freshness lifetime from expires: %i", freshness_lifetime)
|
||||
logger.debug('Freshness lifetime from expires: %i', freshness_lifetime)
|
||||
|
||||
# Determine if we are setting freshness limit in the
|
||||
# request. Note, this overrides what was in the response.
|
||||
if "max-age" in cc:
|
||||
freshness_lifetime = cc["max-age"]
|
||||
if 'max-age' in cc:
|
||||
freshness_lifetime = cc['max-age']
|
||||
logger.debug(
|
||||
"Freshness lifetime from request max-age: %i", freshness_lifetime
|
||||
'Freshness lifetime from request max-age: %i', freshness_lifetime,
|
||||
)
|
||||
|
||||
if "min-fresh" in cc:
|
||||
min_fresh = cc["min-fresh"]
|
||||
if 'min-fresh' in cc:
|
||||
min_fresh = cc['min-fresh']
|
||||
# adjust our current age by our min fresh
|
||||
current_age += min_fresh
|
||||
logger.debug("Adjusted current age from min-fresh: %i", current_age)
|
||||
logger.debug('Adjusted current age from min-fresh: %i', current_age)
|
||||
|
||||
# Return entry if it is fresh enough
|
||||
if freshness_lifetime > current_age:
|
||||
logger.debug('The response is "fresh", returning cached response')
|
||||
logger.debug("%i > %i", freshness_lifetime, current_age)
|
||||
logger.debug('%i > %i', freshness_lifetime, current_age)
|
||||
return resp
|
||||
|
||||
# we're not fresh. If we don't have an Etag, clear it out
|
||||
if "etag" not in headers:
|
||||
if 'etag' not in headers:
|
||||
logger.debug('The cached response is "stale" with no etag, purging')
|
||||
self.cache.delete(cache_url)
|
||||
|
||||
|
|
@ -243,11 +244,11 @@ class CacheController(object):
|
|||
if resp:
|
||||
headers = CaseInsensitiveDict(resp.headers)
|
||||
|
||||
if "etag" in headers:
|
||||
new_headers["If-None-Match"] = headers["ETag"]
|
||||
if 'etag' in headers:
|
||||
new_headers['If-None-Match'] = headers['ETag']
|
||||
|
||||
if "last-modified" in headers:
|
||||
new_headers["If-Modified-Since"] = headers["Last-Modified"]
|
||||
if 'last-modified' in headers:
|
||||
new_headers['If-Modified-Since'] = headers['Last-Modified']
|
||||
|
||||
return new_headers
|
||||
|
||||
|
|
@ -262,14 +263,14 @@ class CacheController(object):
|
|||
cacheable_status_codes = status_codes or self.cacheable_status_codes
|
||||
if response.status not in cacheable_status_codes:
|
||||
logger.debug(
|
||||
"Status code %s not in %s", response.status, cacheable_status_codes
|
||||
'Status code %s not in %s', response.status, cacheable_status_codes,
|
||||
)
|
||||
return
|
||||
|
||||
response_headers = CaseInsensitiveDict(response.headers)
|
||||
|
||||
if "date" in response_headers:
|
||||
date = calendar.timegm(parsedate_tz(response_headers["date"]))
|
||||
if 'date' in response_headers:
|
||||
date = calendar.timegm(parsedate_tz(response_headers['date']))
|
||||
else:
|
||||
date = 0
|
||||
|
||||
|
|
@ -278,10 +279,10 @@ class CacheController(object):
|
|||
# been given matches the expected size, and if it doesn't we'll just
|
||||
# skip trying to cache it.
|
||||
if (
|
||||
body is not None
|
||||
and "content-length" in response_headers
|
||||
and response_headers["content-length"].isdigit()
|
||||
and int(response_headers["content-length"]) != len(body)
|
||||
body is not None and
|
||||
'content-length' in response_headers and
|
||||
response_headers['content-length'].isdigit() and
|
||||
int(response_headers['content-length']) != len(body)
|
||||
):
|
||||
return
|
||||
|
||||
|
|
@ -293,10 +294,10 @@ class CacheController(object):
|
|||
|
||||
# Delete it from the cache if we happen to have it stored there
|
||||
no_store = False
|
||||
if "no-store" in cc:
|
||||
if 'no-store' in cc:
|
||||
no_store = True
|
||||
logger.debug('Response header has "no-store"')
|
||||
if "no-store" in cc_req:
|
||||
if 'no-store' in cc_req:
|
||||
no_store = True
|
||||
logger.debug('Request header has "no-store"')
|
||||
if no_store and self.cache.get(cache_url):
|
||||
|
|
@ -310,22 +311,22 @@ class CacheController(object):
|
|||
# Storing such a response leads to a deserialization warning
|
||||
# during cache lookup and is not allowed to ever be served,
|
||||
# so storing it can be avoided.
|
||||
if "*" in response_headers.get("vary", ""):
|
||||
if '*' in response_headers.get('vary', ''):
|
||||
logger.debug('Response header has "Vary: *"')
|
||||
return
|
||||
|
||||
# If we've been given an etag, then keep the response
|
||||
if self.cache_etags and "etag" in response_headers:
|
||||
if self.cache_etags and 'etag' in response_headers:
|
||||
expires_time = 0
|
||||
if response_headers.get("expires"):
|
||||
expires = parsedate_tz(response_headers["expires"])
|
||||
if response_headers.get('expires'):
|
||||
expires = parsedate_tz(response_headers['expires'])
|
||||
if expires is not None:
|
||||
expires_time = calendar.timegm(expires) - date
|
||||
|
||||
expires_time = max(expires_time, 14 * 86400)
|
||||
|
||||
logger.debug("etag object cached for {0} seconds".format(expires_time))
|
||||
logger.debug("Caching due to etag")
|
||||
logger.debug(f'etag object cached for {expires_time} seconds')
|
||||
logger.debug('Caching due to etag')
|
||||
self.cache.set(
|
||||
cache_url,
|
||||
self.serializer.dumps(request, response, body),
|
||||
|
|
@ -335,18 +336,18 @@ class CacheController(object):
|
|||
# Add to the cache any permanent redirects. We do this before looking
|
||||
# that the Date headers.
|
||||
elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
|
||||
logger.debug("Caching permanent redirect")
|
||||
self.cache.set(cache_url, self.serializer.dumps(request, response, b""))
|
||||
logger.debug('Caching permanent redirect')
|
||||
self.cache.set(cache_url, self.serializer.dumps(request, response, b''))
|
||||
|
||||
# Add to the cache if the response headers demand it. If there
|
||||
# is no date header then we can't do anything about expiring
|
||||
# the cache.
|
||||
elif "date" in response_headers:
|
||||
date = calendar.timegm(parsedate_tz(response_headers["date"]))
|
||||
elif 'date' in response_headers:
|
||||
date = calendar.timegm(parsedate_tz(response_headers['date']))
|
||||
# cache when there is a max-age > 0
|
||||
if "max-age" in cc and cc["max-age"] > 0:
|
||||
logger.debug("Caching b/c date exists and max-age > 0")
|
||||
expires_time = cc["max-age"]
|
||||
if 'max-age' in cc and cc['max-age'] > 0:
|
||||
logger.debug('Caching b/c date exists and max-age > 0')
|
||||
expires_time = cc['max-age']
|
||||
self.cache.set(
|
||||
cache_url,
|
||||
self.serializer.dumps(request, response, body),
|
||||
|
|
@ -355,18 +356,18 @@ class CacheController(object):
|
|||
|
||||
# If the request can expire, it means we should cache it
|
||||
# in the meantime.
|
||||
elif "expires" in response_headers:
|
||||
if response_headers["expires"]:
|
||||
expires = parsedate_tz(response_headers["expires"])
|
||||
elif 'expires' in response_headers:
|
||||
if response_headers['expires']:
|
||||
expires = parsedate_tz(response_headers['expires'])
|
||||
if expires is not None:
|
||||
expires_time = calendar.timegm(expires) - date
|
||||
else:
|
||||
expires_time = None
|
||||
|
||||
logger.debug(
|
||||
"Caching b/c of expires header. expires in {0} seconds".format(
|
||||
expires_time
|
||||
)
|
||||
'Caching b/c of expires header. expires in {} seconds'.format(
|
||||
expires_time,
|
||||
),
|
||||
)
|
||||
self.cache.set(
|
||||
cache_url,
|
||||
|
|
@ -396,14 +397,14 @@ class CacheController(object):
|
|||
# the cached body invalid. But... just in case, we'll be sure
|
||||
# to strip out ones we know that might be problmatic due to
|
||||
# typical assumptions.
|
||||
excluded_headers = ["content-length"]
|
||||
excluded_headers = ['content-length']
|
||||
|
||||
cached_response.headers.update(
|
||||
dict(
|
||||
(k, v)
|
||||
{
|
||||
k: v
|
||||
for k, v in response.headers.items()
|
||||
if k.lower() not in excluded_headers
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
# we want a 200 b/c we have content via the cache
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
from tempfile import NamedTemporaryFile
|
||||
import mmap
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
|
||||
class CallbackFileWrapper(object):
|
||||
class CallbackFileWrapper:
|
||||
"""
|
||||
Small wrapper around a fp object which will tee everything read into a
|
||||
buffer, and when that file is closed it will execute a callback with the
|
||||
|
|
@ -26,7 +27,7 @@ class CallbackFileWrapper(object):
|
|||
"""
|
||||
|
||||
def __init__(self, fp, callback):
|
||||
self.__buf = NamedTemporaryFile("rb+", delete=True)
|
||||
self.__buf = NamedTemporaryFile('rb+', delete=True)
|
||||
self.__fp = fp
|
||||
self.__callback = callback
|
||||
|
||||
|
|
@ -39,7 +40,7 @@ class CallbackFileWrapper(object):
|
|||
# self.__fp hasn't been set.
|
||||
#
|
||||
# [0] https://docs.python.org/2/reference/expressions.html#atom-identifiers
|
||||
fp = self.__getattribute__("_CallbackFileWrapper__fp")
|
||||
fp = self.__getattribute__('_CallbackFileWrapper__fp')
|
||||
return getattr(fp, name)
|
||||
|
||||
def __is_fp_closed(self):
|
||||
|
|
@ -63,7 +64,7 @@ class CallbackFileWrapper(object):
|
|||
if self.__callback:
|
||||
if self.__buf.tell() == 0:
|
||||
# Empty file:
|
||||
result = b""
|
||||
result = b''
|
||||
else:
|
||||
# Return the data without actually loading it into memory,
|
||||
# relying on Python's buffer API and mmap(). mmap() just gives
|
||||
|
|
@ -71,7 +72,7 @@ class CallbackFileWrapper(object):
|
|||
# doesn't result in duplicate memory use.
|
||||
self.__buf.seek(0, 0)
|
||||
result = memoryview(
|
||||
mmap.mmap(self.__buf.fileno(), 0, access=mmap.ACCESS_READ)
|
||||
mmap.mmap(self.__buf.fileno(), 0, access=mmap.ACCESS_READ),
|
||||
)
|
||||
self.__callback(result)
|
||||
|
||||
|
|
@ -99,7 +100,7 @@ class CallbackFileWrapper(object):
|
|||
|
||||
def _safe_read(self, amt):
|
||||
data = self.__fp._safe_read(amt)
|
||||
if amt == 2 and data == b"\r\n":
|
||||
if amt == 2 and data == b'\r\n':
|
||||
# urllib executes this read to toss the CRLF at the end
|
||||
# of the chunk.
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import time
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from email.utils import formatdate
|
||||
from email.utils import parsedate
|
||||
from email.utils import parsedate_tz
|
||||
|
||||
from email.utils import formatdate, parsedate, parsedate_tz
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
|
||||
TIME_FMT = '%a, %d %b %Y %H:%M:%S GMT'
|
||||
|
||||
|
||||
def expire_after(delta, date=None):
|
||||
|
|
@ -21,7 +23,7 @@ def datetime_to_header(dt):
|
|||
return formatdate(calendar.timegm(dt.timetuple()))
|
||||
|
||||
|
||||
class BaseHeuristic(object):
|
||||
class BaseHeuristic:
|
||||
|
||||
def warning(self, response):
|
||||
"""
|
||||
|
|
@ -50,7 +52,7 @@ class BaseHeuristic(object):
|
|||
response.headers.update(updated_headers)
|
||||
warning_header_value = self.warning(response)
|
||||
if warning_header_value is not None:
|
||||
response.headers.update({"Warning": warning_header_value})
|
||||
response.headers.update({'Warning': warning_header_value})
|
||||
|
||||
return response
|
||||
|
||||
|
|
@ -64,11 +66,11 @@ class OneDayCache(BaseHeuristic):
|
|||
def update_headers(self, response):
|
||||
headers = {}
|
||||
|
||||
if "expires" not in response.headers:
|
||||
date = parsedate(response.headers["date"])
|
||||
if 'expires' not in response.headers:
|
||||
date = parsedate(response.headers['date'])
|
||||
expires = expire_after(timedelta(days=1), date=datetime(*date[:6]))
|
||||
headers["expires"] = datetime_to_header(expires)
|
||||
headers["cache-control"] = "public"
|
||||
headers['expires'] = datetime_to_header(expires)
|
||||
headers['cache-control'] = 'public'
|
||||
return headers
|
||||
|
||||
|
||||
|
|
@ -82,10 +84,10 @@ class ExpiresAfter(BaseHeuristic):
|
|||
|
||||
def update_headers(self, response):
|
||||
expires = expire_after(self.delta)
|
||||
return {"expires": datetime_to_header(expires), "cache-control": "public"}
|
||||
return {'expires': datetime_to_header(expires), 'cache-control': 'public'}
|
||||
|
||||
def warning(self, response):
|
||||
tmpl = "110 - Automatically cached for %s. Response might be stale"
|
||||
tmpl = '110 - Automatically cached for %s. Response might be stale'
|
||||
return tmpl % self.delta
|
||||
|
||||
|
||||
|
|
@ -102,26 +104,26 @@ class LastModified(BaseHeuristic):
|
|||
Unlike mozilla we limit this to 24-hr.
|
||||
"""
|
||||
cacheable_by_default_statuses = {
|
||||
200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
|
||||
200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501,
|
||||
}
|
||||
|
||||
def update_headers(self, resp):
|
||||
headers = resp.headers
|
||||
|
||||
if "expires" in headers:
|
||||
if 'expires' in headers:
|
||||
return {}
|
||||
|
||||
if "cache-control" in headers and headers["cache-control"] != "public":
|
||||
if 'cache-control' in headers and headers['cache-control'] != 'public':
|
||||
return {}
|
||||
|
||||
if resp.status not in self.cacheable_by_default_statuses:
|
||||
return {}
|
||||
|
||||
if "date" not in headers or "last-modified" not in headers:
|
||||
if 'date' not in headers or 'last-modified' not in headers:
|
||||
return {}
|
||||
|
||||
date = calendar.timegm(parsedate_tz(headers["date"]))
|
||||
last_modified = parsedate(headers["last-modified"])
|
||||
date = calendar.timegm(parsedate_tz(headers['date']))
|
||||
last_modified = parsedate(headers['last-modified'])
|
||||
if date is None or last_modified is None:
|
||||
return {}
|
||||
|
||||
|
|
@ -133,7 +135,7 @@ class LastModified(BaseHeuristic):
|
|||
return {}
|
||||
|
||||
expires = date + freshness_lifetime
|
||||
return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))}
|
||||
return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))}
|
||||
|
||||
def warning(self, resp):
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
|
|
@ -10,21 +11,23 @@ import zlib
|
|||
from pip._vendor import msgpack
|
||||
from pip._vendor.requests.structures import CaseInsensitiveDict
|
||||
|
||||
from .compat import HTTPResponse, pickle, text_type
|
||||
from .compat import HTTPResponse
|
||||
from .compat import pickle
|
||||
from .compat import text_type
|
||||
|
||||
|
||||
def _b64_decode_bytes(b):
|
||||
return base64.b64decode(b.encode("ascii"))
|
||||
return base64.b64decode(b.encode('ascii'))
|
||||
|
||||
|
||||
def _b64_decode_str(s):
|
||||
return _b64_decode_bytes(s).decode("utf8")
|
||||
return _b64_decode_bytes(s).decode('utf8')
|
||||
|
||||
|
||||
_default_body_read = object()
|
||||
|
||||
|
||||
class Serializer(object):
|
||||
class Serializer:
|
||||
def dumps(self, request, response, body=None):
|
||||
response_headers = CaseInsensitiveDict(response.headers)
|
||||
|
||||
|
|
@ -43,31 +46,31 @@ class Serializer(object):
|
|||
# doesn't know the difference. Forcing these to unicode will be
|
||||
# enough to have msgpack know the difference.
|
||||
data = {
|
||||
u"response": {
|
||||
u"body": body,
|
||||
u"headers": dict(
|
||||
(text_type(k), text_type(v)) for k, v in response.headers.items()
|
||||
),
|
||||
u"status": response.status,
|
||||
u"version": response.version,
|
||||
u"reason": text_type(response.reason),
|
||||
u"strict": response.strict,
|
||||
u"decode_content": response.decode_content,
|
||||
}
|
||||
'response': {
|
||||
'body': body,
|
||||
'headers': {
|
||||
text_type(k): text_type(v) for k, v in response.headers.items()
|
||||
},
|
||||
'status': response.status,
|
||||
'version': response.version,
|
||||
'reason': text_type(response.reason),
|
||||
'strict': response.strict,
|
||||
'decode_content': response.decode_content,
|
||||
},
|
||||
}
|
||||
|
||||
# Construct our vary headers
|
||||
data[u"vary"] = {}
|
||||
if u"vary" in response_headers:
|
||||
varied_headers = response_headers[u"vary"].split(",")
|
||||
data['vary'] = {}
|
||||
if 'vary' in response_headers:
|
||||
varied_headers = response_headers['vary'].split(',')
|
||||
for header in varied_headers:
|
||||
header = text_type(header).strip()
|
||||
header_value = request.headers.get(header, None)
|
||||
if header_value is not None:
|
||||
header_value = text_type(header_value)
|
||||
data[u"vary"][header] = header_value
|
||||
data['vary'][header] = header_value
|
||||
|
||||
return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)])
|
||||
return b','.join([b'cc=4', msgpack.dumps(data, use_bin_type=True)])
|
||||
|
||||
def loads(self, request, data):
|
||||
# Short circuit if we've been given an empty set of data
|
||||
|
|
@ -77,22 +80,22 @@ class Serializer(object):
|
|||
# Determine what version of the serializer the data was serialized
|
||||
# with
|
||||
try:
|
||||
ver, data = data.split(b",", 1)
|
||||
ver, data = data.split(b',', 1)
|
||||
except ValueError:
|
||||
ver = b"cc=0"
|
||||
ver = b'cc=0'
|
||||
|
||||
# Make sure that our "ver" is actually a version and isn't a false
|
||||
# positive from a , being in the data stream.
|
||||
if ver[:3] != b"cc=":
|
||||
if ver[:3] != b'cc=':
|
||||
data = ver + data
|
||||
ver = b"cc=0"
|
||||
ver = b'cc=0'
|
||||
|
||||
# Get the version number out of the cc=N
|
||||
ver = ver.split(b"=", 1)[-1].decode("ascii")
|
||||
ver = ver.split(b'=', 1)[-1].decode('ascii')
|
||||
|
||||
# Dispatch to the actual load method for the given version
|
||||
try:
|
||||
return getattr(self, "_loads_v{}".format(ver))(request, data)
|
||||
return getattr(self, f'_loads_v{ver}')(request, data)
|
||||
|
||||
except AttributeError:
|
||||
# This is a version we don't have a loads function for, so we'll
|
||||
|
|
@ -107,22 +110,22 @@ class Serializer(object):
|
|||
# determine if the cached response is suitable for this request.
|
||||
# This case is also handled in the controller code when creating
|
||||
# a cache entry, but is left here for backwards compatibility.
|
||||
if "*" in cached.get("vary", {}):
|
||||
if '*' in cached.get('vary', {}):
|
||||
return
|
||||
|
||||
# Ensure that the Vary headers for the cached response match our
|
||||
# request
|
||||
for header, value in cached.get("vary", {}).items():
|
||||
for header, value in cached.get('vary', {}).items():
|
||||
if request.headers.get(header, None) != value:
|
||||
return
|
||||
|
||||
body_raw = cached["response"].pop("body")
|
||||
body_raw = cached['response'].pop('body')
|
||||
|
||||
headers = CaseInsensitiveDict(data=cached["response"]["headers"])
|
||||
if headers.get("transfer-encoding", "") == "chunked":
|
||||
headers.pop("transfer-encoding")
|
||||
headers = CaseInsensitiveDict(data=cached['response']['headers'])
|
||||
if headers.get('transfer-encoding', '') == 'chunked':
|
||||
headers.pop('transfer-encoding')
|
||||
|
||||
cached["response"]["headers"] = headers
|
||||
cached['response']['headers'] = headers
|
||||
|
||||
try:
|
||||
body = io.BytesIO(body_raw)
|
||||
|
|
@ -133,9 +136,9 @@ class Serializer(object):
|
|||
# fail with:
|
||||
#
|
||||
# TypeError: 'str' does not support the buffer interface
|
||||
body = io.BytesIO(body_raw.encode("utf8"))
|
||||
body = io.BytesIO(body_raw.encode('utf8'))
|
||||
|
||||
return HTTPResponse(body=body, preload_content=False, **cached["response"])
|
||||
return HTTPResponse(body=body, preload_content=False, **cached['response'])
|
||||
|
||||
def _loads_v0(self, request, data):
|
||||
# The original legacy cache data. This doesn't contain enough
|
||||
|
|
@ -153,21 +156,21 @@ class Serializer(object):
|
|||
|
||||
def _loads_v2(self, request, data):
|
||||
try:
|
||||
cached = json.loads(zlib.decompress(data).decode("utf8"))
|
||||
cached = json.loads(zlib.decompress(data).decode('utf8'))
|
||||
except (ValueError, zlib.error):
|
||||
return
|
||||
|
||||
# We need to decode the items that we've base64 encoded
|
||||
cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"])
|
||||
cached["response"]["headers"] = dict(
|
||||
(_b64_decode_str(k), _b64_decode_str(v))
|
||||
for k, v in cached["response"]["headers"].items()
|
||||
)
|
||||
cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"])
|
||||
cached["vary"] = dict(
|
||||
(_b64_decode_str(k), _b64_decode_str(v) if v is not None else v)
|
||||
for k, v in cached["vary"].items()
|
||||
)
|
||||
cached['response']['body'] = _b64_decode_bytes(cached['response']['body'])
|
||||
cached['response']['headers'] = {
|
||||
_b64_decode_str(k): _b64_decode_str(v)
|
||||
for k, v in cached['response']['headers'].items()
|
||||
}
|
||||
cached['response']['reason'] = _b64_decode_str(cached['response']['reason'])
|
||||
cached['vary'] = {
|
||||
_b64_decode_str(k): _b64_decode_str(v) if v is not None else v
|
||||
for k, v in cached['vary'].items()
|
||||
}
|
||||
|
||||
return self.prepare_response(request, cached)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# SPDX-FileCopyrightText: 2015 Eric Larson
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
from .adapter import CacheControlAdapter
|
||||
from .cache import DictCache
|
||||
|
|
@ -27,7 +28,7 @@ def CacheControl(
|
|||
controller_class=controller_class,
|
||||
cacheable_methods=cacheable_methods,
|
||||
)
|
||||
sess.mount("http://", adapter)
|
||||
sess.mount("https://", adapter)
|
||||
sess.mount('http://', adapter)
|
||||
sess.mount('https://', adapter)
|
||||
|
||||
return sess
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue