controller.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499
  1. # SPDX-FileCopyrightText: 2015 Eric Larson
  2. #
  3. # SPDX-License-Identifier: Apache-2.0
  4. """
  5. The httplib2 algorithms ported for use with requests.
  6. """
  7. from __future__ import annotations
  8. import calendar
  9. import logging
  10. import re
  11. import time
  12. from email.utils import parsedate_tz
  13. from typing import TYPE_CHECKING, Collection, Mapping
  14. from pip._vendor.requests.structures import CaseInsensitiveDict
  15. from pip._vendor.cachecontrol.cache import DictCache, SeparateBodyBaseCache
  16. from pip._vendor.cachecontrol.serialize import Serializer
  17. if TYPE_CHECKING:
  18. from typing import Literal
  19. from pip._vendor.requests import PreparedRequest
  20. from pip._vendor.urllib3 import HTTPResponse
  21. from pip._vendor.cachecontrol.cache import BaseCache
  22. logger = logging.getLogger(__name__)
  23. URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
  24. PERMANENT_REDIRECT_STATUSES = (301, 308)
  25. def parse_uri(uri: str) -> tuple[str, str, str, str, str]:
  26. """Parses a URI using the regex given in Appendix B of RFC 3986.
  27. (scheme, authority, path, query, fragment) = parse_uri(uri)
  28. """
  29. match = URI.match(uri)
  30. assert match is not None
  31. groups = match.groups()
  32. return (groups[1], groups[3], groups[4], groups[6], groups[8])
  33. class CacheController:
  34. """An interface to see if request should cached or not."""
  35. def __init__(
  36. self,
  37. cache: BaseCache | None = None,
  38. cache_etags: bool = True,
  39. serializer: Serializer | None = None,
  40. status_codes: Collection[int] | None = None,
  41. ):
  42. self.cache = DictCache() if cache is None else cache
  43. self.cache_etags = cache_etags
  44. self.serializer = serializer or Serializer()
  45. self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308)
  46. @classmethod
  47. def _urlnorm(cls, uri: str) -> str:
  48. """Normalize the URL to create a safe key for the cache"""
  49. (scheme, authority, path, query, fragment) = parse_uri(uri)
  50. if not scheme or not authority:
  51. raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
  52. scheme = scheme.lower()
  53. authority = authority.lower()
  54. if not path:
  55. path = "/"
  56. # Could do syntax based normalization of the URI before
  57. # computing the digest. See Section 6.2.2 of Std 66.
  58. request_uri = query and "?".join([path, query]) or path
  59. defrag_uri = scheme + "://" + authority + request_uri
  60. return defrag_uri
  61. @classmethod
  62. def cache_url(cls, uri: str) -> str:
  63. return cls._urlnorm(uri)
  64. def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | None]:
  65. known_directives = {
  66. # https://tools.ietf.org/html/rfc7234#section-5.2
  67. "max-age": (int, True),
  68. "max-stale": (int, False),
  69. "min-fresh": (int, True),
  70. "no-cache": (None, False),
  71. "no-store": (None, False),
  72. "no-transform": (None, False),
  73. "only-if-cached": (None, False),
  74. "must-revalidate": (None, False),
  75. "public": (None, False),
  76. "private": (None, False),
  77. "proxy-revalidate": (None, False),
  78. "s-maxage": (int, True),
  79. }
  80. cc_headers = headers.get("cache-control", headers.get("Cache-Control", ""))
  81. retval: dict[str, int | None] = {}
  82. for cc_directive in cc_headers.split(","):
  83. if not cc_directive.strip():
  84. continue
  85. parts = cc_directive.split("=", 1)
  86. directive = parts[0].strip()
  87. try:
  88. typ, required = known_directives[directive]
  89. except KeyError:
  90. logger.debug("Ignoring unknown cache-control directive: %s", directive)
  91. continue
  92. if not typ or not required:
  93. retval[directive] = None
  94. if typ:
  95. try:
  96. retval[directive] = typ(parts[1].strip())
  97. except IndexError:
  98. if required:
  99. logger.debug(
  100. "Missing value for cache-control " "directive: %s",
  101. directive,
  102. )
  103. except ValueError:
  104. logger.debug(
  105. "Invalid value for cache-control directive " "%s, must be %s",
  106. directive,
  107. typ.__name__,
  108. )
  109. return retval
  110. def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None:
  111. """
  112. Load a cached response, or return None if it's not available.
  113. """
  114. # We do not support caching of partial content: so if the request contains a
  115. # Range header then we don't want to load anything from the cache.
  116. if "Range" in request.headers:
  117. return None
  118. cache_url = request.url
  119. assert cache_url is not None
  120. cache_data = self.cache.get(cache_url)
  121. if cache_data is None:
  122. logger.debug("No cache entry available")
  123. return None
  124. if isinstance(self.cache, SeparateBodyBaseCache):
  125. body_file = self.cache.get_body(cache_url)
  126. else:
  127. body_file = None
  128. result = self.serializer.loads(request, cache_data, body_file)
  129. if result is None:
  130. logger.warning("Cache entry deserialization failed, entry ignored")
  131. return result
  132. def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[False]:
  133. """
  134. Return a cached response if it exists in the cache, otherwise
  135. return False.
  136. """
  137. assert request.url is not None
  138. cache_url = self.cache_url(request.url)
  139. logger.debug('Looking up "%s" in the cache', cache_url)
  140. cc = self.parse_cache_control(request.headers)
  141. # Bail out if the request insists on fresh data
  142. if "no-cache" in cc:
  143. logger.debug('Request header has "no-cache", cache bypassed')
  144. return False
  145. if "max-age" in cc and cc["max-age"] == 0:
  146. logger.debug('Request header has "max_age" as 0, cache bypassed')
  147. return False
  148. # Check whether we can load the response from the cache:
  149. resp = self._load_from_cache(request)
  150. if not resp:
  151. return False
  152. # If we have a cached permanent redirect, return it immediately. We
  153. # don't need to test our response for other headers b/c it is
  154. # intrinsically "cacheable" as it is Permanent.
  155. #
  156. # See:
  157. # https://tools.ietf.org/html/rfc7231#section-6.4.2
  158. #
  159. # Client can try to refresh the value by repeating the request
  160. # with cache busting headers as usual (ie no-cache).
  161. if int(resp.status) in PERMANENT_REDIRECT_STATUSES:
  162. msg = (
  163. "Returning cached permanent redirect response "
  164. "(ignoring date and etag information)"
  165. )
  166. logger.debug(msg)
  167. return resp
  168. headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers)
  169. if not headers or "date" not in headers:
  170. if "etag" not in headers:
  171. # Without date or etag, the cached response can never be used
  172. # and should be deleted.
  173. logger.debug("Purging cached response: no date or etag")
  174. self.cache.delete(cache_url)
  175. logger.debug("Ignoring cached response: no date")
  176. return False
  177. now = time.time()
  178. time_tuple = parsedate_tz(headers["date"])
  179. assert time_tuple is not None
  180. date = calendar.timegm(time_tuple[:6])
  181. current_age = max(0, now - date)
  182. logger.debug("Current age based on date: %i", current_age)
  183. # TODO: There is an assumption that the result will be a
  184. # urllib3 response object. This may not be best since we
  185. # could probably avoid instantiating or constructing the
  186. # response until we know we need it.
  187. resp_cc = self.parse_cache_control(headers)
  188. # determine freshness
  189. freshness_lifetime = 0
  190. # Check the max-age pragma in the cache control header
  191. max_age = resp_cc.get("max-age")
  192. if max_age is not None:
  193. freshness_lifetime = max_age
  194. logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime)
  195. # If there isn't a max-age, check for an expires header
  196. elif "expires" in headers:
  197. expires = parsedate_tz(headers["expires"])
  198. if expires is not None:
  199. expire_time = calendar.timegm(expires[:6]) - date
  200. freshness_lifetime = max(0, expire_time)
  201. logger.debug("Freshness lifetime from expires: %i", freshness_lifetime)
  202. # Determine if we are setting freshness limit in the
  203. # request. Note, this overrides what was in the response.
  204. max_age = cc.get("max-age")
  205. if max_age is not None:
  206. freshness_lifetime = max_age
  207. logger.debug(
  208. "Freshness lifetime from request max-age: %i", freshness_lifetime
  209. )
  210. min_fresh = cc.get("min-fresh")
  211. if min_fresh is not None:
  212. # adjust our current age by our min fresh
  213. current_age += min_fresh
  214. logger.debug("Adjusted current age from min-fresh: %i", current_age)
  215. # Return entry if it is fresh enough
  216. if freshness_lifetime > current_age:
  217. logger.debug('The response is "fresh", returning cached response')
  218. logger.debug("%i > %i", freshness_lifetime, current_age)
  219. return resp
  220. # we're not fresh. If we don't have an Etag, clear it out
  221. if "etag" not in headers:
  222. logger.debug('The cached response is "stale" with no etag, purging')
  223. self.cache.delete(cache_url)
  224. # return the original handler
  225. return False
  226. def conditional_headers(self, request: PreparedRequest) -> dict[str, str]:
  227. resp = self._load_from_cache(request)
  228. new_headers = {}
  229. if resp:
  230. headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers)
  231. if "etag" in headers:
  232. new_headers["If-None-Match"] = headers["ETag"]
  233. if "last-modified" in headers:
  234. new_headers["If-Modified-Since"] = headers["Last-Modified"]
  235. return new_headers
  236. def _cache_set(
  237. self,
  238. cache_url: str,
  239. request: PreparedRequest,
  240. response: HTTPResponse,
  241. body: bytes | None = None,
  242. expires_time: int | None = None,
  243. ) -> None:
  244. """
  245. Store the data in the cache.
  246. """
  247. if isinstance(self.cache, SeparateBodyBaseCache):
  248. # We pass in the body separately; just put a placeholder empty
  249. # string in the metadata.
  250. self.cache.set(
  251. cache_url,
  252. self.serializer.dumps(request, response, b""),
  253. expires=expires_time,
  254. )
  255. # body is None can happen when, for example, we're only updating
  256. # headers, as is the case in update_cached_response().
  257. if body is not None:
  258. self.cache.set_body(cache_url, body)
  259. else:
  260. self.cache.set(
  261. cache_url,
  262. self.serializer.dumps(request, response, body),
  263. expires=expires_time,
  264. )
  265. def cache_response(
  266. self,
  267. request: PreparedRequest,
  268. response: HTTPResponse,
  269. body: bytes | None = None,
  270. status_codes: Collection[int] | None = None,
  271. ) -> None:
  272. """
  273. Algorithm for caching requests.
  274. This assumes a requests Response object.
  275. """
  276. # From httplib2: Don't cache 206's since we aren't going to
  277. # handle byte range requests
  278. cacheable_status_codes = status_codes or self.cacheable_status_codes
  279. if response.status not in cacheable_status_codes:
  280. logger.debug(
  281. "Status code %s not in %s", response.status, cacheable_status_codes
  282. )
  283. return
  284. response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
  285. response.headers
  286. )
  287. if "date" in response_headers:
  288. time_tuple = parsedate_tz(response_headers["date"])
  289. assert time_tuple is not None
  290. date = calendar.timegm(time_tuple[:6])
  291. else:
  292. date = 0
  293. # If we've been given a body, our response has a Content-Length, that
  294. # Content-Length is valid then we can check to see if the body we've
  295. # been given matches the expected size, and if it doesn't we'll just
  296. # skip trying to cache it.
  297. if (
  298. body is not None
  299. and "content-length" in response_headers
  300. and response_headers["content-length"].isdigit()
  301. and int(response_headers["content-length"]) != len(body)
  302. ):
  303. return
  304. cc_req = self.parse_cache_control(request.headers)
  305. cc = self.parse_cache_control(response_headers)
  306. assert request.url is not None
  307. cache_url = self.cache_url(request.url)
  308. logger.debug('Updating cache with response from "%s"', cache_url)
  309. # Delete it from the cache if we happen to have it stored there
  310. no_store = False
  311. if "no-store" in cc:
  312. no_store = True
  313. logger.debug('Response header has "no-store"')
  314. if "no-store" in cc_req:
  315. no_store = True
  316. logger.debug('Request header has "no-store"')
  317. if no_store and self.cache.get(cache_url):
  318. logger.debug('Purging existing cache entry to honor "no-store"')
  319. self.cache.delete(cache_url)
  320. if no_store:
  321. return
  322. # https://tools.ietf.org/html/rfc7234#section-4.1:
  323. # A Vary header field-value of "*" always fails to match.
  324. # Storing such a response leads to a deserialization warning
  325. # during cache lookup and is not allowed to ever be served,
  326. # so storing it can be avoided.
  327. if "*" in response_headers.get("vary", ""):
  328. logger.debug('Response header has "Vary: *"')
  329. return
  330. # If we've been given an etag, then keep the response
  331. if self.cache_etags and "etag" in response_headers:
  332. expires_time = 0
  333. if response_headers.get("expires"):
  334. expires = parsedate_tz(response_headers["expires"])
  335. if expires is not None:
  336. expires_time = calendar.timegm(expires[:6]) - date
  337. expires_time = max(expires_time, 14 * 86400)
  338. logger.debug(f"etag object cached for {expires_time} seconds")
  339. logger.debug("Caching due to etag")
  340. self._cache_set(cache_url, request, response, body, expires_time)
  341. # Add to the cache any permanent redirects. We do this before looking
  342. # that the Date headers.
  343. elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
  344. logger.debug("Caching permanent redirect")
  345. self._cache_set(cache_url, request, response, b"")
  346. # Add to the cache if the response headers demand it. If there
  347. # is no date header then we can't do anything about expiring
  348. # the cache.
  349. elif "date" in response_headers:
  350. time_tuple = parsedate_tz(response_headers["date"])
  351. assert time_tuple is not None
  352. date = calendar.timegm(time_tuple[:6])
  353. # cache when there is a max-age > 0
  354. max_age = cc.get("max-age")
  355. if max_age is not None and max_age > 0:
  356. logger.debug("Caching b/c date exists and max-age > 0")
  357. expires_time = max_age
  358. self._cache_set(
  359. cache_url,
  360. request,
  361. response,
  362. body,
  363. expires_time,
  364. )
  365. # If the request can expire, it means we should cache it
  366. # in the meantime.
  367. elif "expires" in response_headers:
  368. if response_headers["expires"]:
  369. expires = parsedate_tz(response_headers["expires"])
  370. if expires is not None:
  371. expires_time = calendar.timegm(expires[:6]) - date
  372. else:
  373. expires_time = None
  374. logger.debug(
  375. "Caching b/c of expires header. expires in {} seconds".format(
  376. expires_time
  377. )
  378. )
  379. self._cache_set(
  380. cache_url,
  381. request,
  382. response,
  383. body,
  384. expires_time,
  385. )
  386. def update_cached_response(
  387. self, request: PreparedRequest, response: HTTPResponse
  388. ) -> HTTPResponse:
  389. """On a 304 we will get a new set of headers that we want to
  390. update our cached value with, assuming we have one.
  391. This should only ever be called when we've sent an ETag and
  392. gotten a 304 as the response.
  393. """
  394. assert request.url is not None
  395. cache_url = self.cache_url(request.url)
  396. cached_response = self._load_from_cache(request)
  397. if not cached_response:
  398. # we didn't have a cached response
  399. return response
  400. # Lets update our headers with the headers from the new request:
  401. # http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
  402. #
  403. # The server isn't supposed to send headers that would make
  404. # the cached body invalid. But... just in case, we'll be sure
  405. # to strip out ones we know that might be problmatic due to
  406. # typical assumptions.
  407. excluded_headers = ["content-length"]
  408. cached_response.headers.update(
  409. {
  410. k: v
  411. for k, v in response.headers.items()
  412. if k.lower() not in excluded_headers
  413. }
  414. )
  415. # we want a 200 b/c we have content via the cache
  416. cached_response.status = 200
  417. # update our cache
  418. self._cache_set(cache_url, request, cached_response)
  419. return cached_response