utils.py 4.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798
  1. from typing import Dict, Generator
  2. from pip._vendor.requests.models import Response
  3. from pip._internal.exceptions import NetworkConnectionError
  4. # The following comments and HTTP headers were originally added by
  5. # Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
  6. #
  7. # We use Accept-Encoding: identity here because requests defaults to
  8. # accepting compressed responses. This breaks in a variety of ways
  9. # depending on how the server is configured.
  10. # - Some servers will notice that the file isn't a compressible file
  11. # and will leave the file alone and with an empty Content-Encoding
  12. # - Some servers will notice that the file is already compressed and
  13. # will leave the file alone, adding a Content-Encoding: gzip header
  14. # - Some servers won't notice anything at all and will take a file
  15. # that's already been compressed and compress it again, and set
  16. # the Content-Encoding: gzip header
  17. # By setting this to request only the identity encoding we're hoping
  18. # to eliminate the third case. Hopefully there does not exist a server
  19. # which when given a file will notice it is already compressed and that
  20. # you're not asking for a compressed file and will then decompress it
  21. # before sending because if that's the case I don't think it'll ever be
  22. # possible to make this work.
  23. HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"}
  24. DOWNLOAD_CHUNK_SIZE = 256 * 1024
  25. def raise_for_status(resp: Response) -> None:
  26. http_error_msg = ""
  27. if isinstance(resp.reason, bytes):
  28. # We attempt to decode utf-8 first because some servers
  29. # choose to localize their reason strings. If the string
  30. # isn't utf-8, we fall back to iso-8859-1 for all other
  31. # encodings.
  32. try:
  33. reason = resp.reason.decode("utf-8")
  34. except UnicodeDecodeError:
  35. reason = resp.reason.decode("iso-8859-1")
  36. else:
  37. reason = resp.reason
  38. if 400 <= resp.status_code < 500:
  39. http_error_msg = (
  40. f"{resp.status_code} Client Error: {reason} for url: {resp.url}"
  41. )
  42. elif 500 <= resp.status_code < 600:
  43. http_error_msg = (
  44. f"{resp.status_code} Server Error: {reason} for url: {resp.url}"
  45. )
  46. if http_error_msg:
  47. raise NetworkConnectionError(http_error_msg, response=resp)
  48. def response_chunks(
  49. response: Response, chunk_size: int = DOWNLOAD_CHUNK_SIZE
  50. ) -> Generator[bytes, None, None]:
  51. """Given a requests Response, provide the data chunks."""
  52. try:
  53. # Special case for urllib3.
  54. for chunk in response.raw.stream(
  55. chunk_size,
  56. # We use decode_content=False here because we don't
  57. # want urllib3 to mess with the raw bytes we get
  58. # from the server. If we decompress inside of
  59. # urllib3 then we cannot verify the checksum
  60. # because the checksum will be of the compressed
  61. # file. This breakage will only occur if the
  62. # server adds a Content-Encoding header, which
  63. # depends on how the server was configured:
  64. # - Some servers will notice that the file isn't a
  65. # compressible file and will leave the file alone
  66. # and with an empty Content-Encoding
  67. # - Some servers will notice that the file is
  68. # already compressed and will leave the file
  69. # alone and will add a Content-Encoding: gzip
  70. # header
  71. # - Some servers won't notice anything at all and
  72. # will take a file that's already been compressed
  73. # and compress it again and set the
  74. # Content-Encoding: gzip header
  75. #
  76. # By setting this not to decode automatically we
  77. # hope to eliminate problems with the second case.
  78. decode_content=False,
  79. ):
  80. yield chunk
  81. except AttributeError:
  82. # Standard file-like object.
  83. while True:
  84. chunk = response.raw.read(chunk_size)
  85. if not chunk:
  86. break
  87. yield chunk