constructors.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560
  1. """Backing implementation for InstallRequirement's various constructors
  2. The idea here is that these formed a major chunk of InstallRequirement's size
  3. so, moving them and support code dedicated to them outside of that class
  4. helps creates for better understandability for the rest of the code.
  5. These are meant to be used elsewhere within pip to create instances of
  6. InstallRequirement.
  7. """
  8. import copy
  9. import logging
  10. import os
  11. import re
  12. from dataclasses import dataclass
  13. from typing import Collection, Dict, List, Optional, Set, Tuple, Union
  14. from pip._vendor.packaging.markers import Marker
  15. from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
  16. from pip._vendor.packaging.specifiers import Specifier
  17. from pip._internal.exceptions import InstallationError
  18. from pip._internal.models.index import PyPI, TestPyPI
  19. from pip._internal.models.link import Link
  20. from pip._internal.models.wheel import Wheel
  21. from pip._internal.req.req_file import ParsedRequirement
  22. from pip._internal.req.req_install import InstallRequirement
  23. from pip._internal.utils.filetypes import is_archive_file
  24. from pip._internal.utils.misc import is_installable_dir
  25. from pip._internal.utils.packaging import get_requirement
  26. from pip._internal.utils.urls import path_to_url
  27. from pip._internal.vcs import is_url, vcs
  28. __all__ = [
  29. "install_req_from_editable",
  30. "install_req_from_line",
  31. "parse_editable",
  32. ]
  33. logger = logging.getLogger(__name__)
  34. operators = Specifier._operators.keys()
  35. def _strip_extras(path: str) -> Tuple[str, Optional[str]]:
  36. m = re.match(r"^(.+)(\[[^\]]+\])$", path)
  37. extras = None
  38. if m:
  39. path_no_extras = m.group(1)
  40. extras = m.group(2)
  41. else:
  42. path_no_extras = path
  43. return path_no_extras, extras
  44. def convert_extras(extras: Optional[str]) -> Set[str]:
  45. if not extras:
  46. return set()
  47. return get_requirement("placeholder" + extras.lower()).extras
  48. def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requirement:
  49. """
  50. Returns a new requirement based on the given one, with the supplied extras. If the
  51. given requirement already has extras those are replaced (or dropped if no new extras
  52. are given).
  53. """
  54. match: Optional[re.Match[str]] = re.fullmatch(
  55. # see https://peps.python.org/pep-0508/#complete-grammar
  56. r"([\w\t .-]+)(\[[^\]]*\])?(.*)",
  57. str(req),
  58. flags=re.ASCII,
  59. )
  60. # ireq.req is a valid requirement so the regex should always match
  61. assert (
  62. match is not None
  63. ), f"regex match on requirement {req} failed, this should never happen"
  64. pre: Optional[str] = match.group(1)
  65. post: Optional[str] = match.group(3)
  66. assert (
  67. pre is not None and post is not None
  68. ), f"regex group selection for requirement {req} failed, this should never happen"
  69. extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else ""
  70. return get_requirement(f"{pre}{extras}{post}")
  71. def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
  72. """Parses an editable requirement into:
  73. - a requirement name
  74. - an URL
  75. - extras
  76. - editable options
  77. Accepted requirements:
  78. svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
  79. .[some_extra]
  80. """
  81. url = editable_req
  82. # If a file path is specified with extras, strip off the extras.
  83. url_no_extras, extras = _strip_extras(url)
  84. if os.path.isdir(url_no_extras):
  85. # Treating it as code that has already been checked out
  86. url_no_extras = path_to_url(url_no_extras)
  87. if url_no_extras.lower().startswith("file:"):
  88. package_name = Link(url_no_extras).egg_fragment
  89. if extras:
  90. return (
  91. package_name,
  92. url_no_extras,
  93. get_requirement("placeholder" + extras.lower()).extras,
  94. )
  95. else:
  96. return package_name, url_no_extras, set()
  97. for version_control in vcs:
  98. if url.lower().startswith(f"{version_control}:"):
  99. url = f"{version_control}+{url}"
  100. break
  101. link = Link(url)
  102. if not link.is_vcs:
  103. backends = ", ".join(vcs.all_schemes)
  104. raise InstallationError(
  105. f"{editable_req} is not a valid editable requirement. "
  106. f"It should either be a path to a local project or a VCS URL "
  107. f"(beginning with {backends})."
  108. )
  109. package_name = link.egg_fragment
  110. if not package_name:
  111. raise InstallationError(
  112. f"Could not detect requirement name for '{editable_req}', "
  113. "please specify one with #egg=your_package_name"
  114. )
  115. return package_name, url, set()
  116. def check_first_requirement_in_file(filename: str) -> None:
  117. """Check if file is parsable as a requirements file.
  118. This is heavily based on ``pkg_resources.parse_requirements``, but
  119. simplified to just check the first meaningful line.
  120. :raises InvalidRequirement: If the first meaningful line cannot be parsed
  121. as an requirement.
  122. """
  123. with open(filename, encoding="utf-8", errors="ignore") as f:
  124. # Create a steppable iterator, so we can handle \-continuations.
  125. lines = (
  126. line
  127. for line in (line.strip() for line in f)
  128. if line and not line.startswith("#") # Skip blank lines/comments.
  129. )
  130. for line in lines:
  131. # Drop comments -- a hash without a space may be in a URL.
  132. if " #" in line:
  133. line = line[: line.find(" #")]
  134. # If there is a line continuation, drop it, and append the next line.
  135. if line.endswith("\\"):
  136. line = line[:-2].strip() + next(lines, "")
  137. get_requirement(line)
  138. return
  139. def deduce_helpful_msg(req: str) -> str:
  140. """Returns helpful msg in case requirements file does not exist,
  141. or cannot be parsed.
  142. :params req: Requirements file path
  143. """
  144. if not os.path.exists(req):
  145. return f" File '{req}' does not exist."
  146. msg = " The path does exist. "
  147. # Try to parse and check if it is a requirements file.
  148. try:
  149. check_first_requirement_in_file(req)
  150. except InvalidRequirement:
  151. logger.debug("Cannot parse '%s' as requirements file", req)
  152. else:
  153. msg += (
  154. f"The argument you provided "
  155. f"({req}) appears to be a"
  156. f" requirements file. If that is the"
  157. f" case, use the '-r' flag to install"
  158. f" the packages specified within it."
  159. )
  160. return msg
  161. @dataclass(frozen=True)
  162. class RequirementParts:
  163. requirement: Optional[Requirement]
  164. link: Optional[Link]
  165. markers: Optional[Marker]
  166. extras: Set[str]
  167. def parse_req_from_editable(editable_req: str) -> RequirementParts:
  168. name, url, extras_override = parse_editable(editable_req)
  169. if name is not None:
  170. try:
  171. req: Optional[Requirement] = get_requirement(name)
  172. except InvalidRequirement as exc:
  173. raise InstallationError(f"Invalid requirement: {name!r}: {exc}")
  174. else:
  175. req = None
  176. link = Link(url)
  177. return RequirementParts(req, link, None, extras_override)
  178. # ---- The actual constructors follow ----
  179. def install_req_from_editable(
  180. editable_req: str,
  181. comes_from: Optional[Union[InstallRequirement, str]] = None,
  182. *,
  183. use_pep517: Optional[bool] = None,
  184. isolated: bool = False,
  185. global_options: Optional[List[str]] = None,
  186. hash_options: Optional[Dict[str, List[str]]] = None,
  187. constraint: bool = False,
  188. user_supplied: bool = False,
  189. permit_editable_wheels: bool = False,
  190. config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
  191. ) -> InstallRequirement:
  192. parts = parse_req_from_editable(editable_req)
  193. return InstallRequirement(
  194. parts.requirement,
  195. comes_from=comes_from,
  196. user_supplied=user_supplied,
  197. editable=True,
  198. permit_editable_wheels=permit_editable_wheels,
  199. link=parts.link,
  200. constraint=constraint,
  201. use_pep517=use_pep517,
  202. isolated=isolated,
  203. global_options=global_options,
  204. hash_options=hash_options,
  205. config_settings=config_settings,
  206. extras=parts.extras,
  207. )
  208. def _looks_like_path(name: str) -> bool:
  209. """Checks whether the string "looks like" a path on the filesystem.
  210. This does not check whether the target actually exists, only judge from the
  211. appearance.
  212. Returns true if any of the following conditions is true:
  213. * a path separator is found (either os.path.sep or os.path.altsep);
  214. * a dot is found (which represents the current directory).
  215. """
  216. if os.path.sep in name:
  217. return True
  218. if os.path.altsep is not None and os.path.altsep in name:
  219. return True
  220. if name.startswith("."):
  221. return True
  222. return False
  223. def _get_url_from_path(path: str, name: str) -> Optional[str]:
  224. """
  225. First, it checks whether a provided path is an installable directory. If it
  226. is, returns the path.
  227. If false, check if the path is an archive file (such as a .whl).
  228. The function checks if the path is a file. If false, if the path has
  229. an @, it will treat it as a PEP 440 URL requirement and return the path.
  230. """
  231. if _looks_like_path(name) and os.path.isdir(path):
  232. if is_installable_dir(path):
  233. return path_to_url(path)
  234. # TODO: The is_installable_dir test here might not be necessary
  235. # now that it is done in load_pyproject_toml too.
  236. raise InstallationError(
  237. f"Directory {name!r} is not installable. Neither 'setup.py' "
  238. "nor 'pyproject.toml' found."
  239. )
  240. if not is_archive_file(path):
  241. return None
  242. if os.path.isfile(path):
  243. return path_to_url(path)
  244. urlreq_parts = name.split("@", 1)
  245. if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]):
  246. # If the path contains '@' and the part before it does not look
  247. # like a path, try to treat it as a PEP 440 URL req instead.
  248. return None
  249. logger.warning(
  250. "Requirement %r looks like a filename, but the file does not exist",
  251. name,
  252. )
  253. return path_to_url(path)
  254. def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementParts:
  255. if is_url(name):
  256. marker_sep = "; "
  257. else:
  258. marker_sep = ";"
  259. if marker_sep in name:
  260. name, markers_as_string = name.split(marker_sep, 1)
  261. markers_as_string = markers_as_string.strip()
  262. if not markers_as_string:
  263. markers = None
  264. else:
  265. markers = Marker(markers_as_string)
  266. else:
  267. markers = None
  268. name = name.strip()
  269. req_as_string = None
  270. path = os.path.normpath(os.path.abspath(name))
  271. link = None
  272. extras_as_string = None
  273. if is_url(name):
  274. link = Link(name)
  275. else:
  276. p, extras_as_string = _strip_extras(path)
  277. url = _get_url_from_path(p, name)
  278. if url is not None:
  279. link = Link(url)
  280. # it's a local file, dir, or url
  281. if link:
  282. # Handle relative file URLs
  283. if link.scheme == "file" and re.search(r"\.\./", link.url):
  284. link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path))))
  285. # wheel file
  286. if link.is_wheel:
  287. wheel = Wheel(link.filename) # can raise InvalidWheelFilename
  288. req_as_string = f"{wheel.name}=={wheel.version}"
  289. else:
  290. # set the req to the egg fragment. when it's not there, this
  291. # will become an 'unnamed' requirement
  292. req_as_string = link.egg_fragment
  293. # a requirement specifier
  294. else:
  295. req_as_string = name
  296. extras = convert_extras(extras_as_string)
  297. def with_source(text: str) -> str:
  298. if not line_source:
  299. return text
  300. return f"{text} (from {line_source})"
  301. def _parse_req_string(req_as_string: str) -> Requirement:
  302. try:
  303. return get_requirement(req_as_string)
  304. except InvalidRequirement as exc:
  305. if os.path.sep in req_as_string:
  306. add_msg = "It looks like a path."
  307. add_msg += deduce_helpful_msg(req_as_string)
  308. elif "=" in req_as_string and not any(
  309. op in req_as_string for op in operators
  310. ):
  311. add_msg = "= is not a valid operator. Did you mean == ?"
  312. else:
  313. add_msg = ""
  314. msg = with_source(f"Invalid requirement: {req_as_string!r}: {exc}")
  315. if add_msg:
  316. msg += f"\nHint: {add_msg}"
  317. raise InstallationError(msg)
  318. if req_as_string is not None:
  319. req: Optional[Requirement] = _parse_req_string(req_as_string)
  320. else:
  321. req = None
  322. return RequirementParts(req, link, markers, extras)
  323. def install_req_from_line(
  324. name: str,
  325. comes_from: Optional[Union[str, InstallRequirement]] = None,
  326. *,
  327. use_pep517: Optional[bool] = None,
  328. isolated: bool = False,
  329. global_options: Optional[List[str]] = None,
  330. hash_options: Optional[Dict[str, List[str]]] = None,
  331. constraint: bool = False,
  332. line_source: Optional[str] = None,
  333. user_supplied: bool = False,
  334. config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
  335. ) -> InstallRequirement:
  336. """Creates an InstallRequirement from a name, which might be a
  337. requirement, directory containing 'setup.py', filename, or URL.
  338. :param line_source: An optional string describing where the line is from,
  339. for logging purposes in case of an error.
  340. """
  341. parts = parse_req_from_line(name, line_source)
  342. return InstallRequirement(
  343. parts.requirement,
  344. comes_from,
  345. link=parts.link,
  346. markers=parts.markers,
  347. use_pep517=use_pep517,
  348. isolated=isolated,
  349. global_options=global_options,
  350. hash_options=hash_options,
  351. config_settings=config_settings,
  352. constraint=constraint,
  353. extras=parts.extras,
  354. user_supplied=user_supplied,
  355. )
  356. def install_req_from_req_string(
  357. req_string: str,
  358. comes_from: Optional[InstallRequirement] = None,
  359. isolated: bool = False,
  360. use_pep517: Optional[bool] = None,
  361. user_supplied: bool = False,
  362. ) -> InstallRequirement:
  363. try:
  364. req = get_requirement(req_string)
  365. except InvalidRequirement as exc:
  366. raise InstallationError(f"Invalid requirement: {req_string!r}: {exc}")
  367. domains_not_allowed = [
  368. PyPI.file_storage_domain,
  369. TestPyPI.file_storage_domain,
  370. ]
  371. if (
  372. req.url
  373. and comes_from
  374. and comes_from.link
  375. and comes_from.link.netloc in domains_not_allowed
  376. ):
  377. # Explicitly disallow pypi packages that depend on external urls
  378. raise InstallationError(
  379. "Packages installed from PyPI cannot depend on packages "
  380. "which are not also hosted on PyPI.\n"
  381. f"{comes_from.name} depends on {req} "
  382. )
  383. return InstallRequirement(
  384. req,
  385. comes_from,
  386. isolated=isolated,
  387. use_pep517=use_pep517,
  388. user_supplied=user_supplied,
  389. )
  390. def install_req_from_parsed_requirement(
  391. parsed_req: ParsedRequirement,
  392. isolated: bool = False,
  393. use_pep517: Optional[bool] = None,
  394. user_supplied: bool = False,
  395. config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
  396. ) -> InstallRequirement:
  397. if parsed_req.is_editable:
  398. req = install_req_from_editable(
  399. parsed_req.requirement,
  400. comes_from=parsed_req.comes_from,
  401. use_pep517=use_pep517,
  402. constraint=parsed_req.constraint,
  403. isolated=isolated,
  404. user_supplied=user_supplied,
  405. config_settings=config_settings,
  406. )
  407. else:
  408. req = install_req_from_line(
  409. parsed_req.requirement,
  410. comes_from=parsed_req.comes_from,
  411. use_pep517=use_pep517,
  412. isolated=isolated,
  413. global_options=(
  414. parsed_req.options.get("global_options", [])
  415. if parsed_req.options
  416. else []
  417. ),
  418. hash_options=(
  419. parsed_req.options.get("hashes", {}) if parsed_req.options else {}
  420. ),
  421. constraint=parsed_req.constraint,
  422. line_source=parsed_req.line_source,
  423. user_supplied=user_supplied,
  424. config_settings=config_settings,
  425. )
  426. return req
  427. def install_req_from_link_and_ireq(
  428. link: Link, ireq: InstallRequirement
  429. ) -> InstallRequirement:
  430. return InstallRequirement(
  431. req=ireq.req,
  432. comes_from=ireq.comes_from,
  433. editable=ireq.editable,
  434. link=link,
  435. markers=ireq.markers,
  436. use_pep517=ireq.use_pep517,
  437. isolated=ireq.isolated,
  438. global_options=ireq.global_options,
  439. hash_options=ireq.hash_options,
  440. config_settings=ireq.config_settings,
  441. user_supplied=ireq.user_supplied,
  442. )
  443. def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement:
  444. """
  445. Creates a new InstallationRequirement using the given template but without
  446. any extras. Sets the original requirement as the new one's parent
  447. (comes_from).
  448. """
  449. return InstallRequirement(
  450. req=(
  451. _set_requirement_extras(ireq.req, set()) if ireq.req is not None else None
  452. ),
  453. comes_from=ireq,
  454. editable=ireq.editable,
  455. link=ireq.link,
  456. markers=ireq.markers,
  457. use_pep517=ireq.use_pep517,
  458. isolated=ireq.isolated,
  459. global_options=ireq.global_options,
  460. hash_options=ireq.hash_options,
  461. constraint=ireq.constraint,
  462. extras=[],
  463. config_settings=ireq.config_settings,
  464. user_supplied=ireq.user_supplied,
  465. permit_editable_wheels=ireq.permit_editable_wheels,
  466. )
  467. def install_req_extend_extras(
  468. ireq: InstallRequirement,
  469. extras: Collection[str],
  470. ) -> InstallRequirement:
  471. """
  472. Returns a copy of an installation requirement with some additional extras.
  473. Makes a shallow copy of the ireq object.
  474. """
  475. result = copy.copy(ireq)
  476. result.extras = {*ireq.extras, *extras}
  477. result.req = (
  478. _set_requirement_extras(ireq.req, result.extras)
  479. if ireq.req is not None
  480. else None
  481. )
  482. return result