req_file.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551
  1. """
  2. Requirements file parsing
  3. """
  4. import logging
  5. import optparse
  6. import os
  7. import re
  8. import shlex
  9. import urllib.parse
  10. from optparse import Values
  11. from typing import (
  12. TYPE_CHECKING,
  13. Any,
  14. Callable,
  15. Dict,
  16. Generator,
  17. Iterable,
  18. List,
  19. NoReturn,
  20. Optional,
  21. Tuple,
  22. )
  23. from pip._internal.cli import cmdoptions
  24. from pip._internal.exceptions import InstallationError, RequirementsFileParseError
  25. from pip._internal.models.search_scope import SearchScope
  26. from pip._internal.utils.encoding import auto_decode
  27. if TYPE_CHECKING:
  28. from pip._internal.index.package_finder import PackageFinder
  29. from pip._internal.network.session import PipSession
  30. __all__ = ["parse_requirements"]
  31. ReqFileLines = Iterable[Tuple[int, str]]
  32. LineParser = Callable[[str], Tuple[str, Values]]
  33. SCHEME_RE = re.compile(r"^(http|https|file):", re.I)
  34. COMMENT_RE = re.compile(r"(^|\s+)#.*$")
  35. # Matches environment variable-style values in '${MY_VARIABLE_1}' with the
  36. # variable name consisting of only uppercase letters, digits or the '_'
  37. # (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
  38. # 2013 Edition.
  39. ENV_VAR_RE = re.compile(r"(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})")
  40. SUPPORTED_OPTIONS: List[Callable[..., optparse.Option]] = [
  41. cmdoptions.index_url,
  42. cmdoptions.extra_index_url,
  43. cmdoptions.no_index,
  44. cmdoptions.constraints,
  45. cmdoptions.requirements,
  46. cmdoptions.editable,
  47. cmdoptions.find_links,
  48. cmdoptions.no_binary,
  49. cmdoptions.only_binary,
  50. cmdoptions.prefer_binary,
  51. cmdoptions.require_hashes,
  52. cmdoptions.pre,
  53. cmdoptions.trusted_host,
  54. cmdoptions.use_new_feature,
  55. ]
  56. # options to be passed to requirements
  57. SUPPORTED_OPTIONS_REQ: List[Callable[..., optparse.Option]] = [
  58. cmdoptions.global_options,
  59. cmdoptions.hash,
  60. cmdoptions.config_settings,
  61. ]
  62. SUPPORTED_OPTIONS_EDITABLE_REQ: List[Callable[..., optparse.Option]] = [
  63. cmdoptions.config_settings,
  64. ]
  65. # the 'dest' string values
  66. SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]
  67. SUPPORTED_OPTIONS_EDITABLE_REQ_DEST = [
  68. str(o().dest) for o in SUPPORTED_OPTIONS_EDITABLE_REQ
  69. ]
  70. logger = logging.getLogger(__name__)
  71. class ParsedRequirement:
  72. def __init__(
  73. self,
  74. requirement: str,
  75. is_editable: bool,
  76. comes_from: str,
  77. constraint: bool,
  78. options: Optional[Dict[str, Any]] = None,
  79. line_source: Optional[str] = None,
  80. ) -> None:
  81. self.requirement = requirement
  82. self.is_editable = is_editable
  83. self.comes_from = comes_from
  84. self.options = options
  85. self.constraint = constraint
  86. self.line_source = line_source
  87. class ParsedLine:
  88. def __init__(
  89. self,
  90. filename: str,
  91. lineno: int,
  92. args: str,
  93. opts: Values,
  94. constraint: bool,
  95. ) -> None:
  96. self.filename = filename
  97. self.lineno = lineno
  98. self.opts = opts
  99. self.constraint = constraint
  100. if args:
  101. self.is_requirement = True
  102. self.is_editable = False
  103. self.requirement = args
  104. elif opts.editables:
  105. self.is_requirement = True
  106. self.is_editable = True
  107. # We don't support multiple -e on one line
  108. self.requirement = opts.editables[0]
  109. else:
  110. self.is_requirement = False
  111. def parse_requirements(
  112. filename: str,
  113. session: "PipSession",
  114. finder: Optional["PackageFinder"] = None,
  115. options: Optional[optparse.Values] = None,
  116. constraint: bool = False,
  117. ) -> Generator[ParsedRequirement, None, None]:
  118. """Parse a requirements file and yield ParsedRequirement instances.
  119. :param filename: Path or url of requirements file.
  120. :param session: PipSession instance.
  121. :param finder: Instance of pip.index.PackageFinder.
  122. :param options: cli options.
  123. :param constraint: If true, parsing a constraint file rather than
  124. requirements file.
  125. """
  126. line_parser = get_line_parser(finder)
  127. parser = RequirementsFileParser(session, line_parser)
  128. for parsed_line in parser.parse(filename, constraint):
  129. parsed_req = handle_line(
  130. parsed_line, options=options, finder=finder, session=session
  131. )
  132. if parsed_req is not None:
  133. yield parsed_req
  134. def preprocess(content: str) -> ReqFileLines:
  135. """Split, filter, and join lines, and return a line iterator
  136. :param content: the content of the requirements file
  137. """
  138. lines_enum: ReqFileLines = enumerate(content.splitlines(), start=1)
  139. lines_enum = join_lines(lines_enum)
  140. lines_enum = ignore_comments(lines_enum)
  141. lines_enum = expand_env_variables(lines_enum)
  142. return lines_enum
  143. def handle_requirement_line(
  144. line: ParsedLine,
  145. options: Optional[optparse.Values] = None,
  146. ) -> ParsedRequirement:
  147. # preserve for the nested code path
  148. line_comes_from = "{} {} (line {})".format(
  149. "-c" if line.constraint else "-r",
  150. line.filename,
  151. line.lineno,
  152. )
  153. assert line.is_requirement
  154. # get the options that apply to requirements
  155. if line.is_editable:
  156. supported_dest = SUPPORTED_OPTIONS_EDITABLE_REQ_DEST
  157. else:
  158. supported_dest = SUPPORTED_OPTIONS_REQ_DEST
  159. req_options = {}
  160. for dest in supported_dest:
  161. if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
  162. req_options[dest] = line.opts.__dict__[dest]
  163. line_source = f"line {line.lineno} of {line.filename}"
  164. return ParsedRequirement(
  165. requirement=line.requirement,
  166. is_editable=line.is_editable,
  167. comes_from=line_comes_from,
  168. constraint=line.constraint,
  169. options=req_options,
  170. line_source=line_source,
  171. )
  172. def handle_option_line(
  173. opts: Values,
  174. filename: str,
  175. lineno: int,
  176. finder: Optional["PackageFinder"] = None,
  177. options: Optional[optparse.Values] = None,
  178. session: Optional["PipSession"] = None,
  179. ) -> None:
  180. if opts.hashes:
  181. logger.warning(
  182. "%s line %s has --hash but no requirement, and will be ignored.",
  183. filename,
  184. lineno,
  185. )
  186. if options:
  187. # percolate options upward
  188. if opts.require_hashes:
  189. options.require_hashes = opts.require_hashes
  190. if opts.features_enabled:
  191. options.features_enabled.extend(
  192. f for f in opts.features_enabled if f not in options.features_enabled
  193. )
  194. # set finder options
  195. if finder:
  196. find_links = finder.find_links
  197. index_urls = finder.index_urls
  198. no_index = finder.search_scope.no_index
  199. if opts.no_index is True:
  200. no_index = True
  201. index_urls = []
  202. if opts.index_url and not no_index:
  203. index_urls = [opts.index_url]
  204. if opts.extra_index_urls and not no_index:
  205. index_urls.extend(opts.extra_index_urls)
  206. if opts.find_links:
  207. # FIXME: it would be nice to keep track of the source
  208. # of the find_links: support a find-links local path
  209. # relative to a requirements file.
  210. value = opts.find_links[0]
  211. req_dir = os.path.dirname(os.path.abspath(filename))
  212. relative_to_reqs_file = os.path.join(req_dir, value)
  213. if os.path.exists(relative_to_reqs_file):
  214. value = relative_to_reqs_file
  215. find_links.append(value)
  216. if session:
  217. # We need to update the auth urls in session
  218. session.update_index_urls(index_urls)
  219. search_scope = SearchScope(
  220. find_links=find_links,
  221. index_urls=index_urls,
  222. no_index=no_index,
  223. )
  224. finder.search_scope = search_scope
  225. if opts.pre:
  226. finder.set_allow_all_prereleases()
  227. if opts.prefer_binary:
  228. finder.set_prefer_binary()
  229. if session:
  230. for host in opts.trusted_hosts or []:
  231. source = f"line {lineno} of {filename}"
  232. session.add_trusted_host(host, source=source)
  233. def handle_line(
  234. line: ParsedLine,
  235. options: Optional[optparse.Values] = None,
  236. finder: Optional["PackageFinder"] = None,
  237. session: Optional["PipSession"] = None,
  238. ) -> Optional[ParsedRequirement]:
  239. """Handle a single parsed requirements line; This can result in
  240. creating/yielding requirements, or updating the finder.
  241. :param line: The parsed line to be processed.
  242. :param options: CLI options.
  243. :param finder: The finder - updated by non-requirement lines.
  244. :param session: The session - updated by non-requirement lines.
  245. Returns a ParsedRequirement object if the line is a requirement line,
  246. otherwise returns None.
  247. For lines that contain requirements, the only options that have an effect
  248. are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
  249. requirement. Other options from SUPPORTED_OPTIONS may be present, but are
  250. ignored.
  251. For lines that do not contain requirements, the only options that have an
  252. effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
  253. be present, but are ignored. These lines may contain multiple options
  254. (although our docs imply only one is supported), and all our parsed and
  255. affect the finder.
  256. """
  257. if line.is_requirement:
  258. parsed_req = handle_requirement_line(line, options)
  259. return parsed_req
  260. else:
  261. handle_option_line(
  262. line.opts,
  263. line.filename,
  264. line.lineno,
  265. finder,
  266. options,
  267. session,
  268. )
  269. return None
  270. class RequirementsFileParser:
  271. def __init__(
  272. self,
  273. session: "PipSession",
  274. line_parser: LineParser,
  275. ) -> None:
  276. self._session = session
  277. self._line_parser = line_parser
  278. def parse(
  279. self, filename: str, constraint: bool
  280. ) -> Generator[ParsedLine, None, None]:
  281. """Parse a given file, yielding parsed lines."""
  282. yield from self._parse_and_recurse(filename, constraint)
  283. def _parse_and_recurse(
  284. self, filename: str, constraint: bool
  285. ) -> Generator[ParsedLine, None, None]:
  286. for line in self._parse_file(filename, constraint):
  287. if not line.is_requirement and (
  288. line.opts.requirements or line.opts.constraints
  289. ):
  290. # parse a nested requirements file
  291. if line.opts.requirements:
  292. req_path = line.opts.requirements[0]
  293. nested_constraint = False
  294. else:
  295. req_path = line.opts.constraints[0]
  296. nested_constraint = True
  297. # original file is over http
  298. if SCHEME_RE.search(filename):
  299. # do a url join so relative paths work
  300. req_path = urllib.parse.urljoin(filename, req_path)
  301. # original file and nested file are paths
  302. elif not SCHEME_RE.search(req_path):
  303. # do a join so relative paths work
  304. req_path = os.path.join(
  305. os.path.dirname(filename),
  306. req_path,
  307. )
  308. yield from self._parse_and_recurse(req_path, nested_constraint)
  309. else:
  310. yield line
  311. def _parse_file(
  312. self, filename: str, constraint: bool
  313. ) -> Generator[ParsedLine, None, None]:
  314. _, content = get_file_content(filename, self._session)
  315. lines_enum = preprocess(content)
  316. for line_number, line in lines_enum:
  317. try:
  318. args_str, opts = self._line_parser(line)
  319. except OptionParsingError as e:
  320. # add offending line
  321. msg = f"Invalid requirement: {line}\n{e.msg}"
  322. raise RequirementsFileParseError(msg)
  323. yield ParsedLine(
  324. filename,
  325. line_number,
  326. args_str,
  327. opts,
  328. constraint,
  329. )
  330. def get_line_parser(finder: Optional["PackageFinder"]) -> LineParser:
  331. def parse_line(line: str) -> Tuple[str, Values]:
  332. # Build new parser for each line since it accumulates appendable
  333. # options.
  334. parser = build_parser()
  335. defaults = parser.get_default_values()
  336. defaults.index_url = None
  337. if finder:
  338. defaults.format_control = finder.format_control
  339. args_str, options_str = break_args_options(line)
  340. try:
  341. options = shlex.split(options_str)
  342. except ValueError as e:
  343. raise OptionParsingError(f"Could not split options: {options_str}") from e
  344. opts, _ = parser.parse_args(options, defaults)
  345. return args_str, opts
  346. return parse_line
  347. def break_args_options(line: str) -> Tuple[str, str]:
  348. """Break up the line into an args and options string. We only want to shlex
  349. (and then optparse) the options, not the args. args can contain markers
  350. which are corrupted by shlex.
  351. """
  352. tokens = line.split(" ")
  353. args = []
  354. options = tokens[:]
  355. for token in tokens:
  356. if token.startswith("-") or token.startswith("--"):
  357. break
  358. else:
  359. args.append(token)
  360. options.pop(0)
  361. return " ".join(args), " ".join(options)
  362. class OptionParsingError(Exception):
  363. def __init__(self, msg: str) -> None:
  364. self.msg = msg
  365. def build_parser() -> optparse.OptionParser:
  366. """
  367. Return a parser for parsing requirement lines
  368. """
  369. parser = optparse.OptionParser(add_help_option=False)
  370. option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
  371. for option_factory in option_factories:
  372. option = option_factory()
  373. parser.add_option(option)
  374. # By default optparse sys.exits on parsing errors. We want to wrap
  375. # that in our own exception.
  376. def parser_exit(self: Any, msg: str) -> "NoReturn":
  377. raise OptionParsingError(msg)
  378. # NOTE: mypy disallows assigning to a method
  379. # https://github.com/python/mypy/issues/2427
  380. parser.exit = parser_exit # type: ignore
  381. return parser
  382. def join_lines(lines_enum: ReqFileLines) -> ReqFileLines:
  383. """Joins a line ending in '\' with the previous line (except when following
  384. comments). The joined line takes on the index of the first line.
  385. """
  386. primary_line_number = None
  387. new_line: List[str] = []
  388. for line_number, line in lines_enum:
  389. if not line.endswith("\\") or COMMENT_RE.match(line):
  390. if COMMENT_RE.match(line):
  391. # this ensures comments are always matched later
  392. line = " " + line
  393. if new_line:
  394. new_line.append(line)
  395. assert primary_line_number is not None
  396. yield primary_line_number, "".join(new_line)
  397. new_line = []
  398. else:
  399. yield line_number, line
  400. else:
  401. if not new_line:
  402. primary_line_number = line_number
  403. new_line.append(line.strip("\\"))
  404. # last line contains \
  405. if new_line:
  406. assert primary_line_number is not None
  407. yield primary_line_number, "".join(new_line)
  408. # TODO: handle space after '\'.
  409. def ignore_comments(lines_enum: ReqFileLines) -> ReqFileLines:
  410. """
  411. Strips comments and filter empty lines.
  412. """
  413. for line_number, line in lines_enum:
  414. line = COMMENT_RE.sub("", line)
  415. line = line.strip()
  416. if line:
  417. yield line_number, line
  418. def expand_env_variables(lines_enum: ReqFileLines) -> ReqFileLines:
  419. """Replace all environment variables that can be retrieved via `os.getenv`.
  420. The only allowed format for environment variables defined in the
  421. requirement file is `${MY_VARIABLE_1}` to ensure two things:
  422. 1. Strings that contain a `$` aren't accidentally (partially) expanded.
  423. 2. Ensure consistency across platforms for requirement files.
  424. These points are the result of a discussion on the `github pull
  425. request #3514 <https://github.com/pypa/pip/pull/3514>`_.
  426. Valid characters in variable names follow the `POSIX standard
  427. <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
  428. to uppercase letter, digits and the `_` (underscore).
  429. """
  430. for line_number, line in lines_enum:
  431. for env_var, var_name in ENV_VAR_RE.findall(line):
  432. value = os.getenv(var_name)
  433. if not value:
  434. continue
  435. line = line.replace(env_var, value)
  436. yield line_number, line
  437. def get_file_content(url: str, session: "PipSession") -> Tuple[str, str]:
  438. """Gets the content of a file; it may be a filename, file: URL, or
  439. http: URL. Returns (location, content). Content is unicode.
  440. Respects # -*- coding: declarations on the retrieved files.
  441. :param url: File path or url.
  442. :param session: PipSession instance.
  443. """
  444. scheme = urllib.parse.urlsplit(url).scheme
  445. # Pip has special support for file:// URLs (LocalFSAdapter).
  446. if scheme in ["http", "https", "file"]:
  447. # Delay importing heavy network modules until absolutely necessary.
  448. from pip._internal.network.utils import raise_for_status
  449. resp = session.get(url)
  450. raise_for_status(resp)
  451. return resp.url, resp.text
  452. # Assume this is a bare path.
  453. try:
  454. with open(url, "rb") as f:
  455. content = auto_decode(f.read())
  456. except OSError as exc:
  457. raise InstallationError(f"Could not open requirements file: {exc}")
  458. return url, content