req_uninstall.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633
  1. import functools
  2. import os
  3. import sys
  4. import sysconfig
  5. from importlib.util import cache_from_source
  6. from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Set, Tuple
  7. from pip._internal.exceptions import LegacyDistutilsInstall, UninstallMissingRecord
  8. from pip._internal.locations import get_bin_prefix, get_bin_user
  9. from pip._internal.metadata import BaseDistribution
  10. from pip._internal.utils.compat import WINDOWS
  11. from pip._internal.utils.egg_link import egg_link_path_from_location
  12. from pip._internal.utils.logging import getLogger, indent_log
  13. from pip._internal.utils.misc import ask, normalize_path, renames, rmtree
  14. from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
  15. from pip._internal.utils.virtualenv import running_under_virtualenv
  16. logger = getLogger(__name__)
  17. def _script_names(
  18. bin_dir: str, script_name: str, is_gui: bool
  19. ) -> Generator[str, None, None]:
  20. """Create the fully qualified name of the files created by
  21. {console,gui}_scripts for the given ``dist``.
  22. Returns the list of file names
  23. """
  24. exe_name = os.path.join(bin_dir, script_name)
  25. yield exe_name
  26. if not WINDOWS:
  27. return
  28. yield f"{exe_name}.exe"
  29. yield f"{exe_name}.exe.manifest"
  30. if is_gui:
  31. yield f"{exe_name}-script.pyw"
  32. else:
  33. yield f"{exe_name}-script.py"
  34. def _unique(
  35. fn: Callable[..., Generator[Any, None, None]]
  36. ) -> Callable[..., Generator[Any, None, None]]:
  37. @functools.wraps(fn)
  38. def unique(*args: Any, **kw: Any) -> Generator[Any, None, None]:
  39. seen: Set[Any] = set()
  40. for item in fn(*args, **kw):
  41. if item not in seen:
  42. seen.add(item)
  43. yield item
  44. return unique
  45. @_unique
  46. def uninstallation_paths(dist: BaseDistribution) -> Generator[str, None, None]:
  47. """
  48. Yield all the uninstallation paths for dist based on RECORD-without-.py[co]
  49. Yield paths to all the files in RECORD. For each .py file in RECORD, add
  50. the .pyc and .pyo in the same directory.
  51. UninstallPathSet.add() takes care of the __pycache__ .py[co].
  52. If RECORD is not found, raises an error,
  53. with possible information from the INSTALLER file.
  54. https://packaging.python.org/specifications/recording-installed-packages/
  55. """
  56. location = dist.location
  57. assert location is not None, "not installed"
  58. entries = dist.iter_declared_entries()
  59. if entries is None:
  60. raise UninstallMissingRecord(distribution=dist)
  61. for entry in entries:
  62. path = os.path.join(location, entry)
  63. yield path
  64. if path.endswith(".py"):
  65. dn, fn = os.path.split(path)
  66. base = fn[:-3]
  67. path = os.path.join(dn, base + ".pyc")
  68. yield path
  69. path = os.path.join(dn, base + ".pyo")
  70. yield path
  71. def compact(paths: Iterable[str]) -> Set[str]:
  72. """Compact a path set to contain the minimal number of paths
  73. necessary to contain all paths in the set. If /a/path/ and
  74. /a/path/to/a/file.txt are both in the set, leave only the
  75. shorter path."""
  76. sep = os.path.sep
  77. short_paths: Set[str] = set()
  78. for path in sorted(paths, key=len):
  79. should_skip = any(
  80. path.startswith(shortpath.rstrip("*"))
  81. and path[len(shortpath.rstrip("*").rstrip(sep))] == sep
  82. for shortpath in short_paths
  83. )
  84. if not should_skip:
  85. short_paths.add(path)
  86. return short_paths
  87. def compress_for_rename(paths: Iterable[str]) -> Set[str]:
  88. """Returns a set containing the paths that need to be renamed.
  89. This set may include directories when the original sequence of paths
  90. included every file on disk.
  91. """
  92. case_map = {os.path.normcase(p): p for p in paths}
  93. remaining = set(case_map)
  94. unchecked = sorted({os.path.split(p)[0] for p in case_map.values()}, key=len)
  95. wildcards: Set[str] = set()
  96. def norm_join(*a: str) -> str:
  97. return os.path.normcase(os.path.join(*a))
  98. for root in unchecked:
  99. if any(os.path.normcase(root).startswith(w) for w in wildcards):
  100. # This directory has already been handled.
  101. continue
  102. all_files: Set[str] = set()
  103. all_subdirs: Set[str] = set()
  104. for dirname, subdirs, files in os.walk(root):
  105. all_subdirs.update(norm_join(root, dirname, d) for d in subdirs)
  106. all_files.update(norm_join(root, dirname, f) for f in files)
  107. # If all the files we found are in our remaining set of files to
  108. # remove, then remove them from the latter set and add a wildcard
  109. # for the directory.
  110. if not (all_files - remaining):
  111. remaining.difference_update(all_files)
  112. wildcards.add(root + os.sep)
  113. return set(map(case_map.__getitem__, remaining)) | wildcards
  114. def compress_for_output_listing(paths: Iterable[str]) -> Tuple[Set[str], Set[str]]:
  115. """Returns a tuple of 2 sets of which paths to display to user
  116. The first set contains paths that would be deleted. Files of a package
  117. are not added and the top-level directory of the package has a '*' added
  118. at the end - to signify that all it's contents are removed.
  119. The second set contains files that would have been skipped in the above
  120. folders.
  121. """
  122. will_remove = set(paths)
  123. will_skip = set()
  124. # Determine folders and files
  125. folders = set()
  126. files = set()
  127. for path in will_remove:
  128. if path.endswith(".pyc"):
  129. continue
  130. if path.endswith("__init__.py") or ".dist-info" in path:
  131. folders.add(os.path.dirname(path))
  132. files.add(path)
  133. _normcased_files = set(map(os.path.normcase, files))
  134. folders = compact(folders)
  135. # This walks the tree using os.walk to not miss extra folders
  136. # that might get added.
  137. for folder in folders:
  138. for dirpath, _, dirfiles in os.walk(folder):
  139. for fname in dirfiles:
  140. if fname.endswith(".pyc"):
  141. continue
  142. file_ = os.path.join(dirpath, fname)
  143. if (
  144. os.path.isfile(file_)
  145. and os.path.normcase(file_) not in _normcased_files
  146. ):
  147. # We are skipping this file. Add it to the set.
  148. will_skip.add(file_)
  149. will_remove = files | {os.path.join(folder, "*") for folder in folders}
  150. return will_remove, will_skip
  151. class StashedUninstallPathSet:
  152. """A set of file rename operations to stash files while
  153. tentatively uninstalling them."""
  154. def __init__(self) -> None:
  155. # Mapping from source file root to [Adjacent]TempDirectory
  156. # for files under that directory.
  157. self._save_dirs: Dict[str, TempDirectory] = {}
  158. # (old path, new path) tuples for each move that may need
  159. # to be undone.
  160. self._moves: List[Tuple[str, str]] = []
  161. def _get_directory_stash(self, path: str) -> str:
  162. """Stashes a directory.
  163. Directories are stashed adjacent to their original location if
  164. possible, or else moved/copied into the user's temp dir."""
  165. try:
  166. save_dir: TempDirectory = AdjacentTempDirectory(path)
  167. except OSError:
  168. save_dir = TempDirectory(kind="uninstall")
  169. self._save_dirs[os.path.normcase(path)] = save_dir
  170. return save_dir.path
  171. def _get_file_stash(self, path: str) -> str:
  172. """Stashes a file.
  173. If no root has been provided, one will be created for the directory
  174. in the user's temp directory."""
  175. path = os.path.normcase(path)
  176. head, old_head = os.path.dirname(path), None
  177. save_dir = None
  178. while head != old_head:
  179. try:
  180. save_dir = self._save_dirs[head]
  181. break
  182. except KeyError:
  183. pass
  184. head, old_head = os.path.dirname(head), head
  185. else:
  186. # Did not find any suitable root
  187. head = os.path.dirname(path)
  188. save_dir = TempDirectory(kind="uninstall")
  189. self._save_dirs[head] = save_dir
  190. relpath = os.path.relpath(path, head)
  191. if relpath and relpath != os.path.curdir:
  192. return os.path.join(save_dir.path, relpath)
  193. return save_dir.path
  194. def stash(self, path: str) -> str:
  195. """Stashes the directory or file and returns its new location.
  196. Handle symlinks as files to avoid modifying the symlink targets.
  197. """
  198. path_is_dir = os.path.isdir(path) and not os.path.islink(path)
  199. if path_is_dir:
  200. new_path = self._get_directory_stash(path)
  201. else:
  202. new_path = self._get_file_stash(path)
  203. self._moves.append((path, new_path))
  204. if path_is_dir and os.path.isdir(new_path):
  205. # If we're moving a directory, we need to
  206. # remove the destination first or else it will be
  207. # moved to inside the existing directory.
  208. # We just created new_path ourselves, so it will
  209. # be removable.
  210. os.rmdir(new_path)
  211. renames(path, new_path)
  212. return new_path
  213. def commit(self) -> None:
  214. """Commits the uninstall by removing stashed files."""
  215. for save_dir in self._save_dirs.values():
  216. save_dir.cleanup()
  217. self._moves = []
  218. self._save_dirs = {}
  219. def rollback(self) -> None:
  220. """Undoes the uninstall by moving stashed files back."""
  221. for p in self._moves:
  222. logger.info("Moving to %s\n from %s", *p)
  223. for new_path, path in self._moves:
  224. try:
  225. logger.debug("Replacing %s from %s", new_path, path)
  226. if os.path.isfile(new_path) or os.path.islink(new_path):
  227. os.unlink(new_path)
  228. elif os.path.isdir(new_path):
  229. rmtree(new_path)
  230. renames(path, new_path)
  231. except OSError as ex:
  232. logger.error("Failed to restore %s", new_path)
  233. logger.debug("Exception: %s", ex)
  234. self.commit()
  235. @property
  236. def can_rollback(self) -> bool:
  237. return bool(self._moves)
  238. class UninstallPathSet:
  239. """A set of file paths to be removed in the uninstallation of a
  240. requirement."""
  241. def __init__(self, dist: BaseDistribution) -> None:
  242. self._paths: Set[str] = set()
  243. self._refuse: Set[str] = set()
  244. self._pth: Dict[str, UninstallPthEntries] = {}
  245. self._dist = dist
  246. self._moved_paths = StashedUninstallPathSet()
  247. # Create local cache of normalize_path results. Creating an UninstallPathSet
  248. # can result in hundreds/thousands of redundant calls to normalize_path with
  249. # the same args, which hurts performance.
  250. self._normalize_path_cached = functools.lru_cache(normalize_path)
  251. def _permitted(self, path: str) -> bool:
  252. """
  253. Return True if the given path is one we are permitted to
  254. remove/modify, False otherwise.
  255. """
  256. # aka is_local, but caching normalized sys.prefix
  257. if not running_under_virtualenv():
  258. return True
  259. return path.startswith(self._normalize_path_cached(sys.prefix))
  260. def add(self, path: str) -> None:
  261. head, tail = os.path.split(path)
  262. # we normalize the head to resolve parent directory symlinks, but not
  263. # the tail, since we only want to uninstall symlinks, not their targets
  264. path = os.path.join(self._normalize_path_cached(head), os.path.normcase(tail))
  265. if not os.path.exists(path):
  266. return
  267. if self._permitted(path):
  268. self._paths.add(path)
  269. else:
  270. self._refuse.add(path)
  271. # __pycache__ files can show up after 'installed-files.txt' is created,
  272. # due to imports
  273. if os.path.splitext(path)[1] == ".py":
  274. self.add(cache_from_source(path))
  275. def add_pth(self, pth_file: str, entry: str) -> None:
  276. pth_file = self._normalize_path_cached(pth_file)
  277. if self._permitted(pth_file):
  278. if pth_file not in self._pth:
  279. self._pth[pth_file] = UninstallPthEntries(pth_file)
  280. self._pth[pth_file].add(entry)
  281. else:
  282. self._refuse.add(pth_file)
  283. def remove(self, auto_confirm: bool = False, verbose: bool = False) -> None:
  284. """Remove paths in ``self._paths`` with confirmation (unless
  285. ``auto_confirm`` is True)."""
  286. if not self._paths:
  287. logger.info(
  288. "Can't uninstall '%s'. No files were found to uninstall.",
  289. self._dist.raw_name,
  290. )
  291. return
  292. dist_name_version = f"{self._dist.raw_name}-{self._dist.raw_version}"
  293. logger.info("Uninstalling %s:", dist_name_version)
  294. with indent_log():
  295. if auto_confirm or self._allowed_to_proceed(verbose):
  296. moved = self._moved_paths
  297. for_rename = compress_for_rename(self._paths)
  298. for path in sorted(compact(for_rename)):
  299. moved.stash(path)
  300. logger.verbose("Removing file or directory %s", path)
  301. for pth in self._pth.values():
  302. pth.remove()
  303. logger.info("Successfully uninstalled %s", dist_name_version)
  304. def _allowed_to_proceed(self, verbose: bool) -> bool:
  305. """Display which files would be deleted and prompt for confirmation"""
  306. def _display(msg: str, paths: Iterable[str]) -> None:
  307. if not paths:
  308. return
  309. logger.info(msg)
  310. with indent_log():
  311. for path in sorted(compact(paths)):
  312. logger.info(path)
  313. if not verbose:
  314. will_remove, will_skip = compress_for_output_listing(self._paths)
  315. else:
  316. # In verbose mode, display all the files that are going to be
  317. # deleted.
  318. will_remove = set(self._paths)
  319. will_skip = set()
  320. _display("Would remove:", will_remove)
  321. _display("Would not remove (might be manually added):", will_skip)
  322. _display("Would not remove (outside of prefix):", self._refuse)
  323. if verbose:
  324. _display("Will actually move:", compress_for_rename(self._paths))
  325. return ask("Proceed (Y/n)? ", ("y", "n", "")) != "n"
  326. def rollback(self) -> None:
  327. """Rollback the changes previously made by remove()."""
  328. if not self._moved_paths.can_rollback:
  329. logger.error(
  330. "Can't roll back %s; was not uninstalled",
  331. self._dist.raw_name,
  332. )
  333. return
  334. logger.info("Rolling back uninstall of %s", self._dist.raw_name)
  335. self._moved_paths.rollback()
  336. for pth in self._pth.values():
  337. pth.rollback()
  338. def commit(self) -> None:
  339. """Remove temporary save dir: rollback will no longer be possible."""
  340. self._moved_paths.commit()
  341. @classmethod
  342. def from_dist(cls, dist: BaseDistribution) -> "UninstallPathSet":
  343. dist_location = dist.location
  344. info_location = dist.info_location
  345. if dist_location is None:
  346. logger.info(
  347. "Not uninstalling %s since it is not installed",
  348. dist.canonical_name,
  349. )
  350. return cls(dist)
  351. normalized_dist_location = normalize_path(dist_location)
  352. if not dist.local:
  353. logger.info(
  354. "Not uninstalling %s at %s, outside environment %s",
  355. dist.canonical_name,
  356. normalized_dist_location,
  357. sys.prefix,
  358. )
  359. return cls(dist)
  360. if normalized_dist_location in {
  361. p
  362. for p in {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
  363. if p
  364. }:
  365. logger.info(
  366. "Not uninstalling %s at %s, as it is in the standard library.",
  367. dist.canonical_name,
  368. normalized_dist_location,
  369. )
  370. return cls(dist)
  371. paths_to_remove = cls(dist)
  372. develop_egg_link = egg_link_path_from_location(dist.raw_name)
  373. # Distribution is installed with metadata in a "flat" .egg-info
  374. # directory. This means it is not a modern .dist-info installation, an
  375. # egg, or legacy editable.
  376. setuptools_flat_installation = (
  377. dist.installed_with_setuptools_egg_info
  378. and info_location is not None
  379. and os.path.exists(info_location)
  380. # If dist is editable and the location points to a ``.egg-info``,
  381. # we are in fact in the legacy editable case.
  382. and not info_location.endswith(f"{dist.setuptools_filename}.egg-info")
  383. )
  384. # Uninstall cases order do matter as in the case of 2 installs of the
  385. # same package, pip needs to uninstall the currently detected version
  386. if setuptools_flat_installation:
  387. if info_location is not None:
  388. paths_to_remove.add(info_location)
  389. installed_files = dist.iter_declared_entries()
  390. if installed_files is not None:
  391. for installed_file in installed_files:
  392. paths_to_remove.add(os.path.join(dist_location, installed_file))
  393. # FIXME: need a test for this elif block
  394. # occurs with --single-version-externally-managed/--record outside
  395. # of pip
  396. elif dist.is_file("top_level.txt"):
  397. try:
  398. namespace_packages = dist.read_text("namespace_packages.txt")
  399. except FileNotFoundError:
  400. namespaces = []
  401. else:
  402. namespaces = namespace_packages.splitlines(keepends=False)
  403. for top_level_pkg in [
  404. p
  405. for p in dist.read_text("top_level.txt").splitlines()
  406. if p and p not in namespaces
  407. ]:
  408. path = os.path.join(dist_location, top_level_pkg)
  409. paths_to_remove.add(path)
  410. paths_to_remove.add(f"{path}.py")
  411. paths_to_remove.add(f"{path}.pyc")
  412. paths_to_remove.add(f"{path}.pyo")
  413. elif dist.installed_by_distutils:
  414. raise LegacyDistutilsInstall(distribution=dist)
  415. elif dist.installed_as_egg:
  416. # package installed by easy_install
  417. # We cannot match on dist.egg_name because it can slightly vary
  418. # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg
  419. paths_to_remove.add(dist_location)
  420. easy_install_egg = os.path.split(dist_location)[1]
  421. easy_install_pth = os.path.join(
  422. os.path.dirname(dist_location),
  423. "easy-install.pth",
  424. )
  425. paths_to_remove.add_pth(easy_install_pth, "./" + easy_install_egg)
  426. elif dist.installed_with_dist_info:
  427. for path in uninstallation_paths(dist):
  428. paths_to_remove.add(path)
  429. elif develop_egg_link:
  430. # PEP 660 modern editable is handled in the ``.dist-info`` case
  431. # above, so this only covers the setuptools-style editable.
  432. with open(develop_egg_link) as fh:
  433. link_pointer = os.path.normcase(fh.readline().strip())
  434. normalized_link_pointer = paths_to_remove._normalize_path_cached(
  435. link_pointer
  436. )
  437. assert os.path.samefile(
  438. normalized_link_pointer, normalized_dist_location
  439. ), (
  440. f"Egg-link {develop_egg_link} (to {link_pointer}) does not match "
  441. f"installed location of {dist.raw_name} (at {dist_location})"
  442. )
  443. paths_to_remove.add(develop_egg_link)
  444. easy_install_pth = os.path.join(
  445. os.path.dirname(develop_egg_link), "easy-install.pth"
  446. )
  447. paths_to_remove.add_pth(easy_install_pth, dist_location)
  448. else:
  449. logger.debug(
  450. "Not sure how to uninstall: %s - Check: %s",
  451. dist,
  452. dist_location,
  453. )
  454. if dist.in_usersite:
  455. bin_dir = get_bin_user()
  456. else:
  457. bin_dir = get_bin_prefix()
  458. # find distutils scripts= scripts
  459. try:
  460. for script in dist.iter_distutils_script_names():
  461. paths_to_remove.add(os.path.join(bin_dir, script))
  462. if WINDOWS:
  463. paths_to_remove.add(os.path.join(bin_dir, f"{script}.bat"))
  464. except (FileNotFoundError, NotADirectoryError):
  465. pass
  466. # find console_scripts and gui_scripts
  467. def iter_scripts_to_remove(
  468. dist: BaseDistribution,
  469. bin_dir: str,
  470. ) -> Generator[str, None, None]:
  471. for entry_point in dist.iter_entry_points():
  472. if entry_point.group == "console_scripts":
  473. yield from _script_names(bin_dir, entry_point.name, False)
  474. elif entry_point.group == "gui_scripts":
  475. yield from _script_names(bin_dir, entry_point.name, True)
  476. for s in iter_scripts_to_remove(dist, bin_dir):
  477. paths_to_remove.add(s)
  478. return paths_to_remove
  479. class UninstallPthEntries:
  480. def __init__(self, pth_file: str) -> None:
  481. self.file = pth_file
  482. self.entries: Set[str] = set()
  483. self._saved_lines: Optional[List[bytes]] = None
  484. def add(self, entry: str) -> None:
  485. entry = os.path.normcase(entry)
  486. # On Windows, os.path.normcase converts the entry to use
  487. # backslashes. This is correct for entries that describe absolute
  488. # paths outside of site-packages, but all the others use forward
  489. # slashes.
  490. # os.path.splitdrive is used instead of os.path.isabs because isabs
  491. # treats non-absolute paths with drive letter markings like c:foo\bar
  492. # as absolute paths. It also does not recognize UNC paths if they don't
  493. # have more than "\\sever\share". Valid examples: "\\server\share\" or
  494. # "\\server\share\folder".
  495. if WINDOWS and not os.path.splitdrive(entry)[0]:
  496. entry = entry.replace("\\", "/")
  497. self.entries.add(entry)
  498. def remove(self) -> None:
  499. logger.verbose("Removing pth entries from %s:", self.file)
  500. # If the file doesn't exist, log a warning and return
  501. if not os.path.isfile(self.file):
  502. logger.warning("Cannot remove entries from nonexistent file %s", self.file)
  503. return
  504. with open(self.file, "rb") as fh:
  505. # windows uses '\r\n' with py3k, but uses '\n' with py2.x
  506. lines = fh.readlines()
  507. self._saved_lines = lines
  508. if any(b"\r\n" in line for line in lines):
  509. endline = "\r\n"
  510. else:
  511. endline = "\n"
  512. # handle missing trailing newline
  513. if lines and not lines[-1].endswith(endline.encode("utf-8")):
  514. lines[-1] = lines[-1] + endline.encode("utf-8")
  515. for entry in self.entries:
  516. try:
  517. logger.verbose("Removing entry: %s", entry)
  518. lines.remove((entry + endline).encode("utf-8"))
  519. except ValueError:
  520. pass
  521. with open(self.file, "wb") as fh:
  522. fh.writelines(lines)
  523. def rollback(self) -> bool:
  524. if self._saved_lines is None:
  525. logger.error("Cannot roll back changes to %s, none were made", self.file)
  526. return False
  527. logger.debug("Rolling %s back to previous state", self.file)
  528. with open(self.file, "wb") as fh:
  529. fh.writelines(self._saved_lines)
  530. return True