scripts.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2013-2023 Vinay Sajip.
  4. # Licensed to the Python Software Foundation under a contributor agreement.
  5. # See LICENSE.txt and CONTRIBUTORS.txt.
  6. #
  7. from io import BytesIO
  8. import logging
  9. import os
  10. import re
  11. import struct
  12. import sys
  13. import time
  14. from zipfile import ZipInfo
  15. from .compat import sysconfig, detect_encoding, ZipFile
  16. from .resources import finder
  17. from .util import (FileOperator, get_export_entry, convert_path,
  18. get_executable, get_platform, in_venv)
  19. logger = logging.getLogger(__name__)
  20. _DEFAULT_MANIFEST = '''
  21. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  22. <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
  23. <assemblyIdentity version="1.0.0.0"
  24. processorArchitecture="X86"
  25. name="%s"
  26. type="win32"/>
  27. <!-- Identify the application security requirements. -->
  28. <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
  29. <security>
  30. <requestedPrivileges>
  31. <requestedExecutionLevel level="asInvoker" uiAccess="false"/>
  32. </requestedPrivileges>
  33. </security>
  34. </trustInfo>
  35. </assembly>'''.strip()
  36. # check if Python is called on the first line with this expression
  37. FIRST_LINE_RE = re.compile(b'^#!.*pythonw?[0-9.]*([ \t].*)?$')
  38. SCRIPT_TEMPLATE = r'''# -*- coding: utf-8 -*-
  39. import re
  40. import sys
  41. from %(module)s import %(import_name)s
  42. if __name__ == '__main__':
  43. sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
  44. sys.exit(%(func)s())
  45. '''
  46. # Pre-fetch the contents of all executable wrapper stubs.
  47. # This is to address https://github.com/pypa/pip/issues/12666.
  48. # When updating pip, we rename the old pip in place before installing the
  49. # new version. If we try to fetch a wrapper *after* that rename, the finder
  50. # machinery will be confused as the package is no longer available at the
  51. # location where it was imported from. So we load everything into memory in
  52. # advance.
  53. # Issue 31: don't hardcode an absolute package name, but
  54. # determine it relative to the current package
  55. distlib_package = __name__.rsplit('.', 1)[0]
  56. WRAPPERS = {
  57. r.name: r.bytes
  58. for r in finder(distlib_package).iterator("")
  59. if r.name.endswith(".exe")
  60. }
  61. def enquote_executable(executable):
  62. if ' ' in executable:
  63. # make sure we quote only the executable in case of env
  64. # for example /usr/bin/env "/dir with spaces/bin/jython"
  65. # instead of "/usr/bin/env /dir with spaces/bin/jython"
  66. # otherwise whole
  67. if executable.startswith('/usr/bin/env '):
  68. env, _executable = executable.split(' ', 1)
  69. if ' ' in _executable and not _executable.startswith('"'):
  70. executable = '%s "%s"' % (env, _executable)
  71. else:
  72. if not executable.startswith('"'):
  73. executable = '"%s"' % executable
  74. return executable
  75. # Keep the old name around (for now), as there is at least one project using it!
  76. _enquote_executable = enquote_executable
  77. class ScriptMaker(object):
  78. """
  79. A class to copy or create scripts from source scripts or callable
  80. specifications.
  81. """
  82. script_template = SCRIPT_TEMPLATE
  83. executable = None # for shebangs
  84. def __init__(self,
  85. source_dir,
  86. target_dir,
  87. add_launchers=True,
  88. dry_run=False,
  89. fileop=None):
  90. self.source_dir = source_dir
  91. self.target_dir = target_dir
  92. self.add_launchers = add_launchers
  93. self.force = False
  94. self.clobber = False
  95. # It only makes sense to set mode bits on POSIX.
  96. self.set_mode = (os.name == 'posix') or (os.name == 'java'
  97. and os._name == 'posix')
  98. self.variants = set(('', 'X.Y'))
  99. self._fileop = fileop or FileOperator(dry_run)
  100. self._is_nt = os.name == 'nt' or (os.name == 'java'
  101. and os._name == 'nt')
  102. self.version_info = sys.version_info
  103. def _get_alternate_executable(self, executable, options):
  104. if options.get('gui', False) and self._is_nt: # pragma: no cover
  105. dn, fn = os.path.split(executable)
  106. fn = fn.replace('python', 'pythonw')
  107. executable = os.path.join(dn, fn)
  108. return executable
  109. if sys.platform.startswith('java'): # pragma: no cover
  110. def _is_shell(self, executable):
  111. """
  112. Determine if the specified executable is a script
  113. (contains a #! line)
  114. """
  115. try:
  116. with open(executable) as fp:
  117. return fp.read(2) == '#!'
  118. except (OSError, IOError):
  119. logger.warning('Failed to open %s', executable)
  120. return False
  121. def _fix_jython_executable(self, executable):
  122. if self._is_shell(executable):
  123. # Workaround for Jython is not needed on Linux systems.
  124. import java
  125. if java.lang.System.getProperty('os.name') == 'Linux':
  126. return executable
  127. elif executable.lower().endswith('jython.exe'):
  128. # Use wrapper exe for Jython on Windows
  129. return executable
  130. return '/usr/bin/env %s' % executable
  131. def _build_shebang(self, executable, post_interp):
  132. """
  133. Build a shebang line. In the simple case (on Windows, or a shebang line
  134. which is not too long or contains spaces) use a simple formulation for
  135. the shebang. Otherwise, use /bin/sh as the executable, with a contrived
  136. shebang which allows the script to run either under Python or sh, using
  137. suitable quoting. Thanks to Harald Nordgren for his input.
  138. See also: http://www.in-ulm.de/~mascheck/various/shebang/#length
  139. https://hg.mozilla.org/mozilla-central/file/tip/mach
  140. """
  141. if os.name != 'posix':
  142. simple_shebang = True
  143. else:
  144. # Add 3 for '#!' prefix and newline suffix.
  145. shebang_length = len(executable) + len(post_interp) + 3
  146. if sys.platform == 'darwin':
  147. max_shebang_length = 512
  148. else:
  149. max_shebang_length = 127
  150. simple_shebang = ((b' ' not in executable)
  151. and (shebang_length <= max_shebang_length))
  152. if simple_shebang:
  153. result = b'#!' + executable + post_interp + b'\n'
  154. else:
  155. result = b'#!/bin/sh\n'
  156. result += b"'''exec' " + executable + post_interp + b' "$0" "$@"\n'
  157. result += b"' '''"
  158. return result
  159. def _get_shebang(self, encoding, post_interp=b'', options=None):
  160. enquote = True
  161. if self.executable:
  162. executable = self.executable
  163. enquote = False # assume this will be taken care of
  164. elif not sysconfig.is_python_build():
  165. executable = get_executable()
  166. elif in_venv(): # pragma: no cover
  167. executable = os.path.join(
  168. sysconfig.get_path('scripts'),
  169. 'python%s' % sysconfig.get_config_var('EXE'))
  170. else: # pragma: no cover
  171. if os.name == 'nt':
  172. # for Python builds from source on Windows, no Python executables with
  173. # a version suffix are created, so we use python.exe
  174. executable = os.path.join(
  175. sysconfig.get_config_var('BINDIR'),
  176. 'python%s' % (sysconfig.get_config_var('EXE')))
  177. else:
  178. executable = os.path.join(
  179. sysconfig.get_config_var('BINDIR'),
  180. 'python%s%s' % (sysconfig.get_config_var('VERSION'),
  181. sysconfig.get_config_var('EXE')))
  182. if options:
  183. executable = self._get_alternate_executable(executable, options)
  184. if sys.platform.startswith('java'): # pragma: no cover
  185. executable = self._fix_jython_executable(executable)
  186. # Normalise case for Windows - COMMENTED OUT
  187. # executable = os.path.normcase(executable)
  188. # N.B. The normalising operation above has been commented out: See
  189. # issue #124. Although paths in Windows are generally case-insensitive,
  190. # they aren't always. For example, a path containing a ẞ (which is a
  191. # LATIN CAPITAL LETTER SHARP S - U+1E9E) is normcased to ß (which is a
  192. # LATIN SMALL LETTER SHARP S' - U+00DF). The two are not considered by
  193. # Windows as equivalent in path names.
  194. # If the user didn't specify an executable, it may be necessary to
  195. # cater for executable paths with spaces (not uncommon on Windows)
  196. if enquote:
  197. executable = enquote_executable(executable)
  198. # Issue #51: don't use fsencode, since we later try to
  199. # check that the shebang is decodable using utf-8.
  200. executable = executable.encode('utf-8')
  201. # in case of IronPython, play safe and enable frames support
  202. if (sys.platform == 'cli' and '-X:Frames' not in post_interp
  203. and '-X:FullFrames' not in post_interp): # pragma: no cover
  204. post_interp += b' -X:Frames'
  205. shebang = self._build_shebang(executable, post_interp)
  206. # Python parser starts to read a script using UTF-8 until
  207. # it gets a #coding:xxx cookie. The shebang has to be the
  208. # first line of a file, the #coding:xxx cookie cannot be
  209. # written before. So the shebang has to be decodable from
  210. # UTF-8.
  211. try:
  212. shebang.decode('utf-8')
  213. except UnicodeDecodeError: # pragma: no cover
  214. raise ValueError('The shebang (%r) is not decodable from utf-8' %
  215. shebang)
  216. # If the script is encoded to a custom encoding (use a
  217. # #coding:xxx cookie), the shebang has to be decodable from
  218. # the script encoding too.
  219. if encoding != 'utf-8':
  220. try:
  221. shebang.decode(encoding)
  222. except UnicodeDecodeError: # pragma: no cover
  223. raise ValueError('The shebang (%r) is not decodable '
  224. 'from the script encoding (%r)' %
  225. (shebang, encoding))
  226. return shebang
  227. def _get_script_text(self, entry):
  228. return self.script_template % dict(
  229. module=entry.prefix,
  230. import_name=entry.suffix.split('.')[0],
  231. func=entry.suffix)
  232. manifest = _DEFAULT_MANIFEST
  233. def get_manifest(self, exename):
  234. base = os.path.basename(exename)
  235. return self.manifest % base
  236. def _write_script(self, names, shebang, script_bytes, filenames, ext):
  237. use_launcher = self.add_launchers and self._is_nt
  238. linesep = os.linesep.encode('utf-8')
  239. if not shebang.endswith(linesep):
  240. shebang += linesep
  241. if not use_launcher:
  242. script_bytes = shebang + script_bytes
  243. else: # pragma: no cover
  244. if ext == 'py':
  245. launcher = self._get_launcher('t')
  246. else:
  247. launcher = self._get_launcher('w')
  248. stream = BytesIO()
  249. with ZipFile(stream, 'w') as zf:
  250. source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH')
  251. if source_date_epoch:
  252. date_time = time.gmtime(int(source_date_epoch))[:6]
  253. zinfo = ZipInfo(filename='__main__.py',
  254. date_time=date_time)
  255. zf.writestr(zinfo, script_bytes)
  256. else:
  257. zf.writestr('__main__.py', script_bytes)
  258. zip_data = stream.getvalue()
  259. script_bytes = launcher + shebang + zip_data
  260. for name in names:
  261. outname = os.path.join(self.target_dir, name)
  262. if use_launcher: # pragma: no cover
  263. n, e = os.path.splitext(outname)
  264. if e.startswith('.py'):
  265. outname = n
  266. outname = '%s.exe' % outname
  267. try:
  268. self._fileop.write_binary_file(outname, script_bytes)
  269. except Exception:
  270. # Failed writing an executable - it might be in use.
  271. logger.warning('Failed to write executable - trying to '
  272. 'use .deleteme logic')
  273. dfname = '%s.deleteme' % outname
  274. if os.path.exists(dfname):
  275. os.remove(dfname) # Not allowed to fail here
  276. os.rename(outname, dfname) # nor here
  277. self._fileop.write_binary_file(outname, script_bytes)
  278. logger.debug('Able to replace executable using '
  279. '.deleteme logic')
  280. try:
  281. os.remove(dfname)
  282. except Exception:
  283. pass # still in use - ignore error
  284. else:
  285. if self._is_nt and not outname.endswith(
  286. '.' + ext): # pragma: no cover
  287. outname = '%s.%s' % (outname, ext)
  288. if os.path.exists(outname) and not self.clobber:
  289. logger.warning('Skipping existing file %s', outname)
  290. continue
  291. self._fileop.write_binary_file(outname, script_bytes)
  292. if self.set_mode:
  293. self._fileop.set_executable_mode([outname])
  294. filenames.append(outname)
  295. variant_separator = '-'
  296. def get_script_filenames(self, name):
  297. result = set()
  298. if '' in self.variants:
  299. result.add(name)
  300. if 'X' in self.variants:
  301. result.add('%s%s' % (name, self.version_info[0]))
  302. if 'X.Y' in self.variants:
  303. result.add('%s%s%s.%s' %
  304. (name, self.variant_separator, self.version_info[0],
  305. self.version_info[1]))
  306. return result
  307. def _make_script(self, entry, filenames, options=None):
  308. post_interp = b''
  309. if options:
  310. args = options.get('interpreter_args', [])
  311. if args:
  312. args = ' %s' % ' '.join(args)
  313. post_interp = args.encode('utf-8')
  314. shebang = self._get_shebang('utf-8', post_interp, options=options)
  315. script = self._get_script_text(entry).encode('utf-8')
  316. scriptnames = self.get_script_filenames(entry.name)
  317. if options and options.get('gui', False):
  318. ext = 'pyw'
  319. else:
  320. ext = 'py'
  321. self._write_script(scriptnames, shebang, script, filenames, ext)
  322. def _copy_script(self, script, filenames):
  323. adjust = False
  324. script = os.path.join(self.source_dir, convert_path(script))
  325. outname = os.path.join(self.target_dir, os.path.basename(script))
  326. if not self.force and not self._fileop.newer(script, outname):
  327. logger.debug('not copying %s (up-to-date)', script)
  328. return
  329. # Always open the file, but ignore failures in dry-run mode --
  330. # that way, we'll get accurate feedback if we can read the
  331. # script.
  332. try:
  333. f = open(script, 'rb')
  334. except IOError: # pragma: no cover
  335. if not self.dry_run:
  336. raise
  337. f = None
  338. else:
  339. first_line = f.readline()
  340. if not first_line: # pragma: no cover
  341. logger.warning('%s is an empty file (skipping)', script)
  342. return
  343. match = FIRST_LINE_RE.match(first_line.replace(b'\r\n', b'\n'))
  344. if match:
  345. adjust = True
  346. post_interp = match.group(1) or b''
  347. if not adjust:
  348. if f:
  349. f.close()
  350. self._fileop.copy_file(script, outname)
  351. if self.set_mode:
  352. self._fileop.set_executable_mode([outname])
  353. filenames.append(outname)
  354. else:
  355. logger.info('copying and adjusting %s -> %s', script,
  356. self.target_dir)
  357. if not self._fileop.dry_run:
  358. encoding, lines = detect_encoding(f.readline)
  359. f.seek(0)
  360. shebang = self._get_shebang(encoding, post_interp)
  361. if b'pythonw' in first_line: # pragma: no cover
  362. ext = 'pyw'
  363. else:
  364. ext = 'py'
  365. n = os.path.basename(outname)
  366. self._write_script([n], shebang, f.read(), filenames, ext)
  367. if f:
  368. f.close()
  369. @property
  370. def dry_run(self):
  371. return self._fileop.dry_run
  372. @dry_run.setter
  373. def dry_run(self, value):
  374. self._fileop.dry_run = value
  375. if os.name == 'nt' or (os.name == 'java'
  376. and os._name == 'nt'): # pragma: no cover
  377. # Executable launcher support.
  378. # Launchers are from https://bitbucket.org/vinay.sajip/simple_launcher/
  379. def _get_launcher(self, kind):
  380. if struct.calcsize('P') == 8: # 64-bit
  381. bits = '64'
  382. else:
  383. bits = '32'
  384. platform_suffix = '-arm' if get_platform() == 'win-arm64' else ''
  385. name = '%s%s%s.exe' % (kind, bits, platform_suffix)
  386. if name not in WRAPPERS:
  387. msg = ('Unable to find resource %s in package %s' %
  388. (name, distlib_package))
  389. raise ValueError(msg)
  390. return WRAPPERS[name]
  391. # Public API follows
  392. def make(self, specification, options=None):
  393. """
  394. Make a script.
  395. :param specification: The specification, which is either a valid export
  396. entry specification (to make a script from a
  397. callable) or a filename (to make a script by
  398. copying from a source location).
  399. :param options: A dictionary of options controlling script generation.
  400. :return: A list of all absolute pathnames written to.
  401. """
  402. filenames = []
  403. entry = get_export_entry(specification)
  404. if entry is None:
  405. self._copy_script(specification, filenames)
  406. else:
  407. self._make_script(entry, filenames, options=options)
  408. return filenames
  409. def make_multiple(self, specifications, options=None):
  410. """
  411. Take a list of specifications and make scripts from them,
  412. :param specifications: A list of specifications.
  413. :return: A list of all absolute pathnames written to,
  414. """
  415. filenames = []
  416. for specification in specifications:
  417. filenames.extend(self.make(specification, options))
  418. return filenames