collectstatic.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. import os
  2. from django.apps import apps
  3. from django.contrib.staticfiles.finders import get_finders
  4. from django.contrib.staticfiles.storage import staticfiles_storage
  5. from django.core.checks import Tags
  6. from django.core.files.storage import FileSystemStorage
  7. from django.core.management.base import BaseCommand, CommandError
  8. from django.core.management.color import no_style
  9. from django.utils.functional import cached_property
  10. class Command(BaseCommand):
  11. """
  12. Copies or symlinks static files from different locations to the
  13. settings.STATIC_ROOT.
  14. """
  15. help = "Collect static files in a single location."
  16. requires_system_checks = [Tags.staticfiles]
  17. def __init__(self, *args, **kwargs):
  18. super().__init__(*args, **kwargs)
  19. self.copied_files = []
  20. self.symlinked_files = []
  21. self.unmodified_files = []
  22. self.post_processed_files = []
  23. self.storage = staticfiles_storage
  24. self.style = no_style()
  25. @cached_property
  26. def local(self):
  27. try:
  28. self.storage.path("")
  29. except NotImplementedError:
  30. return False
  31. return True
  32. def add_arguments(self, parser):
  33. parser.add_argument(
  34. "--noinput",
  35. "--no-input",
  36. action="store_false",
  37. dest="interactive",
  38. help="Do NOT prompt the user for input of any kind.",
  39. )
  40. parser.add_argument(
  41. "--no-post-process",
  42. action="store_false",
  43. dest="post_process",
  44. help="Do NOT post process collected files.",
  45. )
  46. parser.add_argument(
  47. "-i",
  48. "--ignore",
  49. action="append",
  50. default=[],
  51. dest="ignore_patterns",
  52. metavar="PATTERN",
  53. help="Ignore files or directories matching this glob-style "
  54. "pattern. Use multiple times to ignore more.",
  55. )
  56. parser.add_argument(
  57. "-n",
  58. "--dry-run",
  59. action="store_true",
  60. help="Do everything except modify the filesystem.",
  61. )
  62. parser.add_argument(
  63. "-c",
  64. "--clear",
  65. action="store_true",
  66. help="Clear the existing files using the storage "
  67. "before trying to copy or link the original file.",
  68. )
  69. parser.add_argument(
  70. "-l",
  71. "--link",
  72. action="store_true",
  73. help="Create a symbolic link to each file instead of copying.",
  74. )
  75. parser.add_argument(
  76. "--no-default-ignore",
  77. action="store_false",
  78. dest="use_default_ignore_patterns",
  79. help=(
  80. "Don't ignore the common private glob-style patterns (defaults to "
  81. "'CVS', '.*' and '*~')."
  82. ),
  83. )
  84. def set_options(self, **options):
  85. """
  86. Set instance variables based on an options dict
  87. """
  88. self.interactive = options["interactive"]
  89. self.verbosity = options["verbosity"]
  90. self.symlink = options["link"]
  91. self.clear = options["clear"]
  92. self.dry_run = options["dry_run"]
  93. ignore_patterns = options["ignore_patterns"]
  94. if options["use_default_ignore_patterns"]:
  95. ignore_patterns += apps.get_app_config("staticfiles").ignore_patterns
  96. self.ignore_patterns = list({os.path.normpath(p) for p in ignore_patterns})
  97. self.post_process = options["post_process"]
  98. def collect(self):
  99. """
  100. Perform the bulk of the work of collectstatic.
  101. Split off from handle() to facilitate testing.
  102. """
  103. if self.symlink and not self.local:
  104. raise CommandError("Can't symlink to a remote destination.")
  105. if self.clear:
  106. self.clear_dir("")
  107. if self.symlink:
  108. handler = self.link_file
  109. else:
  110. handler = self.copy_file
  111. found_files = {}
  112. for finder in get_finders():
  113. for path, storage in finder.list(self.ignore_patterns):
  114. # Prefix the relative path if the source storage contains it
  115. if getattr(storage, "prefix", None):
  116. prefixed_path = os.path.join(storage.prefix, path)
  117. else:
  118. prefixed_path = path
  119. if prefixed_path not in found_files:
  120. found_files[prefixed_path] = (storage, path)
  121. handler(path, prefixed_path, storage)
  122. else:
  123. self.log(
  124. "Found another file with the destination path '%s'. It "
  125. "will be ignored since only the first encountered file "
  126. "is collected. If this is not what you want, make sure "
  127. "every static file has a unique path." % prefixed_path,
  128. level=1,
  129. )
  130. # Storage backends may define a post_process() method.
  131. if self.post_process and hasattr(self.storage, "post_process"):
  132. processor = self.storage.post_process(found_files, dry_run=self.dry_run)
  133. for original_path, processed_path, processed in processor:
  134. if isinstance(processed, Exception):
  135. self.stderr.write("Post-processing '%s' failed!" % original_path)
  136. # Add a blank line before the traceback, otherwise it's
  137. # too easy to miss the relevant part of the error message.
  138. self.stderr.write()
  139. raise processed
  140. if processed:
  141. self.log(
  142. "Post-processed '%s' as '%s'" % (original_path, processed_path),
  143. level=2,
  144. )
  145. self.post_processed_files.append(original_path)
  146. else:
  147. self.log("Skipped post-processing '%s'" % original_path)
  148. return {
  149. "modified": self.copied_files + self.symlinked_files,
  150. "unmodified": self.unmodified_files,
  151. "post_processed": self.post_processed_files,
  152. }
  153. def handle(self, **options):
  154. self.set_options(**options)
  155. message = ["\n"]
  156. if self.dry_run:
  157. message.append(
  158. "You have activated the --dry-run option so no files will be "
  159. "modified.\n\n"
  160. )
  161. message.append(
  162. "You have requested to collect static files at the destination\n"
  163. "location as specified in your settings"
  164. )
  165. if self.is_local_storage() and self.storage.location:
  166. destination_path = self.storage.location
  167. message.append(":\n\n %s\n\n" % destination_path)
  168. should_warn_user = self.storage.exists(destination_path) and any(
  169. self.storage.listdir(destination_path)
  170. )
  171. else:
  172. destination_path = None
  173. message.append(".\n\n")
  174. # Destination files existence not checked; play it safe and warn.
  175. should_warn_user = True
  176. if self.interactive and should_warn_user:
  177. if self.clear:
  178. message.append("This will DELETE ALL FILES in this location!\n")
  179. else:
  180. message.append("This will overwrite existing files!\n")
  181. message.append(
  182. "Are you sure you want to do this?\n\n"
  183. "Type 'yes' to continue, or 'no' to cancel: "
  184. )
  185. if input("".join(message)) != "yes":
  186. raise CommandError("Collecting static files cancelled.")
  187. collected = self.collect()
  188. if self.verbosity >= 1:
  189. modified_count = len(collected["modified"])
  190. unmodified_count = len(collected["unmodified"])
  191. post_processed_count = len(collected["post_processed"])
  192. return (
  193. "\n%(modified_count)s %(identifier)s %(action)s"
  194. "%(destination)s%(unmodified)s%(post_processed)s."
  195. ) % {
  196. "modified_count": modified_count,
  197. "identifier": "static file" + ("" if modified_count == 1 else "s"),
  198. "action": "symlinked" if self.symlink else "copied",
  199. "destination": (
  200. " to '%s'" % destination_path if destination_path else ""
  201. ),
  202. "unmodified": (
  203. ", %s unmodified" % unmodified_count
  204. if collected["unmodified"]
  205. else ""
  206. ),
  207. "post_processed": (
  208. collected["post_processed"]
  209. and ", %s post-processed" % post_processed_count
  210. or ""
  211. ),
  212. }
  213. def log(self, msg, level=2):
  214. """
  215. Small log helper
  216. """
  217. if self.verbosity >= level:
  218. self.stdout.write(msg)
  219. def is_local_storage(self):
  220. return isinstance(self.storage, FileSystemStorage)
  221. def clear_dir(self, path):
  222. """
  223. Delete the given relative path using the destination storage backend.
  224. """
  225. if not self.storage.exists(path):
  226. return
  227. dirs, files = self.storage.listdir(path)
  228. for f in files:
  229. fpath = os.path.join(path, f)
  230. if self.dry_run:
  231. self.log("Pretending to delete '%s'" % fpath, level=1)
  232. else:
  233. self.log("Deleting '%s'" % fpath, level=1)
  234. try:
  235. full_path = self.storage.path(fpath)
  236. except NotImplementedError:
  237. self.storage.delete(fpath)
  238. else:
  239. if not os.path.exists(full_path) and os.path.lexists(full_path):
  240. # Delete broken symlinks
  241. os.unlink(full_path)
  242. else:
  243. self.storage.delete(fpath)
  244. for d in dirs:
  245. self.clear_dir(os.path.join(path, d))
  246. def delete_file(self, path, prefixed_path, source_storage):
  247. """
  248. Check if the target file should be deleted if it already exists.
  249. """
  250. if self.storage.exists(prefixed_path):
  251. try:
  252. # When was the target file modified last time?
  253. target_last_modified = self.storage.get_modified_time(prefixed_path)
  254. except (OSError, NotImplementedError, AttributeError):
  255. # The storage doesn't support get_modified_time() or failed
  256. pass
  257. else:
  258. try:
  259. # When was the source file modified last time?
  260. source_last_modified = source_storage.get_modified_time(path)
  261. except (OSError, NotImplementedError, AttributeError):
  262. pass
  263. else:
  264. # The full path of the target file
  265. if self.local:
  266. full_path = self.storage.path(prefixed_path)
  267. # If it's --link mode and the path isn't a link (i.e.
  268. # the previous collectstatic wasn't with --link) or if
  269. # it's non-link mode and the path is a link (i.e. the
  270. # previous collectstatic was with --link), the old
  271. # links/files must be deleted so it's not safe to skip
  272. # unmodified files.
  273. can_skip_unmodified_files = not (
  274. self.symlink ^ os.path.islink(full_path)
  275. )
  276. else:
  277. # In remote storages, skipping is only based on the
  278. # modified times since symlinks aren't relevant.
  279. can_skip_unmodified_files = True
  280. # Avoid sub-second precision (see #14665, #19540)
  281. file_is_unmodified = target_last_modified.replace(
  282. microsecond=0
  283. ) >= source_last_modified.replace(microsecond=0)
  284. if file_is_unmodified and can_skip_unmodified_files:
  285. if prefixed_path not in self.unmodified_files:
  286. self.unmodified_files.append(prefixed_path)
  287. self.log("Skipping '%s' (not modified)" % path)
  288. return False
  289. # Then delete the existing file if really needed
  290. if self.dry_run:
  291. self.log("Pretending to delete '%s'" % path)
  292. else:
  293. self.log("Deleting '%s'" % path)
  294. self.storage.delete(prefixed_path)
  295. return True
  296. def link_file(self, path, prefixed_path, source_storage):
  297. """
  298. Attempt to link ``path``
  299. """
  300. # Skip this file if it was already copied earlier
  301. if prefixed_path in self.symlinked_files:
  302. return self.log("Skipping '%s' (already linked earlier)" % path)
  303. # Delete the target file if needed or break
  304. if not self.delete_file(path, prefixed_path, source_storage):
  305. return
  306. # The full path of the source file
  307. source_path = source_storage.path(path)
  308. # Finally link the file
  309. if self.dry_run:
  310. self.log("Pretending to link '%s'" % source_path, level=1)
  311. else:
  312. self.log("Linking '%s'" % source_path, level=2)
  313. full_path = self.storage.path(prefixed_path)
  314. os.makedirs(os.path.dirname(full_path), exist_ok=True)
  315. try:
  316. if os.path.lexists(full_path):
  317. os.unlink(full_path)
  318. os.symlink(source_path, full_path)
  319. except NotImplementedError:
  320. import platform
  321. raise CommandError(
  322. "Symlinking is not supported in this "
  323. "platform (%s)." % platform.platform()
  324. )
  325. except OSError as e:
  326. raise CommandError(e)
  327. if prefixed_path not in self.symlinked_files:
  328. self.symlinked_files.append(prefixed_path)
  329. def copy_file(self, path, prefixed_path, source_storage):
  330. """
  331. Attempt to copy ``path`` with storage
  332. """
  333. # Skip this file if it was already copied earlier
  334. if prefixed_path in self.copied_files:
  335. return self.log("Skipping '%s' (already copied earlier)" % path)
  336. # Delete the target file if needed or break
  337. if not self.delete_file(path, prefixed_path, source_storage):
  338. return
  339. # The full path of the source file
  340. source_path = source_storage.path(path)
  341. # Finally start copying
  342. if self.dry_run:
  343. self.log("Pretending to copy '%s'" % source_path, level=1)
  344. else:
  345. self.log("Copying '%s'" % source_path, level=2)
  346. with source_storage.open(path) as source_file:
  347. self.storage.save(prefixed_path, source_file)
  348. self.copied_files.append(prefixed_path)