executor.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413
  1. from django.apps.registry import apps as global_apps
  2. from django.db import migrations, router
  3. from .exceptions import InvalidMigrationPlan
  4. from .loader import MigrationLoader
  5. from .recorder import MigrationRecorder
  6. from .state import ProjectState
  7. class MigrationExecutor:
  8. """
  9. End-to-end migration execution - load migrations and run them up or down
  10. to a specified set of targets.
  11. """
  12. def __init__(self, connection, progress_callback=None):
  13. self.connection = connection
  14. self.loader = MigrationLoader(self.connection)
  15. self.recorder = MigrationRecorder(self.connection)
  16. self.progress_callback = progress_callback
  17. def migration_plan(self, targets, clean_start=False):
  18. """
  19. Given a set of targets, return a list of (Migration instance, backwards?).
  20. """
  21. plan = []
  22. if clean_start:
  23. applied = {}
  24. else:
  25. applied = dict(self.loader.applied_migrations)
  26. for target in targets:
  27. # If the target is (app_label, None), that means unmigrate everything
  28. if target[1] is None:
  29. for root in self.loader.graph.root_nodes():
  30. if root[0] == target[0]:
  31. for migration in self.loader.graph.backwards_plan(root):
  32. if migration in applied:
  33. plan.append((self.loader.graph.nodes[migration], True))
  34. applied.pop(migration)
  35. # If the migration is already applied, do backwards mode,
  36. # otherwise do forwards mode.
  37. elif target in applied:
  38. # If the target is missing, it's likely a replaced migration.
  39. # Reload the graph without replacements.
  40. if (
  41. self.loader.replace_migrations
  42. and target not in self.loader.graph.node_map
  43. ):
  44. self.loader.replace_migrations = False
  45. self.loader.build_graph()
  46. return self.migration_plan(targets, clean_start=clean_start)
  47. # Don't migrate backwards all the way to the target node (that
  48. # may roll back dependencies in other apps that don't need to
  49. # be rolled back); instead roll back through target's immediate
  50. # child(ren) in the same app, and no further.
  51. next_in_app = sorted(
  52. n
  53. for n in self.loader.graph.node_map[target].children
  54. if n[0] == target[0]
  55. )
  56. for node in next_in_app:
  57. for migration in self.loader.graph.backwards_plan(node):
  58. if migration in applied:
  59. plan.append((self.loader.graph.nodes[migration], True))
  60. applied.pop(migration)
  61. else:
  62. for migration in self.loader.graph.forwards_plan(target):
  63. if migration not in applied:
  64. plan.append((self.loader.graph.nodes[migration], False))
  65. applied[migration] = self.loader.graph.nodes[migration]
  66. return plan
  67. def _create_project_state(self, with_applied_migrations=False):
  68. """
  69. Create a project state including all the applications without
  70. migrations and applied migrations if with_applied_migrations=True.
  71. """
  72. state = ProjectState(real_apps=self.loader.unmigrated_apps)
  73. if with_applied_migrations:
  74. # Create the forwards plan Django would follow on an empty database
  75. full_plan = self.migration_plan(
  76. self.loader.graph.leaf_nodes(), clean_start=True
  77. )
  78. applied_migrations = {
  79. self.loader.graph.nodes[key]
  80. for key in self.loader.applied_migrations
  81. if key in self.loader.graph.nodes
  82. }
  83. for migration, _ in full_plan:
  84. if migration in applied_migrations:
  85. migration.mutate_state(state, preserve=False)
  86. return state
  87. def migrate(self, targets, plan=None, state=None, fake=False, fake_initial=False):
  88. """
  89. Migrate the database up to the given targets.
  90. Django first needs to create all project states before a migration is
  91. (un)applied and in a second step run all the database operations.
  92. """
  93. # The django_migrations table must be present to record applied
  94. # migrations, but don't create it if there are no migrations to apply.
  95. if plan == []:
  96. if not self.recorder.has_table():
  97. return self._create_project_state(with_applied_migrations=False)
  98. else:
  99. self.recorder.ensure_schema()
  100. if plan is None:
  101. plan = self.migration_plan(targets)
  102. # Create the forwards plan Django would follow on an empty database
  103. full_plan = self.migration_plan(
  104. self.loader.graph.leaf_nodes(), clean_start=True
  105. )
  106. all_forwards = all(not backwards for mig, backwards in plan)
  107. all_backwards = all(backwards for mig, backwards in plan)
  108. if not plan:
  109. if state is None:
  110. # The resulting state should include applied migrations.
  111. state = self._create_project_state(with_applied_migrations=True)
  112. elif all_forwards == all_backwards:
  113. # This should only happen if there's a mixed plan
  114. raise InvalidMigrationPlan(
  115. "Migration plans with both forwards and backwards migrations "
  116. "are not supported. Please split your migration process into "
  117. "separate plans of only forwards OR backwards migrations.",
  118. plan,
  119. )
  120. elif all_forwards:
  121. if state is None:
  122. # The resulting state should still include applied migrations.
  123. state = self._create_project_state(with_applied_migrations=True)
  124. state = self._migrate_all_forwards(
  125. state, plan, full_plan, fake=fake, fake_initial=fake_initial
  126. )
  127. else:
  128. # No need to check for `elif all_backwards` here, as that condition
  129. # would always evaluate to true.
  130. state = self._migrate_all_backwards(plan, full_plan, fake=fake)
  131. self.check_replacements()
  132. return state
  133. def _migrate_all_forwards(self, state, plan, full_plan, fake, fake_initial):
  134. """
  135. Take a list of 2-tuples of the form (migration instance, False) and
  136. apply them in the order they occur in the full_plan.
  137. """
  138. migrations_to_run = {m[0] for m in plan}
  139. for migration, _ in full_plan:
  140. if not migrations_to_run:
  141. # We remove every migration that we applied from these sets so
  142. # that we can bail out once the last migration has been applied
  143. # and don't always run until the very end of the migration
  144. # process.
  145. break
  146. if migration in migrations_to_run:
  147. if "apps" not in state.__dict__:
  148. if self.progress_callback:
  149. self.progress_callback("render_start")
  150. state.apps # Render all -- performance critical
  151. if self.progress_callback:
  152. self.progress_callback("render_success")
  153. state = self.apply_migration(
  154. state, migration, fake=fake, fake_initial=fake_initial
  155. )
  156. migrations_to_run.remove(migration)
  157. return state
  158. def _migrate_all_backwards(self, plan, full_plan, fake):
  159. """
  160. Take a list of 2-tuples of the form (migration instance, True) and
  161. unapply them in reverse order they occur in the full_plan.
  162. Since unapplying a migration requires the project state prior to that
  163. migration, Django will compute the migration states before each of them
  164. in a first run over the plan and then unapply them in a second run over
  165. the plan.
  166. """
  167. migrations_to_run = {m[0] for m in plan}
  168. # Holds all migration states prior to the migrations being unapplied
  169. states = {}
  170. state = self._create_project_state()
  171. applied_migrations = {
  172. self.loader.graph.nodes[key]
  173. for key in self.loader.applied_migrations
  174. if key in self.loader.graph.nodes
  175. }
  176. if self.progress_callback:
  177. self.progress_callback("render_start")
  178. for migration, _ in full_plan:
  179. if not migrations_to_run:
  180. # We remove every migration that we applied from this set so
  181. # that we can bail out once the last migration has been applied
  182. # and don't always run until the very end of the migration
  183. # process.
  184. break
  185. if migration in migrations_to_run:
  186. if "apps" not in state.__dict__:
  187. state.apps # Render all -- performance critical
  188. # The state before this migration
  189. states[migration] = state
  190. # The old state keeps as-is, we continue with the new state
  191. state = migration.mutate_state(state, preserve=True)
  192. migrations_to_run.remove(migration)
  193. elif migration in applied_migrations:
  194. # Only mutate the state if the migration is actually applied
  195. # to make sure the resulting state doesn't include changes
  196. # from unrelated migrations.
  197. migration.mutate_state(state, preserve=False)
  198. if self.progress_callback:
  199. self.progress_callback("render_success")
  200. for migration, _ in plan:
  201. self.unapply_migration(states[migration], migration, fake=fake)
  202. applied_migrations.remove(migration)
  203. # Generate the post migration state by starting from the state before
  204. # the last migration is unapplied and mutating it to include all the
  205. # remaining applied migrations.
  206. last_unapplied_migration = plan[-1][0]
  207. state = states[last_unapplied_migration]
  208. # Avoid mutating state with apps rendered as it's an expensive
  209. # operation.
  210. del state.apps
  211. for index, (migration, _) in enumerate(full_plan):
  212. if migration == last_unapplied_migration:
  213. for migration, _ in full_plan[index:]:
  214. if migration in applied_migrations:
  215. migration.mutate_state(state, preserve=False)
  216. break
  217. return state
  218. def apply_migration(self, state, migration, fake=False, fake_initial=False):
  219. """Run a migration forwards."""
  220. migration_recorded = False
  221. if self.progress_callback:
  222. self.progress_callback("apply_start", migration, fake)
  223. if not fake:
  224. if fake_initial:
  225. # Test to see if this is an already-applied initial migration
  226. applied, state = self.detect_soft_applied(state, migration)
  227. if applied:
  228. fake = True
  229. if not fake:
  230. # Alright, do it normally
  231. with self.connection.schema_editor(
  232. atomic=migration.atomic
  233. ) as schema_editor:
  234. state = migration.apply(state, schema_editor)
  235. if not schema_editor.deferred_sql:
  236. self.record_migration(migration)
  237. migration_recorded = True
  238. if not migration_recorded:
  239. self.record_migration(migration)
  240. # Report progress
  241. if self.progress_callback:
  242. self.progress_callback("apply_success", migration, fake)
  243. return state
  244. def record_migration(self, migration):
  245. # For replacement migrations, record individual statuses
  246. if migration.replaces:
  247. for app_label, name in migration.replaces:
  248. self.recorder.record_applied(app_label, name)
  249. else:
  250. self.recorder.record_applied(migration.app_label, migration.name)
  251. def unapply_migration(self, state, migration, fake=False):
  252. """Run a migration backwards."""
  253. if self.progress_callback:
  254. self.progress_callback("unapply_start", migration, fake)
  255. if not fake:
  256. with self.connection.schema_editor(
  257. atomic=migration.atomic
  258. ) as schema_editor:
  259. state = migration.unapply(state, schema_editor)
  260. # For replacement migrations, also record individual statuses.
  261. if migration.replaces:
  262. for app_label, name in migration.replaces:
  263. self.recorder.record_unapplied(app_label, name)
  264. self.recorder.record_unapplied(migration.app_label, migration.name)
  265. # Report progress
  266. if self.progress_callback:
  267. self.progress_callback("unapply_success", migration, fake)
  268. return state
  269. def check_replacements(self):
  270. """
  271. Mark replacement migrations applied if their replaced set all are.
  272. Do this unconditionally on every migrate, rather than just when
  273. migrations are applied or unapplied, to correctly handle the case
  274. when a new squash migration is pushed to a deployment that already had
  275. all its replaced migrations applied. In this case no new migration will
  276. be applied, but the applied state of the squashed migration must be
  277. maintained.
  278. """
  279. applied = self.recorder.applied_migrations()
  280. for key, migration in self.loader.replacements.items():
  281. all_applied = all(m in applied for m in migration.replaces)
  282. if all_applied and key not in applied:
  283. self.recorder.record_applied(*key)
  284. def detect_soft_applied(self, project_state, migration):
  285. """
  286. Test whether a migration has been implicitly applied - that the
  287. tables or columns it would create exist. This is intended only for use
  288. on initial migrations (as it only looks for CreateModel and AddField).
  289. """
  290. def should_skip_detecting_model(migration, model):
  291. """
  292. No need to detect tables for proxy models, unmanaged models, or
  293. models that can't be migrated on the current database.
  294. """
  295. return (
  296. model._meta.proxy
  297. or not model._meta.managed
  298. or not router.allow_migrate(
  299. self.connection.alias,
  300. migration.app_label,
  301. model_name=model._meta.model_name,
  302. )
  303. )
  304. if migration.initial is None:
  305. # Bail if the migration isn't the first one in its app
  306. if any(app == migration.app_label for app, name in migration.dependencies):
  307. return False, project_state
  308. elif migration.initial is False:
  309. # Bail if it's NOT an initial migration
  310. return False, project_state
  311. if project_state is None:
  312. after_state = self.loader.project_state(
  313. (migration.app_label, migration.name), at_end=True
  314. )
  315. else:
  316. after_state = migration.mutate_state(project_state)
  317. apps = after_state.apps
  318. found_create_model_migration = False
  319. found_add_field_migration = False
  320. fold_identifier_case = self.connection.features.ignores_table_name_case
  321. with self.connection.cursor() as cursor:
  322. existing_table_names = set(
  323. self.connection.introspection.table_names(cursor)
  324. )
  325. if fold_identifier_case:
  326. existing_table_names = {
  327. name.casefold() for name in existing_table_names
  328. }
  329. # Make sure all create model and add field operations are done
  330. for operation in migration.operations:
  331. if isinstance(operation, migrations.CreateModel):
  332. model = apps.get_model(migration.app_label, operation.name)
  333. if model._meta.swapped:
  334. # We have to fetch the model to test with from the
  335. # main app cache, as it's not a direct dependency.
  336. model = global_apps.get_model(model._meta.swapped)
  337. if should_skip_detecting_model(migration, model):
  338. continue
  339. db_table = model._meta.db_table
  340. if fold_identifier_case:
  341. db_table = db_table.casefold()
  342. if db_table not in existing_table_names:
  343. return False, project_state
  344. found_create_model_migration = True
  345. elif isinstance(operation, migrations.AddField):
  346. model = apps.get_model(migration.app_label, operation.model_name)
  347. if model._meta.swapped:
  348. # We have to fetch the model to test with from the
  349. # main app cache, as it's not a direct dependency.
  350. model = global_apps.get_model(model._meta.swapped)
  351. if should_skip_detecting_model(migration, model):
  352. continue
  353. table = model._meta.db_table
  354. field = model._meta.get_field(operation.name)
  355. # Handle implicit many-to-many tables created by AddField.
  356. if field.many_to_many:
  357. through_db_table = field.remote_field.through._meta.db_table
  358. if fold_identifier_case:
  359. through_db_table = through_db_table.casefold()
  360. if through_db_table not in existing_table_names:
  361. return False, project_state
  362. else:
  363. found_add_field_migration = True
  364. continue
  365. with self.connection.cursor() as cursor:
  366. columns = self.connection.introspection.get_table_description(
  367. cursor, table
  368. )
  369. for column in columns:
  370. field_column = field.column
  371. column_name = column.name
  372. if fold_identifier_case:
  373. column_name = column_name.casefold()
  374. field_column = field_column.casefold()
  375. if column_name == field_column:
  376. found_add_field_migration = True
  377. break
  378. else:
  379. return False, project_state
  380. # If we get this far and we found at least one CreateModel or AddField
  381. # migration, the migration is considered implicitly applied.
  382. return (found_create_model_migration or found_add_field_migration), after_state