query.py 114 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735
  1. """
  2. Create SQL statements for QuerySets.
  3. The code in here encapsulates all of the SQL construction so that QuerySets
  4. themselves do not have to (and could be backed by things other than SQL
  5. databases). The abstraction barrier only works one way: this module has to know
  6. all about the internals of models in order to get the information it needs.
  7. """
  8. import copy
  9. import difflib
  10. import functools
  11. import sys
  12. from collections import Counter, namedtuple
  13. from collections.abc import Iterator, Mapping
  14. from itertools import chain, count, product
  15. from string import ascii_uppercase
  16. from django.core.exceptions import FieldDoesNotExist, FieldError
  17. from django.db import DEFAULT_DB_ALIAS, NotSupportedError, connections
  18. from django.db.models.aggregates import Count
  19. from django.db.models.constants import LOOKUP_SEP
  20. from django.db.models.expressions import (
  21. BaseExpression,
  22. Col,
  23. Exists,
  24. F,
  25. OuterRef,
  26. Ref,
  27. ResolvedOuterRef,
  28. Value,
  29. )
  30. from django.db.models.fields import Field
  31. from django.db.models.fields.related_lookups import MultiColSource
  32. from django.db.models.lookups import Lookup
  33. from django.db.models.query_utils import (
  34. Q,
  35. check_rel_lookup_compatibility,
  36. refs_expression,
  37. )
  38. from django.db.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE
  39. from django.db.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin
  40. from django.db.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode
  41. from django.utils.functional import cached_property
  42. from django.utils.regex_helper import _lazy_re_compile
  43. from django.utils.tree import Node
  44. __all__ = ["Query", "RawQuery"]
  45. # Quotation marks ('"`[]), whitespace characters, semicolons, or inline
  46. # SQL comments are forbidden in column aliases.
  47. FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/")
  48. # Inspired from
  49. # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
  50. EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w-]+")
  51. def get_field_names_from_opts(opts):
  52. if opts is None:
  53. return set()
  54. return set(
  55. chain.from_iterable(
  56. (f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields()
  57. )
  58. )
  59. def get_paths_from_expression(expr):
  60. if isinstance(expr, F):
  61. yield expr.name
  62. elif hasattr(expr, "flatten"):
  63. for child in expr.flatten():
  64. if isinstance(child, F):
  65. yield child.name
  66. elif isinstance(child, Q):
  67. yield from get_children_from_q(child)
  68. def get_children_from_q(q):
  69. for child in q.children:
  70. if isinstance(child, Node):
  71. yield from get_children_from_q(child)
  72. elif isinstance(child, tuple):
  73. lhs, rhs = child
  74. yield lhs
  75. if hasattr(rhs, "resolve_expression"):
  76. yield from get_paths_from_expression(rhs)
  77. elif hasattr(child, "resolve_expression"):
  78. yield from get_paths_from_expression(child)
  79. def get_child_with_renamed_prefix(prefix, replacement, child):
  80. from django.db.models.query import QuerySet
  81. if isinstance(child, Node):
  82. return rename_prefix_from_q(prefix, replacement, child)
  83. if isinstance(child, tuple):
  84. lhs, rhs = child
  85. if lhs.startswith(prefix + LOOKUP_SEP):
  86. lhs = lhs.replace(prefix, replacement, 1)
  87. if not isinstance(rhs, F) and hasattr(rhs, "resolve_expression"):
  88. rhs = get_child_with_renamed_prefix(prefix, replacement, rhs)
  89. return lhs, rhs
  90. if isinstance(child, F):
  91. child = child.copy()
  92. if child.name.startswith(prefix + LOOKUP_SEP):
  93. child.name = child.name.replace(prefix, replacement, 1)
  94. elif isinstance(child, QuerySet):
  95. # QuerySet may contain OuterRef() references which cannot work properly
  96. # without repointing to the filtered annotation and will spawn a
  97. # different JOIN. Always raise ValueError instead of providing partial
  98. # support in other cases.
  99. raise ValueError(
  100. "Passing a QuerySet within a FilteredRelation is not supported."
  101. )
  102. elif hasattr(child, "resolve_expression"):
  103. child = child.copy()
  104. child.set_source_expressions(
  105. [
  106. get_child_with_renamed_prefix(prefix, replacement, grand_child)
  107. for grand_child in child.get_source_expressions()
  108. ]
  109. )
  110. return child
  111. def rename_prefix_from_q(prefix, replacement, q):
  112. return Q.create(
  113. [get_child_with_renamed_prefix(prefix, replacement, c) for c in q.children],
  114. q.connector,
  115. q.negated,
  116. )
  117. JoinInfo = namedtuple(
  118. "JoinInfo",
  119. ("final_field", "targets", "opts", "joins", "path", "transform_function"),
  120. )
  121. class RawQuery:
  122. """A single raw SQL query."""
  123. def __init__(self, sql, using, params=()):
  124. self.params = params
  125. self.sql = sql
  126. self.using = using
  127. self.cursor = None
  128. # Mirror some properties of a normal query so that
  129. # the compiler can be used to process results.
  130. self.low_mark, self.high_mark = 0, None # Used for offset/limit
  131. self.extra_select = {}
  132. self.annotation_select = {}
  133. def chain(self, using):
  134. return self.clone(using)
  135. def clone(self, using):
  136. return RawQuery(self.sql, using, params=self.params)
  137. def get_columns(self):
  138. if self.cursor is None:
  139. self._execute_query()
  140. converter = connections[self.using].introspection.identifier_converter
  141. return [converter(column_meta[0]) for column_meta in self.cursor.description]
  142. def __iter__(self):
  143. # Always execute a new query for a new iterator.
  144. # This could be optimized with a cache at the expense of RAM.
  145. self._execute_query()
  146. if not connections[self.using].features.can_use_chunked_reads:
  147. # If the database can't use chunked reads we need to make sure we
  148. # evaluate the entire query up front.
  149. result = list(self.cursor)
  150. else:
  151. result = self.cursor
  152. return iter(result)
  153. def __repr__(self):
  154. return "<%s: %s>" % (self.__class__.__name__, self)
  155. @property
  156. def params_type(self):
  157. if self.params is None:
  158. return None
  159. return dict if isinstance(self.params, Mapping) else tuple
  160. def __str__(self):
  161. if self.params_type is None:
  162. return self.sql
  163. return self.sql % self.params_type(self.params)
  164. def _execute_query(self):
  165. connection = connections[self.using]
  166. # Adapt parameters to the database, as much as possible considering
  167. # that the target type isn't known. See #17755.
  168. params_type = self.params_type
  169. adapter = connection.ops.adapt_unknown_value
  170. if params_type is tuple:
  171. params = tuple(adapter(val) for val in self.params)
  172. elif params_type is dict:
  173. params = {key: adapter(val) for key, val in self.params.items()}
  174. elif params_type is None:
  175. params = None
  176. else:
  177. raise RuntimeError("Unexpected params type: %s" % params_type)
  178. self.cursor = connection.cursor()
  179. self.cursor.execute(self.sql, params)
  180. ExplainInfo = namedtuple("ExplainInfo", ("format", "options"))
  181. class Query(BaseExpression):
  182. """A single SQL query."""
  183. alias_prefix = "T"
  184. empty_result_set_value = None
  185. subq_aliases = frozenset([alias_prefix])
  186. compiler = "SQLCompiler"
  187. base_table_class = BaseTable
  188. join_class = Join
  189. default_cols = True
  190. default_ordering = True
  191. standard_ordering = True
  192. filter_is_sticky = False
  193. subquery = False
  194. # SQL-related attributes.
  195. # Select and related select clauses are expressions to use in the SELECT
  196. # clause of the query. The select is used for cases where we want to set up
  197. # the select clause to contain other than default fields (values(),
  198. # subqueries...). Note that annotations go to annotations dictionary.
  199. select = ()
  200. # The group_by attribute can have one of the following forms:
  201. # - None: no group by at all in the query
  202. # - A tuple of expressions: group by (at least) those expressions.
  203. # String refs are also allowed for now.
  204. # - True: group by all select fields of the model
  205. # See compiler.get_group_by() for details.
  206. group_by = None
  207. order_by = ()
  208. low_mark = 0 # Used for offset/limit.
  209. high_mark = None # Used for offset/limit.
  210. distinct = False
  211. distinct_fields = ()
  212. select_for_update = False
  213. select_for_update_nowait = False
  214. select_for_update_skip_locked = False
  215. select_for_update_of = ()
  216. select_for_no_key_update = False
  217. select_related = False
  218. has_select_fields = False
  219. # Arbitrary limit for select_related to prevents infinite recursion.
  220. max_depth = 5
  221. # Holds the selects defined by a call to values() or values_list()
  222. # excluding annotation_select and extra_select.
  223. values_select = ()
  224. # SQL annotation-related attributes.
  225. annotation_select_mask = None
  226. _annotation_select_cache = None
  227. # Set combination attributes.
  228. combinator = None
  229. combinator_all = False
  230. combined_queries = ()
  231. # These are for extensions. The contents are more or less appended verbatim
  232. # to the appropriate clause.
  233. extra_select_mask = None
  234. _extra_select_cache = None
  235. extra_tables = ()
  236. extra_order_by = ()
  237. # A tuple that is a set of model field names and either True, if these are
  238. # the fields to defer, or False if these are the only fields to load.
  239. deferred_loading = (frozenset(), True)
  240. explain_info = None
  241. def __init__(self, model, alias_cols=True):
  242. self.model = model
  243. self.alias_refcount = {}
  244. # alias_map is the most important data structure regarding joins.
  245. # It's used for recording which joins exist in the query and what
  246. # types they are. The key is the alias of the joined table (possibly
  247. # the table name) and the value is a Join-like object (see
  248. # sql.datastructures.Join for more information).
  249. self.alias_map = {}
  250. # Whether to provide alias to columns during reference resolving.
  251. self.alias_cols = alias_cols
  252. # Sometimes the query contains references to aliases in outer queries (as
  253. # a result of split_exclude). Correct alias quoting needs to know these
  254. # aliases too.
  255. # Map external tables to whether they are aliased.
  256. self.external_aliases = {}
  257. self.table_map = {} # Maps table names to list of aliases.
  258. self.used_aliases = set()
  259. self.where = WhereNode()
  260. # Maps alias -> Annotation Expression.
  261. self.annotations = {}
  262. # These are for extensions. The contents are more or less appended
  263. # verbatim to the appropriate clause.
  264. self.extra = {} # Maps col_alias -> (col_sql, params).
  265. self._filtered_relations = {}
  266. @property
  267. def output_field(self):
  268. if len(self.select) == 1:
  269. select = self.select[0]
  270. return getattr(select, "target", None) or select.field
  271. elif len(self.annotation_select) == 1:
  272. return next(iter(self.annotation_select.values())).output_field
  273. @cached_property
  274. def base_table(self):
  275. for alias in self.alias_map:
  276. return alias
  277. def __str__(self):
  278. """
  279. Return the query as a string of SQL with the parameter values
  280. substituted in (use sql_with_params() to see the unsubstituted string).
  281. Parameter values won't necessarily be quoted correctly, since that is
  282. done by the database interface at execution time.
  283. """
  284. sql, params = self.sql_with_params()
  285. return sql % params
  286. def sql_with_params(self):
  287. """
  288. Return the query as an SQL string and the parameters that will be
  289. substituted into the query.
  290. """
  291. return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()
  292. def __deepcopy__(self, memo):
  293. """Limit the amount of work when a Query is deepcopied."""
  294. result = self.clone()
  295. memo[id(self)] = result
  296. return result
  297. def get_compiler(self, using=None, connection=None, elide_empty=True):
  298. if using is None and connection is None:
  299. raise ValueError("Need either using or connection")
  300. if using:
  301. connection = connections[using]
  302. return connection.ops.compiler(self.compiler)(
  303. self, connection, using, elide_empty
  304. )
  305. def get_meta(self):
  306. """
  307. Return the Options instance (the model._meta) from which to start
  308. processing. Normally, this is self.model._meta, but it can be changed
  309. by subclasses.
  310. """
  311. if self.model:
  312. return self.model._meta
  313. def clone(self):
  314. """
  315. Return a copy of the current Query. A lightweight alternative to
  316. deepcopy().
  317. """
  318. obj = Empty()
  319. obj.__class__ = self.__class__
  320. # Copy references to everything.
  321. obj.__dict__ = self.__dict__.copy()
  322. # Clone attributes that can't use shallow copy.
  323. obj.alias_refcount = self.alias_refcount.copy()
  324. obj.alias_map = self.alias_map.copy()
  325. obj.external_aliases = self.external_aliases.copy()
  326. obj.table_map = self.table_map.copy()
  327. obj.where = self.where.clone()
  328. obj.annotations = self.annotations.copy()
  329. if self.annotation_select_mask is not None:
  330. obj.annotation_select_mask = self.annotation_select_mask.copy()
  331. if self.combined_queries:
  332. obj.combined_queries = tuple(
  333. [query.clone() for query in self.combined_queries]
  334. )
  335. # _annotation_select_cache cannot be copied, as doing so breaks the
  336. # (necessary) state in which both annotations and
  337. # _annotation_select_cache point to the same underlying objects.
  338. # It will get re-populated in the cloned queryset the next time it's
  339. # used.
  340. obj._annotation_select_cache = None
  341. obj.extra = self.extra.copy()
  342. if self.extra_select_mask is not None:
  343. obj.extra_select_mask = self.extra_select_mask.copy()
  344. if self._extra_select_cache is not None:
  345. obj._extra_select_cache = self._extra_select_cache.copy()
  346. if self.select_related is not False:
  347. # Use deepcopy because select_related stores fields in nested
  348. # dicts.
  349. obj.select_related = copy.deepcopy(obj.select_related)
  350. if "subq_aliases" in self.__dict__:
  351. obj.subq_aliases = self.subq_aliases.copy()
  352. obj.used_aliases = self.used_aliases.copy()
  353. obj._filtered_relations = self._filtered_relations.copy()
  354. # Clear the cached_property, if it exists.
  355. obj.__dict__.pop("base_table", None)
  356. return obj
  357. def chain(self, klass=None):
  358. """
  359. Return a copy of the current Query that's ready for another operation.
  360. The klass argument changes the type of the Query, e.g. UpdateQuery.
  361. """
  362. obj = self.clone()
  363. if klass and obj.__class__ != klass:
  364. obj.__class__ = klass
  365. if not obj.filter_is_sticky:
  366. obj.used_aliases = set()
  367. obj.filter_is_sticky = False
  368. if hasattr(obj, "_setup_query"):
  369. obj._setup_query()
  370. return obj
  371. def relabeled_clone(self, change_map):
  372. clone = self.clone()
  373. clone.change_aliases(change_map)
  374. return clone
  375. def _get_col(self, target, field, alias):
  376. if not self.alias_cols:
  377. alias = None
  378. return target.get_col(alias, field)
  379. def get_aggregation(self, using, aggregate_exprs):
  380. """
  381. Return the dictionary with the values of the existing aggregations.
  382. """
  383. if not aggregate_exprs:
  384. return {}
  385. # Store annotation mask prior to temporarily adding aggregations for
  386. # resolving purpose to facilitate their subsequent removal.
  387. refs_subquery = False
  388. refs_window = False
  389. replacements = {}
  390. annotation_select_mask = self.annotation_select_mask
  391. for alias, aggregate_expr in aggregate_exprs.items():
  392. self.check_alias(alias)
  393. aggregate = aggregate_expr.resolve_expression(
  394. self, allow_joins=True, reuse=None, summarize=True
  395. )
  396. if not aggregate.contains_aggregate:
  397. raise TypeError("%s is not an aggregate expression" % alias)
  398. # Temporarily add aggregate to annotations to allow remaining
  399. # members of `aggregates` to resolve against each others.
  400. self.append_annotation_mask([alias])
  401. aggregate_refs = aggregate.get_refs()
  402. refs_subquery |= any(
  403. getattr(self.annotations[ref], "contains_subquery", False)
  404. for ref in aggregate_refs
  405. )
  406. refs_window |= any(
  407. getattr(self.annotations[ref], "contains_over_clause", True)
  408. for ref in aggregate_refs
  409. )
  410. aggregate = aggregate.replace_expressions(replacements)
  411. self.annotations[alias] = aggregate
  412. replacements[Ref(alias, aggregate)] = aggregate
  413. # Stash resolved aggregates now that they have been allowed to resolve
  414. # against each other.
  415. aggregates = {alias: self.annotations.pop(alias) for alias in aggregate_exprs}
  416. self.set_annotation_mask(annotation_select_mask)
  417. # Existing usage of aggregation can be determined by the presence of
  418. # selected aggregates but also by filters against aliased aggregates.
  419. _, having, qualify = self.where.split_having_qualify()
  420. has_existing_aggregation = (
  421. any(
  422. getattr(annotation, "contains_aggregate", True)
  423. for annotation in self.annotations.values()
  424. )
  425. or having
  426. )
  427. # Decide if we need to use a subquery.
  428. #
  429. # Existing aggregations would cause incorrect results as
  430. # get_aggregation() must produce just one result and thus must not use
  431. # GROUP BY.
  432. #
  433. # If the query has limit or distinct, or uses set operations, then
  434. # those operations must be done in a subquery so that the query
  435. # aggregates on the limit and/or distinct results instead of applying
  436. # the distinct and limit after the aggregation.
  437. if (
  438. isinstance(self.group_by, tuple)
  439. or self.is_sliced
  440. or has_existing_aggregation
  441. or refs_subquery
  442. or refs_window
  443. or qualify
  444. or self.distinct
  445. or self.combinator
  446. ):
  447. from django.db.models.sql.subqueries import AggregateQuery
  448. inner_query = self.clone()
  449. inner_query.subquery = True
  450. outer_query = AggregateQuery(self.model, inner_query)
  451. inner_query.select_for_update = False
  452. inner_query.select_related = False
  453. inner_query.set_annotation_mask(self.annotation_select)
  454. # Queries with distinct_fields need ordering and when a limit is
  455. # applied we must take the slice from the ordered query. Otherwise
  456. # no need for ordering.
  457. inner_query.clear_ordering(force=False)
  458. if not inner_query.distinct:
  459. # If the inner query uses default select and it has some
  460. # aggregate annotations, then we must make sure the inner
  461. # query is grouped by the main model's primary key. However,
  462. # clearing the select clause can alter results if distinct is
  463. # used.
  464. if inner_query.default_cols and has_existing_aggregation:
  465. inner_query.group_by = (
  466. self.model._meta.pk.get_col(inner_query.get_initial_alias()),
  467. )
  468. inner_query.default_cols = False
  469. if not qualify and not self.combinator:
  470. # Mask existing annotations that are not referenced by
  471. # aggregates to be pushed to the outer query unless
  472. # filtering against window functions or if the query is
  473. # combined as both would require complex realiasing logic.
  474. annotation_mask = set()
  475. if isinstance(self.group_by, tuple):
  476. for expr in self.group_by:
  477. annotation_mask |= expr.get_refs()
  478. for aggregate in aggregates.values():
  479. annotation_mask |= aggregate.get_refs()
  480. # Avoid eliding expressions that might have an incidence on
  481. # the implicit grouping logic.
  482. for annotation_alias, annotation in self.annotation_select.items():
  483. if annotation.get_group_by_cols():
  484. annotation_mask.add(annotation_alias)
  485. inner_query.set_annotation_mask(annotation_mask)
  486. # Add aggregates to the outer AggregateQuery. This requires making
  487. # sure all columns referenced by the aggregates are selected in the
  488. # inner query. It is achieved by retrieving all column references
  489. # by the aggregates, explicitly selecting them in the inner query,
  490. # and making sure the aggregates are repointed to them.
  491. col_refs = {}
  492. for alias, aggregate in aggregates.items():
  493. replacements = {}
  494. for col in self._gen_cols([aggregate], resolve_refs=False):
  495. if not (col_ref := col_refs.get(col)):
  496. index = len(col_refs) + 1
  497. col_alias = f"__col{index}"
  498. col_ref = Ref(col_alias, col)
  499. col_refs[col] = col_ref
  500. inner_query.annotations[col_alias] = col
  501. inner_query.append_annotation_mask([col_alias])
  502. replacements[col] = col_ref
  503. outer_query.annotations[alias] = aggregate.replace_expressions(
  504. replacements
  505. )
  506. if (
  507. inner_query.select == ()
  508. and not inner_query.default_cols
  509. and not inner_query.annotation_select_mask
  510. ):
  511. # In case of Model.objects[0:3].count(), there would be no
  512. # field selected in the inner query, yet we must use a subquery.
  513. # So, make sure at least one field is selected.
  514. inner_query.select = (
  515. self.model._meta.pk.get_col(inner_query.get_initial_alias()),
  516. )
  517. else:
  518. outer_query = self
  519. self.select = ()
  520. self.default_cols = False
  521. self.extra = {}
  522. if self.annotations:
  523. # Inline reference to existing annotations and mask them as
  524. # they are unnecessary given only the summarized aggregations
  525. # are requested.
  526. replacements = {
  527. Ref(alias, annotation): annotation
  528. for alias, annotation in self.annotations.items()
  529. }
  530. self.annotations = {
  531. alias: aggregate.replace_expressions(replacements)
  532. for alias, aggregate in aggregates.items()
  533. }
  534. else:
  535. self.annotations = aggregates
  536. self.set_annotation_mask(aggregates)
  537. empty_set_result = [
  538. expression.empty_result_set_value
  539. for expression in outer_query.annotation_select.values()
  540. ]
  541. elide_empty = not any(result is NotImplemented for result in empty_set_result)
  542. outer_query.clear_ordering(force=True)
  543. outer_query.clear_limits()
  544. outer_query.select_for_update = False
  545. outer_query.select_related = False
  546. compiler = outer_query.get_compiler(using, elide_empty=elide_empty)
  547. result = compiler.execute_sql(SINGLE)
  548. if result is None:
  549. result = empty_set_result
  550. else:
  551. converters = compiler.get_converters(outer_query.annotation_select.values())
  552. result = next(compiler.apply_converters((result,), converters))
  553. return dict(zip(outer_query.annotation_select, result))
  554. def get_count(self, using):
  555. """
  556. Perform a COUNT() query using the current filter constraints.
  557. """
  558. obj = self.clone()
  559. return obj.get_aggregation(using, {"__count": Count("*")})["__count"]
  560. def has_filters(self):
  561. return self.where
  562. def exists(self, limit=True):
  563. q = self.clone()
  564. if not (q.distinct and q.is_sliced):
  565. if q.group_by is True:
  566. q.add_fields(
  567. (f.attname for f in self.model._meta.concrete_fields), False
  568. )
  569. # Disable GROUP BY aliases to avoid orphaning references to the
  570. # SELECT clause which is about to be cleared.
  571. q.set_group_by(allow_aliases=False)
  572. q.clear_select_clause()
  573. if q.combined_queries and q.combinator == "union":
  574. q.combined_queries = tuple(
  575. combined_query.exists(limit=False)
  576. for combined_query in q.combined_queries
  577. )
  578. q.clear_ordering(force=True)
  579. if limit:
  580. q.set_limits(high=1)
  581. q.add_annotation(Value(1), "a")
  582. return q
  583. def has_results(self, using):
  584. q = self.exists(using)
  585. compiler = q.get_compiler(using=using)
  586. return compiler.has_results()
  587. def explain(self, using, format=None, **options):
  588. q = self.clone()
  589. for option_name in options:
  590. if (
  591. not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name)
  592. or "--" in option_name
  593. ):
  594. raise ValueError(f"Invalid option name: {option_name!r}.")
  595. q.explain_info = ExplainInfo(format, options)
  596. compiler = q.get_compiler(using=using)
  597. return "\n".join(compiler.explain_query())
  598. def combine(self, rhs, connector):
  599. """
  600. Merge the 'rhs' query into the current one (with any 'rhs' effects
  601. being applied *after* (that is, "to the right of") anything in the
  602. current query. 'rhs' is not modified during a call to this function.
  603. The 'connector' parameter describes how to connect filters from the
  604. 'rhs' query.
  605. """
  606. if self.model != rhs.model:
  607. raise TypeError("Cannot combine queries on two different base models.")
  608. if self.is_sliced:
  609. raise TypeError("Cannot combine queries once a slice has been taken.")
  610. if self.distinct != rhs.distinct:
  611. raise TypeError("Cannot combine a unique query with a non-unique query.")
  612. if self.distinct_fields != rhs.distinct_fields:
  613. raise TypeError("Cannot combine queries with different distinct fields.")
  614. # If lhs and rhs shares the same alias prefix, it is possible to have
  615. # conflicting alias changes like T4 -> T5, T5 -> T6, which might end up
  616. # as T4 -> T6 while combining two querysets. To prevent this, change an
  617. # alias prefix of the rhs and update current aliases accordingly,
  618. # except if the alias is the base table since it must be present in the
  619. # query on both sides.
  620. initial_alias = self.get_initial_alias()
  621. rhs = rhs.clone()
  622. rhs.bump_prefix(self, exclude={initial_alias})
  623. # Work out how to relabel the rhs aliases, if necessary.
  624. change_map = {}
  625. conjunction = connector == AND
  626. # Determine which existing joins can be reused. When combining the
  627. # query with AND we must recreate all joins for m2m filters. When
  628. # combining with OR we can reuse joins. The reason is that in AND
  629. # case a single row can't fulfill a condition like:
  630. # revrel__col=1 & revrel__col=2
  631. # But, there might be two different related rows matching this
  632. # condition. In OR case a single True is enough, so single row is
  633. # enough, too.
  634. #
  635. # Note that we will be creating duplicate joins for non-m2m joins in
  636. # the AND case. The results will be correct but this creates too many
  637. # joins. This is something that could be fixed later on.
  638. reuse = set() if conjunction else set(self.alias_map)
  639. joinpromoter = JoinPromoter(connector, 2, False)
  640. joinpromoter.add_votes(
  641. j for j in self.alias_map if self.alias_map[j].join_type == INNER
  642. )
  643. rhs_votes = set()
  644. # Now, add the joins from rhs query into the new query (skipping base
  645. # table).
  646. rhs_tables = list(rhs.alias_map)[1:]
  647. for alias in rhs_tables:
  648. join = rhs.alias_map[alias]
  649. # If the left side of the join was already relabeled, use the
  650. # updated alias.
  651. join = join.relabeled_clone(change_map)
  652. new_alias = self.join(join, reuse=reuse)
  653. if join.join_type == INNER:
  654. rhs_votes.add(new_alias)
  655. # We can't reuse the same join again in the query. If we have two
  656. # distinct joins for the same connection in rhs query, then the
  657. # combined query must have two joins, too.
  658. reuse.discard(new_alias)
  659. if alias != new_alias:
  660. change_map[alias] = new_alias
  661. if not rhs.alias_refcount[alias]:
  662. # The alias was unused in the rhs query. Unref it so that it
  663. # will be unused in the new query, too. We have to add and
  664. # unref the alias so that join promotion has information of
  665. # the join type for the unused alias.
  666. self.unref_alias(new_alias)
  667. joinpromoter.add_votes(rhs_votes)
  668. joinpromoter.update_join_types(self)
  669. # Combine subqueries aliases to ensure aliases relabelling properly
  670. # handle subqueries when combining where and select clauses.
  671. self.subq_aliases |= rhs.subq_aliases
  672. # Now relabel a copy of the rhs where-clause and add it to the current
  673. # one.
  674. w = rhs.where.clone()
  675. w.relabel_aliases(change_map)
  676. self.where.add(w, connector)
  677. # Selection columns and extra extensions are those provided by 'rhs'.
  678. if rhs.select:
  679. self.set_select([col.relabeled_clone(change_map) for col in rhs.select])
  680. else:
  681. self.select = ()
  682. if connector == OR:
  683. # It would be nice to be able to handle this, but the queries don't
  684. # really make sense (or return consistent value sets). Not worth
  685. # the extra complexity when you can write a real query instead.
  686. if self.extra and rhs.extra:
  687. raise ValueError(
  688. "When merging querysets using 'or', you cannot have "
  689. "extra(select=...) on both sides."
  690. )
  691. self.extra.update(rhs.extra)
  692. extra_select_mask = set()
  693. if self.extra_select_mask is not None:
  694. extra_select_mask.update(self.extra_select_mask)
  695. if rhs.extra_select_mask is not None:
  696. extra_select_mask.update(rhs.extra_select_mask)
  697. if extra_select_mask:
  698. self.set_extra_mask(extra_select_mask)
  699. self.extra_tables += rhs.extra_tables
  700. # Ordering uses the 'rhs' ordering, unless it has none, in which case
  701. # the current ordering is used.
  702. self.order_by = rhs.order_by or self.order_by
  703. self.extra_order_by = rhs.extra_order_by or self.extra_order_by
  704. def _get_defer_select_mask(self, opts, mask, select_mask=None):
  705. if select_mask is None:
  706. select_mask = {}
  707. select_mask[opts.pk] = {}
  708. # All concrete fields and related objects that are not part of the
  709. # defer mask must be included. If a relational field is encountered it
  710. # gets added to the mask for it be considered if `select_related` and
  711. # the cycle continues by recursively calling this function.
  712. for field in opts.concrete_fields + opts.related_objects:
  713. field_mask = mask.pop(field.name, None)
  714. field_att_mask = None
  715. if field_attname := getattr(field, "attname", None):
  716. field_att_mask = mask.pop(field_attname, None)
  717. if field_mask is None and field_att_mask is None:
  718. select_mask.setdefault(field, {})
  719. elif field_mask:
  720. if not field.is_relation:
  721. raise FieldError(next(iter(field_mask)))
  722. # Virtual fields such as many-to-many and generic foreign keys
  723. # cannot be effectively deferred. Historically, they were
  724. # allowed to be passed to QuerySet.defer(). Ignore such field
  725. # references until a layer of validation at mask alteration
  726. # time is eventually implemented.
  727. if field.many_to_many:
  728. continue
  729. field_select_mask = select_mask.setdefault(field, {})
  730. related_model = field.related_model._meta.concrete_model
  731. self._get_defer_select_mask(
  732. related_model._meta, field_mask, field_select_mask
  733. )
  734. # Remaining defer entries must be references to filtered relations
  735. # otherwise they are surfaced as missing field errors.
  736. for field_name, field_mask in mask.items():
  737. if filtered_relation := self._filtered_relations.get(field_name):
  738. relation = opts.get_field(filtered_relation.relation_name)
  739. field_select_mask = select_mask.setdefault((field_name, relation), {})
  740. related_model = relation.related_model._meta.concrete_model
  741. self._get_defer_select_mask(
  742. related_model._meta, field_mask, field_select_mask
  743. )
  744. else:
  745. opts.get_field(field_name)
  746. return select_mask
  747. def _get_only_select_mask(self, opts, mask, select_mask=None):
  748. if select_mask is None:
  749. select_mask = {}
  750. select_mask[opts.pk] = {}
  751. # Only include fields mentioned in the mask.
  752. for field_name, field_mask in mask.items():
  753. field = opts.get_field(field_name)
  754. field_select_mask = select_mask.setdefault(field, {})
  755. if field_mask:
  756. if not field.is_relation:
  757. raise FieldError(next(iter(field_mask)))
  758. related_model = field.remote_field.model._meta.concrete_model
  759. self._get_only_select_mask(
  760. related_model._meta, field_mask, field_select_mask
  761. )
  762. return select_mask
  763. def get_select_mask(self):
  764. """
  765. Convert the self.deferred_loading data structure to an alternate data
  766. structure, describing the field that *will* be loaded. This is used to
  767. compute the columns to select from the database and also by the
  768. QuerySet class to work out which fields are being initialized on each
  769. model. Models that have all their fields included aren't mentioned in
  770. the result, only those that have field restrictions in place.
  771. """
  772. field_names, defer = self.deferred_loading
  773. if not field_names:
  774. return {}
  775. mask = {}
  776. for field_name in field_names:
  777. part_mask = mask
  778. for part in field_name.split(LOOKUP_SEP):
  779. part_mask = part_mask.setdefault(part, {})
  780. opts = self.get_meta()
  781. if defer:
  782. return self._get_defer_select_mask(opts, mask)
  783. return self._get_only_select_mask(opts, mask)
  784. def table_alias(self, table_name, create=False, filtered_relation=None):
  785. """
  786. Return a table alias for the given table_name and whether this is a
  787. new alias or not.
  788. If 'create' is true, a new alias is always created. Otherwise, the
  789. most recently created alias for the table (if one exists) is reused.
  790. """
  791. alias_list = self.table_map.get(table_name)
  792. if not create and alias_list:
  793. alias = alias_list[0]
  794. self.alias_refcount[alias] += 1
  795. return alias, False
  796. # Create a new alias for this table.
  797. if alias_list:
  798. alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1)
  799. alias_list.append(alias)
  800. else:
  801. # The first occurrence of a table uses the table name directly.
  802. alias = (
  803. filtered_relation.alias if filtered_relation is not None else table_name
  804. )
  805. self.table_map[table_name] = [alias]
  806. self.alias_refcount[alias] = 1
  807. return alias, True
  808. def ref_alias(self, alias):
  809. """Increases the reference count for this alias."""
  810. self.alias_refcount[alias] += 1
  811. def unref_alias(self, alias, amount=1):
  812. """Decreases the reference count for this alias."""
  813. self.alias_refcount[alias] -= amount
  814. def promote_joins(self, aliases):
  815. """
  816. Promote recursively the join type of given aliases and its children to
  817. an outer join. If 'unconditional' is False, only promote the join if
  818. it is nullable or the parent join is an outer join.
  819. The children promotion is done to avoid join chains that contain a LOUTER
  820. b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted,
  821. then we must also promote b->c automatically, or otherwise the promotion
  822. of a->b doesn't actually change anything in the query results.
  823. """
  824. aliases = list(aliases)
  825. while aliases:
  826. alias = aliases.pop(0)
  827. if self.alias_map[alias].join_type is None:
  828. # This is the base table (first FROM entry) - this table
  829. # isn't really joined at all in the query, so we should not
  830. # alter its join type.
  831. continue
  832. # Only the first alias (skipped above) should have None join_type
  833. assert self.alias_map[alias].join_type is not None
  834. parent_alias = self.alias_map[alias].parent_alias
  835. parent_louter = (
  836. parent_alias and self.alias_map[parent_alias].join_type == LOUTER
  837. )
  838. already_louter = self.alias_map[alias].join_type == LOUTER
  839. if (self.alias_map[alias].nullable or parent_louter) and not already_louter:
  840. self.alias_map[alias] = self.alias_map[alias].promote()
  841. # Join type of 'alias' changed, so re-examine all aliases that
  842. # refer to this one.
  843. aliases.extend(
  844. join
  845. for join in self.alias_map
  846. if self.alias_map[join].parent_alias == alias
  847. and join not in aliases
  848. )
  849. def demote_joins(self, aliases):
  850. """
  851. Change join type from LOUTER to INNER for all joins in aliases.
  852. Similarly to promote_joins(), this method must ensure no join chains
  853. containing first an outer, then an inner join are generated. If we
  854. are demoting b->c join in chain a LOUTER b LOUTER c then we must
  855. demote a->b automatically, or otherwise the demotion of b->c doesn't
  856. actually change anything in the query results. .
  857. """
  858. aliases = list(aliases)
  859. while aliases:
  860. alias = aliases.pop(0)
  861. if self.alias_map[alias].join_type == LOUTER:
  862. self.alias_map[alias] = self.alias_map[alias].demote()
  863. parent_alias = self.alias_map[alias].parent_alias
  864. if self.alias_map[parent_alias].join_type == INNER:
  865. aliases.append(parent_alias)
  866. def reset_refcounts(self, to_counts):
  867. """
  868. Reset reference counts for aliases so that they match the value passed
  869. in `to_counts`.
  870. """
  871. for alias, cur_refcount in self.alias_refcount.copy().items():
  872. unref_amount = cur_refcount - to_counts.get(alias, 0)
  873. self.unref_alias(alias, unref_amount)
  874. def change_aliases(self, change_map):
  875. """
  876. Change the aliases in change_map (which maps old-alias -> new-alias),
  877. relabelling any references to them in select columns and the where
  878. clause.
  879. """
  880. if not change_map:
  881. return self
  882. # If keys and values of change_map were to intersect, an alias might be
  883. # updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending
  884. # on their order in change_map.
  885. assert set(change_map).isdisjoint(change_map.values())
  886. # 1. Update references in "select" (normal columns plus aliases),
  887. # "group by" and "where".
  888. self.where.relabel_aliases(change_map)
  889. if isinstance(self.group_by, tuple):
  890. self.group_by = tuple(
  891. [col.relabeled_clone(change_map) for col in self.group_by]
  892. )
  893. self.select = tuple([col.relabeled_clone(change_map) for col in self.select])
  894. self.annotations = self.annotations and {
  895. key: col.relabeled_clone(change_map)
  896. for key, col in self.annotations.items()
  897. }
  898. # 2. Rename the alias in the internal table/alias datastructures.
  899. for old_alias, new_alias in change_map.items():
  900. if old_alias not in self.alias_map:
  901. continue
  902. alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
  903. self.alias_map[new_alias] = alias_data
  904. self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
  905. del self.alias_refcount[old_alias]
  906. del self.alias_map[old_alias]
  907. table_aliases = self.table_map[alias_data.table_name]
  908. for pos, alias in enumerate(table_aliases):
  909. if alias == old_alias:
  910. table_aliases[pos] = new_alias
  911. break
  912. self.external_aliases = {
  913. # Table is aliased or it's being changed and thus is aliased.
  914. change_map.get(alias, alias): (aliased or alias in change_map)
  915. for alias, aliased in self.external_aliases.items()
  916. }
  917. def bump_prefix(self, other_query, exclude=None):
  918. """
  919. Change the alias prefix to the next letter in the alphabet in a way
  920. that the other query's aliases and this query's aliases will not
  921. conflict. Even tables that previously had no alias will get an alias
  922. after this call. To prevent changing aliases use the exclude parameter.
  923. """
  924. def prefix_gen():
  925. """
  926. Generate a sequence of characters in alphabetical order:
  927. -> 'A', 'B', 'C', ...
  928. When the alphabet is finished, the sequence will continue with the
  929. Cartesian product:
  930. -> 'AA', 'AB', 'AC', ...
  931. """
  932. alphabet = ascii_uppercase
  933. prefix = chr(ord(self.alias_prefix) + 1)
  934. yield prefix
  935. for n in count(1):
  936. seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet
  937. for s in product(seq, repeat=n):
  938. yield "".join(s)
  939. prefix = None
  940. if self.alias_prefix != other_query.alias_prefix:
  941. # No clashes between self and outer query should be possible.
  942. return
  943. # Explicitly avoid infinite loop. The constant divider is based on how
  944. # much depth recursive subquery references add to the stack. This value
  945. # might need to be adjusted when adding or removing function calls from
  946. # the code path in charge of performing these operations.
  947. local_recursion_limit = sys.getrecursionlimit() // 16
  948. for pos, prefix in enumerate(prefix_gen()):
  949. if prefix not in self.subq_aliases:
  950. self.alias_prefix = prefix
  951. break
  952. if pos > local_recursion_limit:
  953. raise RecursionError(
  954. "Maximum recursion depth exceeded: too many subqueries."
  955. )
  956. self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
  957. other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases)
  958. if exclude is None:
  959. exclude = {}
  960. self.change_aliases(
  961. {
  962. alias: "%s%d" % (self.alias_prefix, pos)
  963. for pos, alias in enumerate(self.alias_map)
  964. if alias not in exclude
  965. }
  966. )
  967. def get_initial_alias(self):
  968. """
  969. Return the first alias for this query, after increasing its reference
  970. count.
  971. """
  972. if self.alias_map:
  973. alias = self.base_table
  974. self.ref_alias(alias)
  975. elif self.model:
  976. alias = self.join(self.base_table_class(self.get_meta().db_table, None))
  977. else:
  978. alias = None
  979. return alias
  980. def count_active_tables(self):
  981. """
  982. Return the number of tables in this query with a non-zero reference
  983. count. After execution, the reference counts are zeroed, so tables
  984. added in compiler will not be seen by this method.
  985. """
  986. return len([1 for count in self.alias_refcount.values() if count])
  987. def join(self, join, reuse=None):
  988. """
  989. Return an alias for the 'join', either reusing an existing alias for
  990. that join or creating a new one. 'join' is either a base_table_class or
  991. join_class.
  992. The 'reuse' parameter can be either None which means all joins are
  993. reusable, or it can be a set containing the aliases that can be reused.
  994. A join is always created as LOUTER if the lhs alias is LOUTER to make
  995. sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new
  996. joins are created as LOUTER if the join is nullable.
  997. """
  998. reuse_aliases = [
  999. a
  1000. for a, j in self.alias_map.items()
  1001. if (reuse is None or a in reuse) and j == join
  1002. ]
  1003. if reuse_aliases:
  1004. if join.table_alias in reuse_aliases:
  1005. reuse_alias = join.table_alias
  1006. else:
  1007. # Reuse the most recent alias of the joined table
  1008. # (a many-to-many relation may be joined multiple times).
  1009. reuse_alias = reuse_aliases[-1]
  1010. self.ref_alias(reuse_alias)
  1011. return reuse_alias
  1012. # No reuse is possible, so we need a new alias.
  1013. alias, _ = self.table_alias(
  1014. join.table_name, create=True, filtered_relation=join.filtered_relation
  1015. )
  1016. if join.join_type:
  1017. if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
  1018. join_type = LOUTER
  1019. else:
  1020. join_type = INNER
  1021. join.join_type = join_type
  1022. join.table_alias = alias
  1023. self.alias_map[alias] = join
  1024. if filtered_relation := join.filtered_relation:
  1025. resolve_reuse = reuse
  1026. if resolve_reuse is not None:
  1027. resolve_reuse = set(reuse) | {alias}
  1028. joins_len = len(self.alias_map)
  1029. join.filtered_relation = filtered_relation.resolve_expression(
  1030. self, reuse=resolve_reuse
  1031. )
  1032. # Some joins were during expression resolving, they must be present
  1033. # before the one we just added.
  1034. if joins_len < len(self.alias_map):
  1035. self.alias_map[alias] = self.alias_map.pop(alias)
  1036. return alias
  1037. def join_parent_model(self, opts, model, alias, seen):
  1038. """
  1039. Make sure the given 'model' is joined in the query. If 'model' isn't
  1040. a parent of 'opts' or if it is None this method is a no-op.
  1041. The 'alias' is the root alias for starting the join, 'seen' is a dict
  1042. of model -> alias of existing joins. It must also contain a mapping
  1043. of None -> some alias. This will be returned in the no-op case.
  1044. """
  1045. if model in seen:
  1046. return seen[model]
  1047. chain = opts.get_base_chain(model)
  1048. if not chain:
  1049. return alias
  1050. curr_opts = opts
  1051. for int_model in chain:
  1052. if int_model in seen:
  1053. curr_opts = int_model._meta
  1054. alias = seen[int_model]
  1055. continue
  1056. # Proxy model have elements in base chain
  1057. # with no parents, assign the new options
  1058. # object and skip to the next base in that
  1059. # case
  1060. if not curr_opts.parents[int_model]:
  1061. curr_opts = int_model._meta
  1062. continue
  1063. link_field = curr_opts.get_ancestor_link(int_model)
  1064. join_info = self.setup_joins([link_field.name], curr_opts, alias)
  1065. curr_opts = int_model._meta
  1066. alias = seen[int_model] = join_info.joins[-1]
  1067. return alias or seen[None]
  1068. def check_alias(self, alias):
  1069. if FORBIDDEN_ALIAS_PATTERN.search(alias):
  1070. raise ValueError(
  1071. "Column aliases cannot contain whitespace characters, quotation marks, "
  1072. "semicolons, or SQL comments."
  1073. )
  1074. def add_annotation(self, annotation, alias, select=True):
  1075. """Add a single annotation expression to the Query."""
  1076. self.check_alias(alias)
  1077. annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)
  1078. if select:
  1079. self.append_annotation_mask([alias])
  1080. else:
  1081. annotation_mask = (
  1082. value
  1083. for value in dict.fromkeys(self.annotation_select)
  1084. if value != alias
  1085. )
  1086. self.set_annotation_mask(annotation_mask)
  1087. self.annotations[alias] = annotation
  1088. def resolve_expression(self, query, *args, **kwargs):
  1089. clone = self.clone()
  1090. # Subqueries need to use a different set of aliases than the outer query.
  1091. clone.bump_prefix(query)
  1092. clone.subquery = True
  1093. clone.where.resolve_expression(query, *args, **kwargs)
  1094. # Resolve combined queries.
  1095. if clone.combinator:
  1096. clone.combined_queries = tuple(
  1097. [
  1098. combined_query.resolve_expression(query, *args, **kwargs)
  1099. for combined_query in clone.combined_queries
  1100. ]
  1101. )
  1102. for key, value in clone.annotations.items():
  1103. resolved = value.resolve_expression(query, *args, **kwargs)
  1104. if hasattr(resolved, "external_aliases"):
  1105. resolved.external_aliases.update(clone.external_aliases)
  1106. clone.annotations[key] = resolved
  1107. # Outer query's aliases are considered external.
  1108. for alias, table in query.alias_map.items():
  1109. clone.external_aliases[alias] = (
  1110. isinstance(table, Join)
  1111. and table.join_field.related_model._meta.db_table != alias
  1112. ) or (
  1113. isinstance(table, BaseTable) and table.table_name != table.table_alias
  1114. )
  1115. return clone
  1116. def get_external_cols(self):
  1117. exprs = chain(self.annotations.values(), self.where.children)
  1118. return [
  1119. col
  1120. for col in self._gen_cols(exprs, include_external=True)
  1121. if col.alias in self.external_aliases
  1122. ]
  1123. def get_group_by_cols(self, wrapper=None):
  1124. # If wrapper is referenced by an alias for an explicit GROUP BY through
  1125. # values() a reference to this expression and not the self must be
  1126. # returned to ensure external column references are not grouped against
  1127. # as well.
  1128. external_cols = self.get_external_cols()
  1129. if any(col.possibly_multivalued for col in external_cols):
  1130. return [wrapper or self]
  1131. return external_cols
  1132. def as_sql(self, compiler, connection):
  1133. # Some backends (e.g. Oracle) raise an error when a subquery contains
  1134. # unnecessary ORDER BY clause.
  1135. if (
  1136. self.subquery
  1137. and not connection.features.ignores_unnecessary_order_by_in_subqueries
  1138. ):
  1139. self.clear_ordering(force=False)
  1140. for query in self.combined_queries:
  1141. query.clear_ordering(force=False)
  1142. sql, params = self.get_compiler(connection=connection).as_sql()
  1143. if self.subquery:
  1144. sql = "(%s)" % sql
  1145. return sql, params
  1146. def resolve_lookup_value(self, value, can_reuse, allow_joins, summarize=False):
  1147. if hasattr(value, "resolve_expression"):
  1148. value = value.resolve_expression(
  1149. self,
  1150. reuse=can_reuse,
  1151. allow_joins=allow_joins,
  1152. summarize=summarize,
  1153. )
  1154. elif isinstance(value, (list, tuple)):
  1155. # The items of the iterable may be expressions and therefore need
  1156. # to be resolved independently.
  1157. values = (
  1158. self.resolve_lookup_value(sub_value, can_reuse, allow_joins, summarize)
  1159. for sub_value in value
  1160. )
  1161. type_ = type(value)
  1162. if hasattr(type_, "_make"): # namedtuple
  1163. return type_(*values)
  1164. return type_(values)
  1165. return value
  1166. def solve_lookup_type(self, lookup, summarize=False):
  1167. """
  1168. Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains').
  1169. """
  1170. lookup_splitted = lookup.split(LOOKUP_SEP)
  1171. if self.annotations:
  1172. annotation, expression_lookups = refs_expression(
  1173. lookup_splitted, self.annotations
  1174. )
  1175. if annotation:
  1176. expression = self.annotations[annotation]
  1177. if summarize:
  1178. expression = Ref(annotation, expression)
  1179. return expression_lookups, (), expression
  1180. _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
  1181. field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)]
  1182. if len(lookup_parts) > 1 and not field_parts:
  1183. raise FieldError(
  1184. 'Invalid lookup "%s" for model %s".'
  1185. % (lookup, self.get_meta().model.__name__)
  1186. )
  1187. return lookup_parts, field_parts, False
  1188. def check_query_object_type(self, value, opts, field):
  1189. """
  1190. Check whether the object passed while querying is of the correct type.
  1191. If not, raise a ValueError specifying the wrong object.
  1192. """
  1193. if hasattr(value, "_meta"):
  1194. if not check_rel_lookup_compatibility(value._meta.model, opts, field):
  1195. raise ValueError(
  1196. 'Cannot query "%s": Must be "%s" instance.'
  1197. % (value, opts.object_name)
  1198. )
  1199. def check_related_objects(self, field, value, opts):
  1200. """Check the type of object passed to query relations."""
  1201. if field.is_relation:
  1202. # Check that the field and the queryset use the same model in a
  1203. # query like .filter(author=Author.objects.all()). For example, the
  1204. # opts would be Author's (from the author field) and value.model
  1205. # would be Author.objects.all() queryset's .model (Author also).
  1206. # The field is the related field on the lhs side.
  1207. if (
  1208. isinstance(value, Query)
  1209. and not value.has_select_fields
  1210. and not check_rel_lookup_compatibility(value.model, opts, field)
  1211. ):
  1212. raise ValueError(
  1213. 'Cannot use QuerySet for "%s": Use a QuerySet for "%s".'
  1214. % (value.model._meta.object_name, opts.object_name)
  1215. )
  1216. elif hasattr(value, "_meta"):
  1217. self.check_query_object_type(value, opts, field)
  1218. elif hasattr(value, "__iter__"):
  1219. for v in value:
  1220. self.check_query_object_type(v, opts, field)
  1221. def check_filterable(self, expression):
  1222. """Raise an error if expression cannot be used in a WHERE clause."""
  1223. if hasattr(expression, "resolve_expression") and not getattr(
  1224. expression, "filterable", True
  1225. ):
  1226. raise NotSupportedError(
  1227. expression.__class__.__name__ + " is disallowed in the filter "
  1228. "clause."
  1229. )
  1230. if hasattr(expression, "get_source_expressions"):
  1231. for expr in expression.get_source_expressions():
  1232. self.check_filterable(expr)
  1233. def build_lookup(self, lookups, lhs, rhs):
  1234. """
  1235. Try to extract transforms and lookup from given lhs.
  1236. The lhs value is something that works like SQLExpression.
  1237. The rhs value is what the lookup is going to compare against.
  1238. The lookups is a list of names to extract using get_lookup()
  1239. and get_transform().
  1240. """
  1241. # __exact is the default lookup if one isn't given.
  1242. *transforms, lookup_name = lookups or ["exact"]
  1243. for name in transforms:
  1244. lhs = self.try_transform(lhs, name)
  1245. # First try get_lookup() so that the lookup takes precedence if the lhs
  1246. # supports both transform and lookup for the name.
  1247. lookup_class = lhs.get_lookup(lookup_name)
  1248. if not lookup_class:
  1249. # A lookup wasn't found. Try to interpret the name as a transform
  1250. # and do an Exact lookup against it.
  1251. lhs = self.try_transform(lhs, lookup_name)
  1252. lookup_name = "exact"
  1253. lookup_class = lhs.get_lookup(lookup_name)
  1254. if not lookup_class:
  1255. return
  1256. lookup = lookup_class(lhs, rhs)
  1257. # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
  1258. # uses of None as a query value unless the lookup supports it.
  1259. if lookup.rhs is None and not lookup.can_use_none_as_rhs:
  1260. if lookup_name not in ("exact", "iexact"):
  1261. raise ValueError("Cannot use None as a query value")
  1262. return lhs.get_lookup("isnull")(lhs, True)
  1263. # For Oracle '' is equivalent to null. The check must be done at this
  1264. # stage because join promotion can't be done in the compiler. Using
  1265. # DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here.
  1266. # A similar thing is done in is_nullable(), too.
  1267. if (
  1268. lookup_name == "exact"
  1269. and lookup.rhs == ""
  1270. and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
  1271. ):
  1272. return lhs.get_lookup("isnull")(lhs, True)
  1273. return lookup
  1274. def try_transform(self, lhs, name):
  1275. """
  1276. Helper method for build_lookup(). Try to fetch and initialize
  1277. a transform for name parameter from lhs.
  1278. """
  1279. transform_class = lhs.get_transform(name)
  1280. if transform_class:
  1281. return transform_class(lhs)
  1282. else:
  1283. output_field = lhs.output_field.__class__
  1284. suggested_lookups = difflib.get_close_matches(
  1285. name, lhs.output_field.get_lookups()
  1286. )
  1287. if suggested_lookups:
  1288. suggestion = ", perhaps you meant %s?" % " or ".join(suggested_lookups)
  1289. else:
  1290. suggestion = "."
  1291. raise FieldError(
  1292. "Unsupported lookup '%s' for %s or join on the field not "
  1293. "permitted%s" % (name, output_field.__name__, suggestion)
  1294. )
  1295. def build_filter(
  1296. self,
  1297. filter_expr,
  1298. branch_negated=False,
  1299. current_negated=False,
  1300. can_reuse=None,
  1301. allow_joins=True,
  1302. split_subq=True,
  1303. check_filterable=True,
  1304. summarize=False,
  1305. update_join_types=True,
  1306. ):
  1307. """
  1308. Build a WhereNode for a single filter clause but don't add it
  1309. to this Query. Query.add_q() will then add this filter to the where
  1310. Node.
  1311. The 'branch_negated' tells us if the current branch contains any
  1312. negations. This will be used to determine if subqueries are needed.
  1313. The 'current_negated' is used to determine if the current filter is
  1314. negated or not and this will be used to determine if IS NULL filtering
  1315. is needed.
  1316. The difference between current_negated and branch_negated is that
  1317. branch_negated is set on first negation, but current_negated is
  1318. flipped for each negation.
  1319. Note that add_filter will not do any negating itself, that is done
  1320. upper in the code by add_q().
  1321. The 'can_reuse' is a set of reusable joins for multijoins.
  1322. The method will create a filter clause that can be added to the current
  1323. query. However, if the filter isn't added to the query then the caller
  1324. is responsible for unreffing the joins used.
  1325. """
  1326. if isinstance(filter_expr, dict):
  1327. raise FieldError("Cannot parse keyword query as dict")
  1328. if isinstance(filter_expr, Q):
  1329. return self._add_q(
  1330. filter_expr,
  1331. branch_negated=branch_negated,
  1332. current_negated=current_negated,
  1333. used_aliases=can_reuse,
  1334. allow_joins=allow_joins,
  1335. split_subq=split_subq,
  1336. check_filterable=check_filterable,
  1337. summarize=summarize,
  1338. update_join_types=update_join_types,
  1339. )
  1340. if hasattr(filter_expr, "resolve_expression"):
  1341. if not getattr(filter_expr, "conditional", False):
  1342. raise TypeError("Cannot filter against a non-conditional expression.")
  1343. condition = filter_expr.resolve_expression(
  1344. self, allow_joins=allow_joins, reuse=can_reuse, summarize=summarize
  1345. )
  1346. if not isinstance(condition, Lookup):
  1347. condition = self.build_lookup(["exact"], condition, True)
  1348. return WhereNode([condition], connector=AND), []
  1349. arg, value = filter_expr
  1350. if not arg:
  1351. raise FieldError("Cannot parse keyword query %r" % arg)
  1352. lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize)
  1353. if check_filterable:
  1354. self.check_filterable(reffed_expression)
  1355. if not allow_joins and len(parts) > 1:
  1356. raise FieldError("Joined field references are not permitted in this query")
  1357. pre_joins = self.alias_refcount.copy()
  1358. value = self.resolve_lookup_value(value, can_reuse, allow_joins, summarize)
  1359. used_joins = {
  1360. k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)
  1361. }
  1362. if check_filterable:
  1363. self.check_filterable(value)
  1364. if reffed_expression:
  1365. condition = self.build_lookup(lookups, reffed_expression, value)
  1366. return WhereNode([condition], connector=AND), []
  1367. opts = self.get_meta()
  1368. alias = self.get_initial_alias()
  1369. allow_many = not branch_negated or not split_subq
  1370. try:
  1371. join_info = self.setup_joins(
  1372. parts,
  1373. opts,
  1374. alias,
  1375. can_reuse=can_reuse,
  1376. allow_many=allow_many,
  1377. )
  1378. # Prevent iterator from being consumed by check_related_objects()
  1379. if isinstance(value, Iterator):
  1380. value = list(value)
  1381. self.check_related_objects(join_info.final_field, value, join_info.opts)
  1382. # split_exclude() needs to know which joins were generated for the
  1383. # lookup parts
  1384. self._lookup_joins = join_info.joins
  1385. except MultiJoin as e:
  1386. return self.split_exclude(filter_expr, can_reuse, e.names_with_path)
  1387. # Update used_joins before trimming since they are reused to determine
  1388. # which joins could be later promoted to INNER.
  1389. used_joins.update(join_info.joins)
  1390. targets, alias, join_list = self.trim_joins(
  1391. join_info.targets, join_info.joins, join_info.path
  1392. )
  1393. if can_reuse is not None:
  1394. can_reuse.update(join_list)
  1395. if join_info.final_field.is_relation:
  1396. if len(targets) == 1:
  1397. col = self._get_col(targets[0], join_info.final_field, alias)
  1398. else:
  1399. col = MultiColSource(
  1400. alias, targets, join_info.targets, join_info.final_field
  1401. )
  1402. else:
  1403. col = self._get_col(targets[0], join_info.final_field, alias)
  1404. condition = self.build_lookup(lookups, col, value)
  1405. lookup_type = condition.lookup_name
  1406. clause = WhereNode([condition], connector=AND)
  1407. require_outer = (
  1408. lookup_type == "isnull" and condition.rhs is True and not current_negated
  1409. )
  1410. if (
  1411. current_negated
  1412. and (lookup_type != "isnull" or condition.rhs is False)
  1413. and condition.rhs is not None
  1414. ):
  1415. require_outer = True
  1416. if lookup_type != "isnull":
  1417. # The condition added here will be SQL like this:
  1418. # NOT (col IS NOT NULL), where the first NOT is added in
  1419. # upper layers of code. The reason for addition is that if col
  1420. # is null, then col != someval will result in SQL "unknown"
  1421. # which isn't the same as in Python. The Python None handling
  1422. # is wanted, and it can be gotten by
  1423. # (col IS NULL OR col != someval)
  1424. # <=>
  1425. # NOT (col IS NOT NULL AND col = someval).
  1426. if (
  1427. self.is_nullable(targets[0])
  1428. or self.alias_map[join_list[-1]].join_type == LOUTER
  1429. ):
  1430. lookup_class = targets[0].get_lookup("isnull")
  1431. col = self._get_col(targets[0], join_info.targets[0], alias)
  1432. clause.add(lookup_class(col, False), AND)
  1433. # If someval is a nullable column, someval IS NOT NULL is
  1434. # added.
  1435. if isinstance(value, Col) and self.is_nullable(value.target):
  1436. lookup_class = value.target.get_lookup("isnull")
  1437. clause.add(lookup_class(value, False), AND)
  1438. return clause, used_joins if not require_outer else ()
  1439. def add_filter(self, filter_lhs, filter_rhs):
  1440. self.add_q(Q((filter_lhs, filter_rhs)))
  1441. def add_q(self, q_object):
  1442. """
  1443. A preprocessor for the internal _add_q(). Responsible for doing final
  1444. join promotion.
  1445. """
  1446. # For join promotion this case is doing an AND for the added q_object
  1447. # and existing conditions. So, any existing inner join forces the join
  1448. # type to remain inner. Existing outer joins can however be demoted.
  1449. # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if
  1450. # rel_a doesn't produce any rows, then the whole condition must fail.
  1451. # So, demotion is OK.
  1452. existing_inner = {
  1453. a for a in self.alias_map if self.alias_map[a].join_type == INNER
  1454. }
  1455. clause, _ = self._add_q(q_object, self.used_aliases)
  1456. if clause:
  1457. self.where.add(clause, AND)
  1458. self.demote_joins(existing_inner)
  1459. def build_where(self, filter_expr):
  1460. return self.build_filter(filter_expr, allow_joins=False)[0]
  1461. def clear_where(self):
  1462. self.where = WhereNode()
  1463. def _add_q(
  1464. self,
  1465. q_object,
  1466. used_aliases,
  1467. branch_negated=False,
  1468. current_negated=False,
  1469. allow_joins=True,
  1470. split_subq=True,
  1471. check_filterable=True,
  1472. summarize=False,
  1473. update_join_types=True,
  1474. ):
  1475. """Add a Q-object to the current filter."""
  1476. connector = q_object.connector
  1477. current_negated ^= q_object.negated
  1478. branch_negated = branch_negated or q_object.negated
  1479. target_clause = WhereNode(connector=connector, negated=q_object.negated)
  1480. joinpromoter = JoinPromoter(
  1481. q_object.connector, len(q_object.children), current_negated
  1482. )
  1483. for child in q_object.children:
  1484. child_clause, needed_inner = self.build_filter(
  1485. child,
  1486. can_reuse=used_aliases,
  1487. branch_negated=branch_negated,
  1488. current_negated=current_negated,
  1489. allow_joins=allow_joins,
  1490. split_subq=split_subq,
  1491. check_filterable=check_filterable,
  1492. summarize=summarize,
  1493. update_join_types=update_join_types,
  1494. )
  1495. joinpromoter.add_votes(needed_inner)
  1496. if child_clause:
  1497. target_clause.add(child_clause, connector)
  1498. if update_join_types:
  1499. needed_inner = joinpromoter.update_join_types(self)
  1500. else:
  1501. needed_inner = []
  1502. return target_clause, needed_inner
  1503. def add_filtered_relation(self, filtered_relation, alias):
  1504. filtered_relation.alias = alias
  1505. relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type(
  1506. filtered_relation.relation_name
  1507. )
  1508. if relation_lookup_parts:
  1509. raise ValueError(
  1510. "FilteredRelation's relation_name cannot contain lookups "
  1511. "(got %r)." % filtered_relation.relation_name
  1512. )
  1513. for lookup in get_children_from_q(filtered_relation.condition):
  1514. lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup)
  1515. shift = 2 if not lookup_parts else 1
  1516. lookup_field_path = lookup_field_parts[:-shift]
  1517. for idx, lookup_field_part in enumerate(lookup_field_path):
  1518. if len(relation_field_parts) > idx:
  1519. if relation_field_parts[idx] != lookup_field_part:
  1520. raise ValueError(
  1521. "FilteredRelation's condition doesn't support "
  1522. "relations outside the %r (got %r)."
  1523. % (filtered_relation.relation_name, lookup)
  1524. )
  1525. else:
  1526. raise ValueError(
  1527. "FilteredRelation's condition doesn't support nested "
  1528. "relations deeper than the relation_name (got %r for "
  1529. "%r)." % (lookup, filtered_relation.relation_name)
  1530. )
  1531. filtered_relation.condition = rename_prefix_from_q(
  1532. filtered_relation.relation_name,
  1533. alias,
  1534. filtered_relation.condition,
  1535. )
  1536. self._filtered_relations[filtered_relation.alias] = filtered_relation
  1537. def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
  1538. """
  1539. Walk the list of names and turns them into PathInfo tuples. A single
  1540. name in 'names' can generate multiple PathInfos (m2m, for example).
  1541. 'names' is the path of names to travel, 'opts' is the model Options we
  1542. start the name resolving from, 'allow_many' is as for setup_joins().
  1543. If fail_on_missing is set to True, then a name that can't be resolved
  1544. will generate a FieldError.
  1545. Return a list of PathInfo tuples. In addition return the final field
  1546. (the last used join field) and target (which is a field guaranteed to
  1547. contain the same value as the final field). Finally, return those names
  1548. that weren't found (which are likely transforms and the final lookup).
  1549. """
  1550. path, names_with_path = [], []
  1551. for pos, name in enumerate(names):
  1552. cur_names_with_path = (name, [])
  1553. if name == "pk" and opts is not None:
  1554. name = opts.pk.name
  1555. field = None
  1556. filtered_relation = None
  1557. try:
  1558. if opts is None:
  1559. raise FieldDoesNotExist
  1560. field = opts.get_field(name)
  1561. except FieldDoesNotExist:
  1562. if name in self.annotation_select:
  1563. field = self.annotation_select[name].output_field
  1564. elif name in self._filtered_relations and pos == 0:
  1565. filtered_relation = self._filtered_relations[name]
  1566. if LOOKUP_SEP in filtered_relation.relation_name:
  1567. parts = filtered_relation.relation_name.split(LOOKUP_SEP)
  1568. filtered_relation_path, field, _, _ = self.names_to_path(
  1569. parts,
  1570. opts,
  1571. allow_many,
  1572. fail_on_missing,
  1573. )
  1574. path.extend(filtered_relation_path[:-1])
  1575. else:
  1576. field = opts.get_field(filtered_relation.relation_name)
  1577. if field is not None:
  1578. # Fields that contain one-to-many relations with a generic
  1579. # model (like a GenericForeignKey) cannot generate reverse
  1580. # relations and therefore cannot be used for reverse querying.
  1581. if field.is_relation and not field.related_model:
  1582. raise FieldError(
  1583. "Field %r does not generate an automatic reverse "
  1584. "relation and therefore cannot be used for reverse "
  1585. "querying. If it is a GenericForeignKey, consider "
  1586. "adding a GenericRelation." % name
  1587. )
  1588. try:
  1589. model = field.model._meta.concrete_model
  1590. except AttributeError:
  1591. # QuerySet.annotate() may introduce fields that aren't
  1592. # attached to a model.
  1593. model = None
  1594. else:
  1595. # We didn't find the current field, so move position back
  1596. # one step.
  1597. pos -= 1
  1598. if pos == -1 or fail_on_missing:
  1599. available = sorted(
  1600. [
  1601. *get_field_names_from_opts(opts),
  1602. *self.annotation_select,
  1603. *self._filtered_relations,
  1604. ]
  1605. )
  1606. raise FieldError(
  1607. "Cannot resolve keyword '%s' into field. "
  1608. "Choices are: %s" % (name, ", ".join(available))
  1609. )
  1610. break
  1611. # Check if we need any joins for concrete inheritance cases (the
  1612. # field lives in parent, but we are currently in one of its
  1613. # children)
  1614. if opts is not None and model is not opts.model:
  1615. path_to_parent = opts.get_path_to_parent(model)
  1616. if path_to_parent:
  1617. path.extend(path_to_parent)
  1618. cur_names_with_path[1].extend(path_to_parent)
  1619. opts = path_to_parent[-1].to_opts
  1620. if hasattr(field, "path_infos"):
  1621. if filtered_relation:
  1622. pathinfos = field.get_path_info(filtered_relation)
  1623. else:
  1624. pathinfos = field.path_infos
  1625. if not allow_many:
  1626. for inner_pos, p in enumerate(pathinfos):
  1627. if p.m2m:
  1628. cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1])
  1629. names_with_path.append(cur_names_with_path)
  1630. raise MultiJoin(pos + 1, names_with_path)
  1631. last = pathinfos[-1]
  1632. path.extend(pathinfos)
  1633. final_field = last.join_field
  1634. opts = last.to_opts
  1635. targets = last.target_fields
  1636. cur_names_with_path[1].extend(pathinfos)
  1637. names_with_path.append(cur_names_with_path)
  1638. else:
  1639. # Local non-relational field.
  1640. final_field = field
  1641. targets = (field,)
  1642. if fail_on_missing and pos + 1 != len(names):
  1643. raise FieldError(
  1644. "Cannot resolve keyword %r into field. Join on '%s'"
  1645. " not permitted." % (names[pos + 1], name)
  1646. )
  1647. break
  1648. return path, final_field, targets, names[pos + 1 :]
  1649. def setup_joins(
  1650. self,
  1651. names,
  1652. opts,
  1653. alias,
  1654. can_reuse=None,
  1655. allow_many=True,
  1656. ):
  1657. """
  1658. Compute the necessary table joins for the passage through the fields
  1659. given in 'names'. 'opts' is the Options class for the current model
  1660. (which gives the table we are starting from), 'alias' is the alias for
  1661. the table to start the joining from.
  1662. The 'can_reuse' defines the reverse foreign key joins we can reuse. It
  1663. can be None in which case all joins are reusable or a set of aliases
  1664. that can be reused. Note that non-reverse foreign keys are always
  1665. reusable when using setup_joins().
  1666. If 'allow_many' is False, then any reverse foreign key seen will
  1667. generate a MultiJoin exception.
  1668. Return the final field involved in the joins, the target field (used
  1669. for any 'where' constraint), the final 'opts' value, the joins, the
  1670. field path traveled to generate the joins, and a transform function
  1671. that takes a field and alias and is equivalent to `field.get_col(alias)`
  1672. in the simple case but wraps field transforms if they were included in
  1673. names.
  1674. The target field is the field containing the concrete value. Final
  1675. field can be something different, for example foreign key pointing to
  1676. that value. Final field is needed for example in some value
  1677. conversions (convert 'obj' in fk__id=obj to pk val using the foreign
  1678. key field for example).
  1679. """
  1680. joins = [alias]
  1681. # The transform can't be applied yet, as joins must be trimmed later.
  1682. # To avoid making every caller of this method look up transforms
  1683. # directly, compute transforms here and create a partial that converts
  1684. # fields to the appropriate wrapped version.
  1685. def final_transformer(field, alias):
  1686. if not self.alias_cols:
  1687. alias = None
  1688. return field.get_col(alias)
  1689. # Try resolving all the names as fields first. If there's an error,
  1690. # treat trailing names as lookups until a field can be resolved.
  1691. last_field_exception = None
  1692. for pivot in range(len(names), 0, -1):
  1693. try:
  1694. path, final_field, targets, rest = self.names_to_path(
  1695. names[:pivot],
  1696. opts,
  1697. allow_many,
  1698. fail_on_missing=True,
  1699. )
  1700. except FieldError as exc:
  1701. if pivot == 1:
  1702. # The first item cannot be a lookup, so it's safe
  1703. # to raise the field error here.
  1704. raise
  1705. else:
  1706. last_field_exception = exc
  1707. else:
  1708. # The transforms are the remaining items that couldn't be
  1709. # resolved into fields.
  1710. transforms = names[pivot:]
  1711. break
  1712. for name in transforms:
  1713. def transform(field, alias, *, name, previous):
  1714. try:
  1715. wrapped = previous(field, alias)
  1716. return self.try_transform(wrapped, name)
  1717. except FieldError:
  1718. # FieldError is raised if the transform doesn't exist.
  1719. if isinstance(final_field, Field) and last_field_exception:
  1720. raise last_field_exception
  1721. else:
  1722. raise
  1723. final_transformer = functools.partial(
  1724. transform, name=name, previous=final_transformer
  1725. )
  1726. final_transformer.has_transforms = True
  1727. # Then, add the path to the query's joins. Note that we can't trim
  1728. # joins at this stage - we will need the information about join type
  1729. # of the trimmed joins.
  1730. for join in path:
  1731. if join.filtered_relation:
  1732. filtered_relation = join.filtered_relation.clone()
  1733. table_alias = filtered_relation.alias
  1734. else:
  1735. filtered_relation = None
  1736. table_alias = None
  1737. opts = join.to_opts
  1738. if join.direct:
  1739. nullable = self.is_nullable(join.join_field)
  1740. else:
  1741. nullable = True
  1742. connection = self.join_class(
  1743. opts.db_table,
  1744. alias,
  1745. table_alias,
  1746. INNER,
  1747. join.join_field,
  1748. nullable,
  1749. filtered_relation=filtered_relation,
  1750. )
  1751. reuse = can_reuse if join.m2m else None
  1752. alias = self.join(connection, reuse=reuse)
  1753. joins.append(alias)
  1754. return JoinInfo(final_field, targets, opts, joins, path, final_transformer)
  1755. def trim_joins(self, targets, joins, path):
  1756. """
  1757. The 'target' parameter is the final field being joined to, 'joins'
  1758. is the full list of join aliases. The 'path' contain the PathInfos
  1759. used to create the joins.
  1760. Return the final target field and table alias and the new active
  1761. joins.
  1762. Always trim any direct join if the target column is already in the
  1763. previous table. Can't trim reverse joins as it's unknown if there's
  1764. anything on the other side of the join.
  1765. """
  1766. joins = joins[:]
  1767. for pos, info in enumerate(reversed(path)):
  1768. if len(joins) == 1 or not info.direct:
  1769. break
  1770. if info.filtered_relation:
  1771. break
  1772. join_targets = {t.column for t in info.join_field.foreign_related_fields}
  1773. cur_targets = {t.column for t in targets}
  1774. if not cur_targets.issubset(join_targets):
  1775. break
  1776. targets_dict = {
  1777. r[1].column: r[0]
  1778. for r in info.join_field.related_fields
  1779. if r[1].column in cur_targets
  1780. }
  1781. targets = tuple(targets_dict[t.column] for t in targets)
  1782. self.unref_alias(joins.pop())
  1783. return targets, joins[-1], joins
  1784. @classmethod
  1785. def _gen_cols(cls, exprs, include_external=False, resolve_refs=True):
  1786. for expr in exprs:
  1787. if isinstance(expr, Col):
  1788. yield expr
  1789. elif include_external and callable(
  1790. getattr(expr, "get_external_cols", None)
  1791. ):
  1792. yield from expr.get_external_cols()
  1793. elif hasattr(expr, "get_source_expressions"):
  1794. if not resolve_refs and isinstance(expr, Ref):
  1795. continue
  1796. yield from cls._gen_cols(
  1797. expr.get_source_expressions(),
  1798. include_external=include_external,
  1799. resolve_refs=resolve_refs,
  1800. )
  1801. @classmethod
  1802. def _gen_col_aliases(cls, exprs):
  1803. yield from (expr.alias for expr in cls._gen_cols(exprs))
  1804. def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
  1805. annotation = self.annotations.get(name)
  1806. if annotation is not None:
  1807. if not allow_joins:
  1808. for alias in self._gen_col_aliases([annotation]):
  1809. if isinstance(self.alias_map[alias], Join):
  1810. raise FieldError(
  1811. "Joined field references are not permitted in this query"
  1812. )
  1813. if summarize:
  1814. # Summarize currently means we are doing an aggregate() query
  1815. # which is executed as a wrapped subquery if any of the
  1816. # aggregate() elements reference an existing annotation. In
  1817. # that case we need to return a Ref to the subquery's annotation.
  1818. if name not in self.annotation_select:
  1819. raise FieldError(
  1820. "Cannot aggregate over the '%s' alias. Use annotate() "
  1821. "to promote it." % name
  1822. )
  1823. return Ref(name, self.annotation_select[name])
  1824. else:
  1825. return annotation
  1826. else:
  1827. field_list = name.split(LOOKUP_SEP)
  1828. annotation = self.annotations.get(field_list[0])
  1829. if annotation is not None:
  1830. for transform in field_list[1:]:
  1831. annotation = self.try_transform(annotation, transform)
  1832. return annotation
  1833. join_info = self.setup_joins(
  1834. field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse
  1835. )
  1836. targets, final_alias, join_list = self.trim_joins(
  1837. join_info.targets, join_info.joins, join_info.path
  1838. )
  1839. if not allow_joins and len(join_list) > 1:
  1840. raise FieldError(
  1841. "Joined field references are not permitted in this query"
  1842. )
  1843. if len(targets) > 1:
  1844. raise FieldError(
  1845. "Referencing multicolumn fields with F() objects isn't supported"
  1846. )
  1847. # Verify that the last lookup in name is a field or a transform:
  1848. # transform_function() raises FieldError if not.
  1849. transform = join_info.transform_function(targets[0], final_alias)
  1850. if reuse is not None:
  1851. reuse.update(join_list)
  1852. return transform
  1853. def split_exclude(self, filter_expr, can_reuse, names_with_path):
  1854. """
  1855. When doing an exclude against any kind of N-to-many relation, we need
  1856. to use a subquery. This method constructs the nested query, given the
  1857. original exclude filter (filter_expr) and the portion up to the first
  1858. N-to-many relation field.
  1859. For example, if the origin filter is ~Q(child__name='foo'), filter_expr
  1860. is ('child__name', 'foo') and can_reuse is a set of joins usable for
  1861. filters in the original query.
  1862. We will turn this into equivalent of:
  1863. WHERE NOT EXISTS(
  1864. SELECT 1
  1865. FROM child
  1866. WHERE name = 'foo' AND child.parent_id = parent.id
  1867. LIMIT 1
  1868. )
  1869. """
  1870. # Generate the inner query.
  1871. query = self.__class__(self.model)
  1872. query._filtered_relations = self._filtered_relations
  1873. filter_lhs, filter_rhs = filter_expr
  1874. if isinstance(filter_rhs, OuterRef):
  1875. filter_rhs = OuterRef(filter_rhs)
  1876. elif isinstance(filter_rhs, F):
  1877. filter_rhs = OuterRef(filter_rhs.name)
  1878. query.add_filter(filter_lhs, filter_rhs)
  1879. query.clear_ordering(force=True)
  1880. # Try to have as simple as possible subquery -> trim leading joins from
  1881. # the subquery.
  1882. trimmed_prefix, contains_louter = query.trim_start(names_with_path)
  1883. col = query.select[0]
  1884. select_field = col.target
  1885. alias = col.alias
  1886. if alias in can_reuse:
  1887. pk = select_field.model._meta.pk
  1888. # Need to add a restriction so that outer query's filters are in effect for
  1889. # the subquery, too.
  1890. query.bump_prefix(self)
  1891. lookup_class = select_field.get_lookup("exact")
  1892. # Note that the query.select[0].alias is different from alias
  1893. # due to bump_prefix above.
  1894. lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias))
  1895. query.where.add(lookup, AND)
  1896. query.external_aliases[alias] = True
  1897. else:
  1898. lookup_class = select_field.get_lookup("exact")
  1899. lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix))
  1900. query.where.add(lookup, AND)
  1901. condition, needed_inner = self.build_filter(Exists(query))
  1902. if contains_louter:
  1903. or_null_condition, _ = self.build_filter(
  1904. ("%s__isnull" % trimmed_prefix, True),
  1905. current_negated=True,
  1906. branch_negated=True,
  1907. can_reuse=can_reuse,
  1908. )
  1909. condition.add(or_null_condition, OR)
  1910. # Note that the end result will be:
  1911. # NOT EXISTS (inner_q) OR outercol IS NULL
  1912. # this might look crazy but due to how NULL works, this seems to be
  1913. # correct. If the IS NULL check is removed, then if outercol
  1914. # IS NULL we will not match the row.
  1915. return condition, needed_inner
  1916. def set_empty(self):
  1917. self.where.add(NothingNode(), AND)
  1918. for query in self.combined_queries:
  1919. query.set_empty()
  1920. def is_empty(self):
  1921. return any(isinstance(c, NothingNode) for c in self.where.children)
  1922. def set_limits(self, low=None, high=None):
  1923. """
  1924. Adjust the limits on the rows retrieved. Use low/high to set these,
  1925. as it makes it more Pythonic to read and write. When the SQL query is
  1926. created, convert them to the appropriate offset and limit values.
  1927. Apply any limits passed in here to the existing constraints. Add low
  1928. to the current low value and clamp both to any existing high value.
  1929. """
  1930. if high is not None:
  1931. if self.high_mark is not None:
  1932. self.high_mark = min(self.high_mark, self.low_mark + high)
  1933. else:
  1934. self.high_mark = self.low_mark + high
  1935. if low is not None:
  1936. if self.high_mark is not None:
  1937. self.low_mark = min(self.high_mark, self.low_mark + low)
  1938. else:
  1939. self.low_mark = self.low_mark + low
  1940. if self.low_mark == self.high_mark:
  1941. self.set_empty()
  1942. def clear_limits(self):
  1943. """Clear any existing limits."""
  1944. self.low_mark, self.high_mark = 0, None
  1945. @property
  1946. def is_sliced(self):
  1947. return self.low_mark != 0 or self.high_mark is not None
  1948. def has_limit_one(self):
  1949. return self.high_mark is not None and (self.high_mark - self.low_mark) == 1
  1950. def can_filter(self):
  1951. """
  1952. Return True if adding filters to this instance is still possible.
  1953. Typically, this means no limits or offsets have been put on the results.
  1954. """
  1955. return not self.is_sliced
  1956. def clear_select_clause(self):
  1957. """Remove all fields from SELECT clause."""
  1958. self.select = ()
  1959. self.default_cols = False
  1960. self.select_related = False
  1961. self.set_extra_mask(())
  1962. self.set_annotation_mask(())
  1963. def clear_select_fields(self):
  1964. """
  1965. Clear the list of fields to select (but not extra_select columns).
  1966. Some queryset types completely replace any existing list of select
  1967. columns.
  1968. """
  1969. self.select = ()
  1970. self.values_select = ()
  1971. def add_select_col(self, col, name):
  1972. self.select += (col,)
  1973. self.values_select += (name,)
  1974. def set_select(self, cols):
  1975. self.default_cols = False
  1976. self.select = tuple(cols)
  1977. def add_distinct_fields(self, *field_names):
  1978. """
  1979. Add and resolve the given fields to the query's "distinct on" clause.
  1980. """
  1981. self.distinct_fields = field_names
  1982. self.distinct = True
  1983. def add_fields(self, field_names, allow_m2m=True):
  1984. """
  1985. Add the given (model) fields to the select set. Add the field names in
  1986. the order specified.
  1987. """
  1988. alias = self.get_initial_alias()
  1989. opts = self.get_meta()
  1990. try:
  1991. cols = []
  1992. for name in field_names:
  1993. # Join promotion note - we must not remove any rows here, so
  1994. # if there is no existing joins, use outer join.
  1995. join_info = self.setup_joins(
  1996. name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m
  1997. )
  1998. targets, final_alias, joins = self.trim_joins(
  1999. join_info.targets,
  2000. join_info.joins,
  2001. join_info.path,
  2002. )
  2003. for target in targets:
  2004. cols.append(join_info.transform_function(target, final_alias))
  2005. if cols:
  2006. self.set_select(cols)
  2007. except MultiJoin:
  2008. raise FieldError("Invalid field name: '%s'" % name)
  2009. except FieldError:
  2010. if LOOKUP_SEP in name:
  2011. # For lookups spanning over relationships, show the error
  2012. # from the model on which the lookup failed.
  2013. raise
  2014. else:
  2015. names = sorted(
  2016. [
  2017. *get_field_names_from_opts(opts),
  2018. *self.extra,
  2019. *self.annotation_select,
  2020. *self._filtered_relations,
  2021. ]
  2022. )
  2023. raise FieldError(
  2024. "Cannot resolve keyword %r into field. "
  2025. "Choices are: %s" % (name, ", ".join(names))
  2026. )
  2027. def add_ordering(self, *ordering):
  2028. """
  2029. Add items from the 'ordering' sequence to the query's "order by"
  2030. clause. These items are either field names (not column names) --
  2031. possibly with a direction prefix ('-' or '?') -- or OrderBy
  2032. expressions.
  2033. If 'ordering' is empty, clear all ordering from the query.
  2034. """
  2035. errors = []
  2036. for item in ordering:
  2037. if isinstance(item, str):
  2038. if item == "?":
  2039. continue
  2040. item = item.removeprefix("-")
  2041. if item in self.annotations:
  2042. continue
  2043. if self.extra and item in self.extra:
  2044. continue
  2045. # names_to_path() validates the lookup. A descriptive
  2046. # FieldError will be raise if it's not.
  2047. self.names_to_path(item.split(LOOKUP_SEP), self.model._meta)
  2048. elif not hasattr(item, "resolve_expression"):
  2049. errors.append(item)
  2050. if getattr(item, "contains_aggregate", False):
  2051. raise FieldError(
  2052. "Using an aggregate in order_by() without also including "
  2053. "it in annotate() is not allowed: %s" % item
  2054. )
  2055. if errors:
  2056. raise FieldError("Invalid order_by arguments: %s" % errors)
  2057. if ordering:
  2058. self.order_by += ordering
  2059. else:
  2060. self.default_ordering = False
  2061. def clear_ordering(self, force=False, clear_default=True):
  2062. """
  2063. Remove any ordering settings if the current query allows it without
  2064. side effects, set 'force' to True to clear the ordering regardless.
  2065. If 'clear_default' is True, there will be no ordering in the resulting
  2066. query (not even the model's default).
  2067. """
  2068. if not force and (
  2069. self.is_sliced or self.distinct_fields or self.select_for_update
  2070. ):
  2071. return
  2072. self.order_by = ()
  2073. self.extra_order_by = ()
  2074. if clear_default:
  2075. self.default_ordering = False
  2076. def set_group_by(self, allow_aliases=True):
  2077. """
  2078. Expand the GROUP BY clause required by the query.
  2079. This will usually be the set of all non-aggregate fields in the
  2080. return data. If the database backend supports grouping by the
  2081. primary key, and the query would be equivalent, the optimization
  2082. will be made automatically.
  2083. """
  2084. if allow_aliases and self.values_select:
  2085. # If grouping by aliases is allowed assign selected value aliases
  2086. # by moving them to annotations.
  2087. group_by_annotations = {}
  2088. values_select = {}
  2089. for alias, expr in zip(self.values_select, self.select):
  2090. if isinstance(expr, Col):
  2091. values_select[alias] = expr
  2092. else:
  2093. group_by_annotations[alias] = expr
  2094. self.annotations = {**group_by_annotations, **self.annotations}
  2095. self.append_annotation_mask(group_by_annotations)
  2096. self.select = tuple(values_select.values())
  2097. self.values_select = tuple(values_select)
  2098. group_by = list(self.select)
  2099. for alias, annotation in self.annotation_select.items():
  2100. if not (group_by_cols := annotation.get_group_by_cols()):
  2101. continue
  2102. if allow_aliases and not annotation.contains_aggregate:
  2103. group_by.append(Ref(alias, annotation))
  2104. else:
  2105. group_by.extend(group_by_cols)
  2106. self.group_by = tuple(group_by)
  2107. def add_select_related(self, fields):
  2108. """
  2109. Set up the select_related data structure so that we only select
  2110. certain related models (as opposed to all models, when
  2111. self.select_related=True).
  2112. """
  2113. if isinstance(self.select_related, bool):
  2114. field_dict = {}
  2115. else:
  2116. field_dict = self.select_related
  2117. for field in fields:
  2118. d = field_dict
  2119. for part in field.split(LOOKUP_SEP):
  2120. d = d.setdefault(part, {})
  2121. self.select_related = field_dict
  2122. def add_extra(self, select, select_params, where, params, tables, order_by):
  2123. """
  2124. Add data to the various extra_* attributes for user-created additions
  2125. to the query.
  2126. """
  2127. if select:
  2128. # We need to pair any placeholder markers in the 'select'
  2129. # dictionary with their parameters in 'select_params' so that
  2130. # subsequent updates to the select dictionary also adjust the
  2131. # parameters appropriately.
  2132. select_pairs = {}
  2133. if select_params:
  2134. param_iter = iter(select_params)
  2135. else:
  2136. param_iter = iter([])
  2137. for name, entry in select.items():
  2138. self.check_alias(name)
  2139. entry = str(entry)
  2140. entry_params = []
  2141. pos = entry.find("%s")
  2142. while pos != -1:
  2143. if pos == 0 or entry[pos - 1] != "%":
  2144. entry_params.append(next(param_iter))
  2145. pos = entry.find("%s", pos + 2)
  2146. select_pairs[name] = (entry, entry_params)
  2147. self.extra.update(select_pairs)
  2148. if where or params:
  2149. self.where.add(ExtraWhere(where, params), AND)
  2150. if tables:
  2151. self.extra_tables += tuple(tables)
  2152. if order_by:
  2153. self.extra_order_by = order_by
  2154. def clear_deferred_loading(self):
  2155. """Remove any fields from the deferred loading set."""
  2156. self.deferred_loading = (frozenset(), True)
  2157. def add_deferred_loading(self, field_names):
  2158. """
  2159. Add the given list of model field names to the set of fields to
  2160. exclude from loading from the database when automatic column selection
  2161. is done. Add the new field names to any existing field names that
  2162. are deferred (or removed from any existing field names that are marked
  2163. as the only ones for immediate loading).
  2164. """
  2165. # Fields on related models are stored in the literal double-underscore
  2166. # format, so that we can use a set datastructure. We do the foo__bar
  2167. # splitting and handling when computing the SQL column names (as part of
  2168. # get_columns()).
  2169. existing, defer = self.deferred_loading
  2170. if defer:
  2171. # Add to existing deferred names.
  2172. self.deferred_loading = existing.union(field_names), True
  2173. else:
  2174. # Remove names from the set of any existing "immediate load" names.
  2175. if new_existing := existing.difference(field_names):
  2176. self.deferred_loading = new_existing, False
  2177. else:
  2178. self.clear_deferred_loading()
  2179. if new_only := set(field_names).difference(existing):
  2180. self.deferred_loading = new_only, True
  2181. def add_immediate_loading(self, field_names):
  2182. """
  2183. Add the given list of model field names to the set of fields to
  2184. retrieve when the SQL is executed ("immediate loading" fields). The
  2185. field names replace any existing immediate loading field names. If
  2186. there are field names already specified for deferred loading, remove
  2187. those names from the new field_names before storing the new names
  2188. for immediate loading. (That is, immediate loading overrides any
  2189. existing immediate values, but respects existing deferrals.)
  2190. """
  2191. existing, defer = self.deferred_loading
  2192. field_names = set(field_names)
  2193. if "pk" in field_names:
  2194. field_names.remove("pk")
  2195. field_names.add(self.get_meta().pk.name)
  2196. if defer:
  2197. # Remove any existing deferred names from the current set before
  2198. # setting the new names.
  2199. self.deferred_loading = field_names.difference(existing), False
  2200. else:
  2201. # Replace any existing "immediate load" field names.
  2202. self.deferred_loading = frozenset(field_names), False
  2203. def set_annotation_mask(self, names):
  2204. """Set the mask of annotations that will be returned by the SELECT."""
  2205. if names is None:
  2206. self.annotation_select_mask = None
  2207. else:
  2208. self.annotation_select_mask = list(dict.fromkeys(names))
  2209. self._annotation_select_cache = None
  2210. def append_annotation_mask(self, names):
  2211. if self.annotation_select_mask is not None:
  2212. self.set_annotation_mask((*self.annotation_select_mask, *names))
  2213. def set_extra_mask(self, names):
  2214. """
  2215. Set the mask of extra select items that will be returned by SELECT.
  2216. Don't remove them from the Query since they might be used later.
  2217. """
  2218. if names is None:
  2219. self.extra_select_mask = None
  2220. else:
  2221. self.extra_select_mask = set(names)
  2222. self._extra_select_cache = None
  2223. def set_values(self, fields):
  2224. self.select_related = False
  2225. self.clear_deferred_loading()
  2226. self.clear_select_fields()
  2227. self.has_select_fields = True
  2228. if fields:
  2229. for field in fields:
  2230. self.check_alias(field)
  2231. field_names = []
  2232. extra_names = []
  2233. annotation_names = []
  2234. if not self.extra and not self.annotations:
  2235. # Shortcut - if there are no extra or annotations, then
  2236. # the values() clause must be just field names.
  2237. field_names = list(fields)
  2238. else:
  2239. self.default_cols = False
  2240. for f in fields:
  2241. if f in self.extra_select:
  2242. extra_names.append(f)
  2243. elif f in self.annotation_select:
  2244. annotation_names.append(f)
  2245. elif f in self.annotations:
  2246. raise FieldError(
  2247. f"Cannot select the '{f}' alias. Use annotate() to "
  2248. "promote it."
  2249. )
  2250. else:
  2251. # Call `names_to_path` to ensure a FieldError including
  2252. # annotations about to be masked as valid choices if
  2253. # `f` is not resolvable.
  2254. if self.annotation_select:
  2255. self.names_to_path(f.split(LOOKUP_SEP), self.model._meta)
  2256. field_names.append(f)
  2257. self.set_extra_mask(extra_names)
  2258. self.set_annotation_mask(annotation_names)
  2259. selected = frozenset(field_names + extra_names + annotation_names)
  2260. else:
  2261. field_names = [f.attname for f in self.model._meta.concrete_fields]
  2262. selected = frozenset(field_names)
  2263. # Selected annotations must be known before setting the GROUP BY
  2264. # clause.
  2265. if self.group_by is True:
  2266. self.add_fields(
  2267. (f.attname for f in self.model._meta.concrete_fields), False
  2268. )
  2269. # Disable GROUP BY aliases to avoid orphaning references to the
  2270. # SELECT clause which is about to be cleared.
  2271. self.set_group_by(allow_aliases=False)
  2272. self.clear_select_fields()
  2273. elif self.group_by:
  2274. # Resolve GROUP BY annotation references if they are not part of
  2275. # the selected fields anymore.
  2276. group_by = []
  2277. for expr in self.group_by:
  2278. if isinstance(expr, Ref) and expr.refs not in selected:
  2279. expr = self.annotations[expr.refs]
  2280. group_by.append(expr)
  2281. self.group_by = tuple(group_by)
  2282. self.values_select = tuple(field_names)
  2283. self.add_fields(field_names, True)
  2284. @property
  2285. def annotation_select(self):
  2286. """
  2287. Return the dictionary of aggregate columns that are not masked and
  2288. should be used in the SELECT clause. Cache this result for performance.
  2289. """
  2290. if self._annotation_select_cache is not None:
  2291. return self._annotation_select_cache
  2292. elif not self.annotations:
  2293. return {}
  2294. elif self.annotation_select_mask is not None:
  2295. self._annotation_select_cache = {
  2296. k: self.annotations[k]
  2297. for k in self.annotation_select_mask
  2298. if k in self.annotations
  2299. }
  2300. return self._annotation_select_cache
  2301. else:
  2302. return self.annotations
  2303. @property
  2304. def extra_select(self):
  2305. if self._extra_select_cache is not None:
  2306. return self._extra_select_cache
  2307. if not self.extra:
  2308. return {}
  2309. elif self.extra_select_mask is not None:
  2310. self._extra_select_cache = {
  2311. k: v for k, v in self.extra.items() if k in self.extra_select_mask
  2312. }
  2313. return self._extra_select_cache
  2314. else:
  2315. return self.extra
  2316. def trim_start(self, names_with_path):
  2317. """
  2318. Trim joins from the start of the join path. The candidates for trim
  2319. are the PathInfos in names_with_path structure that are m2m joins.
  2320. Also set the select column so the start matches the join.
  2321. This method is meant to be used for generating the subquery joins &
  2322. cols in split_exclude().
  2323. Return a lookup usable for doing outerq.filter(lookup=self) and a
  2324. boolean indicating if the joins in the prefix contain a LEFT OUTER join.
  2325. _"""
  2326. all_paths = []
  2327. for _, paths in names_with_path:
  2328. all_paths.extend(paths)
  2329. contains_louter = False
  2330. # Trim and operate only on tables that were generated for
  2331. # the lookup part of the query. That is, avoid trimming
  2332. # joins generated for F() expressions.
  2333. lookup_tables = [
  2334. t for t in self.alias_map if t in self._lookup_joins or t == self.base_table
  2335. ]
  2336. for trimmed_paths, path in enumerate(all_paths):
  2337. if path.m2m:
  2338. break
  2339. if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
  2340. contains_louter = True
  2341. alias = lookup_tables[trimmed_paths]
  2342. self.unref_alias(alias)
  2343. # The path.join_field is a Rel, lets get the other side's field
  2344. join_field = path.join_field.field
  2345. # Build the filter prefix.
  2346. paths_in_prefix = trimmed_paths
  2347. trimmed_prefix = []
  2348. for name, path in names_with_path:
  2349. if paths_in_prefix - len(path) < 0:
  2350. break
  2351. trimmed_prefix.append(name)
  2352. paths_in_prefix -= len(path)
  2353. trimmed_prefix.append(join_field.foreign_related_fields[0].name)
  2354. trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
  2355. # Lets still see if we can trim the first join from the inner query
  2356. # (that is, self). We can't do this for:
  2357. # - LEFT JOINs because we would miss those rows that have nothing on
  2358. # the outer side,
  2359. # - INNER JOINs from filtered relations because we would miss their
  2360. # filters.
  2361. first_join = self.alias_map[lookup_tables[trimmed_paths + 1]]
  2362. if first_join.join_type != LOUTER and not first_join.filtered_relation:
  2363. select_fields = [r[0] for r in join_field.related_fields]
  2364. select_alias = lookup_tables[trimmed_paths + 1]
  2365. self.unref_alias(lookup_tables[trimmed_paths])
  2366. extra_restriction = join_field.get_extra_restriction(
  2367. None, lookup_tables[trimmed_paths + 1]
  2368. )
  2369. if extra_restriction:
  2370. self.where.add(extra_restriction, AND)
  2371. else:
  2372. # TODO: It might be possible to trim more joins from the start of the
  2373. # inner query if it happens to have a longer join chain containing the
  2374. # values in select_fields. Lets punt this one for now.
  2375. select_fields = [r[1] for r in join_field.related_fields]
  2376. select_alias = lookup_tables[trimmed_paths]
  2377. # The found starting point is likely a join_class instead of a
  2378. # base_table_class reference. But the first entry in the query's FROM
  2379. # clause must not be a JOIN.
  2380. for table in self.alias_map:
  2381. if self.alias_refcount[table] > 0:
  2382. self.alias_map[table] = self.base_table_class(
  2383. self.alias_map[table].table_name,
  2384. table,
  2385. )
  2386. break
  2387. self.set_select([f.get_col(select_alias) for f in select_fields])
  2388. return trimmed_prefix, contains_louter
  2389. def is_nullable(self, field):
  2390. """
  2391. Check if the given field should be treated as nullable.
  2392. Some backends treat '' as null and Django treats such fields as
  2393. nullable for those backends. In such situations field.null can be
  2394. False even if we should treat the field as nullable.
  2395. """
  2396. # We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have
  2397. # (nor should it have) knowledge of which connection is going to be
  2398. # used. The proper fix would be to defer all decisions where
  2399. # is_nullable() is needed to the compiler stage, but that is not easy
  2400. # to do currently.
  2401. return field.null or (
  2402. field.empty_strings_allowed
  2403. and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
  2404. )
  2405. def get_order_dir(field, default="ASC"):
  2406. """
  2407. Return the field name and direction for an order specification. For
  2408. example, '-foo' is returned as ('foo', 'DESC').
  2409. The 'default' param is used to indicate which way no prefix (or a '+'
  2410. prefix) should sort. The '-' prefix always sorts the opposite way.
  2411. """
  2412. dirn = ORDER_DIR[default]
  2413. if field[0] == "-":
  2414. return field[1:], dirn[1]
  2415. return field, dirn[0]
  2416. class JoinPromoter:
  2417. """
  2418. A class to abstract away join promotion problems for complex filter
  2419. conditions.
  2420. """
  2421. def __init__(self, connector, num_children, negated):
  2422. self.connector = connector
  2423. self.negated = negated
  2424. if self.negated:
  2425. if connector == AND:
  2426. self.effective_connector = OR
  2427. else:
  2428. self.effective_connector = AND
  2429. else:
  2430. self.effective_connector = self.connector
  2431. self.num_children = num_children
  2432. # Maps of table alias to how many times it is seen as required for
  2433. # inner and/or outer joins.
  2434. self.votes = Counter()
  2435. def __repr__(self):
  2436. return (
  2437. f"{self.__class__.__qualname__}(connector={self.connector!r}, "
  2438. f"num_children={self.num_children!r}, negated={self.negated!r})"
  2439. )
  2440. def add_votes(self, votes):
  2441. """
  2442. Add single vote per item to self.votes. Parameter can be any
  2443. iterable.
  2444. """
  2445. self.votes.update(votes)
  2446. def update_join_types(self, query):
  2447. """
  2448. Change join types so that the generated query is as efficient as
  2449. possible, but still correct. So, change as many joins as possible
  2450. to INNER, but don't make OUTER joins INNER if that could remove
  2451. results from the query.
  2452. """
  2453. to_promote = set()
  2454. to_demote = set()
  2455. # The effective_connector is used so that NOT (a AND b) is treated
  2456. # similarly to (a OR b) for join promotion.
  2457. for table, votes in self.votes.items():
  2458. # We must use outer joins in OR case when the join isn't contained
  2459. # in all of the joins. Otherwise the INNER JOIN itself could remove
  2460. # valid results. Consider the case where a model with rel_a and
  2461. # rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now,
  2462. # if rel_a join doesn't produce any results is null (for example
  2463. # reverse foreign key or null value in direct foreign key), and
  2464. # there is a matching row in rel_b with col=2, then an INNER join
  2465. # to rel_a would remove a valid match from the query. So, we need
  2466. # to promote any existing INNER to LOUTER (it is possible this
  2467. # promotion in turn will be demoted later on).
  2468. if self.effective_connector == OR and votes < self.num_children:
  2469. to_promote.add(table)
  2470. # If connector is AND and there is a filter that can match only
  2471. # when there is a joinable row, then use INNER. For example, in
  2472. # rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL
  2473. # as join output, then the col=1 or col=2 can't match (as
  2474. # NULL=anything is always false).
  2475. # For the OR case, if all children voted for a join to be inner,
  2476. # then we can use INNER for the join. For example:
  2477. # (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell)
  2478. # then if rel_a doesn't produce any rows, the whole condition
  2479. # can't match. Hence we can safely use INNER join.
  2480. if self.effective_connector == AND or (
  2481. self.effective_connector == OR and votes == self.num_children
  2482. ):
  2483. to_demote.add(table)
  2484. # Finally, what happens in cases where we have:
  2485. # (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0
  2486. # Now, we first generate the OR clause, and promote joins for it
  2487. # in the first if branch above. Both rel_a and rel_b are promoted
  2488. # to LOUTER joins. After that we do the AND case. The OR case
  2489. # voted no inner joins but the rel_a__col__gte=0 votes inner join
  2490. # for rel_a. We demote it back to INNER join (in AND case a single
  2491. # vote is enough). The demotion is OK, if rel_a doesn't produce
  2492. # rows, then the rel_a__col__gte=0 clause can't be true, and thus
  2493. # the whole clause must be false. So, it is safe to use INNER
  2494. # join.
  2495. # Note that in this example we could just as well have the __gte
  2496. # clause and the OR clause swapped. Or we could replace the __gte
  2497. # clause with an OR clause containing rel_a__col=1|rel_a__col=2,
  2498. # and again we could safely demote to INNER.
  2499. query.promote_joins(to_promote)
  2500. query.demote_joins(to_demote)
  2501. return to_demote