Coverage for /Users/davegaeddert/Developer/dropseed/plain/plain-models/plain/models/sql/query.py: 46%
1276 statements
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-23 11:16 -0600
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-23 11:16 -0600
1"""
2Create SQL statements for QuerySets.
4The code in here encapsulates all of the SQL construction so that QuerySets
5themselves do not have to (and could be backed by things other than SQL
6databases). The abstraction barrier only works one way: this module has to know
7all about the internals of models in order to get the information it needs.
8"""
10import copy
11import difflib
12import functools
13import sys
14from collections import Counter, namedtuple
15from collections.abc import Iterator, Mapping
16from itertools import chain, count, product
17from string import ascii_uppercase
19from plain.exceptions import FieldDoesNotExist, FieldError
20from plain.models.aggregates import Count
21from plain.models.constants import LOOKUP_SEP
22from plain.models.db import DEFAULT_DB_ALIAS, NotSupportedError, connections
23from plain.models.expressions import (
24 BaseExpression,
25 Col,
26 Exists,
27 F,
28 OuterRef,
29 Ref,
30 ResolvedOuterRef,
31 Value,
32)
33from plain.models.fields import Field
34from plain.models.fields.related_lookups import MultiColSource
35from plain.models.lookups import Lookup
36from plain.models.query_utils import (
37 Q,
38 check_rel_lookup_compatibility,
39 refs_expression,
40)
41from plain.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE
42from plain.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin
43from plain.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode
44from plain.utils.functional import cached_property
45from plain.utils.regex_helper import _lazy_re_compile
46from plain.utils.tree import Node
48__all__ = ["Query", "RawQuery"]
50# Quotation marks ('"`[]), whitespace characters, semicolons, or inline
51# SQL comments are forbidden in column aliases.
52FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/")
54# Inspired from
55# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
56EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w\-]+")
59def get_field_names_from_opts(opts):
60 if opts is None:
61 return set()
62 return set(
63 chain.from_iterable(
64 (f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields()
65 )
66 )
69def get_children_from_q(q):
70 for child in q.children:
71 if isinstance(child, Node):
72 yield from get_children_from_q(child)
73 else:
74 yield child
77JoinInfo = namedtuple(
78 "JoinInfo",
79 ("final_field", "targets", "opts", "joins", "path", "transform_function"),
80)
83class RawQuery:
84 """A single raw SQL query."""
86 def __init__(self, sql, using, params=()):
87 self.params = params
88 self.sql = sql
89 self.using = using
90 self.cursor = None
92 # Mirror some properties of a normal query so that
93 # the compiler can be used to process results.
94 self.low_mark, self.high_mark = 0, None # Used for offset/limit
95 self.extra_select = {}
96 self.annotation_select = {}
98 def chain(self, using):
99 return self.clone(using)
101 def clone(self, using):
102 return RawQuery(self.sql, using, params=self.params)
104 def get_columns(self):
105 if self.cursor is None:
106 self._execute_query()
107 converter = connections[self.using].introspection.identifier_converter
108 return [converter(column_meta[0]) for column_meta in self.cursor.description]
110 def __iter__(self):
111 # Always execute a new query for a new iterator.
112 # This could be optimized with a cache at the expense of RAM.
113 self._execute_query()
114 if not connections[self.using].features.can_use_chunked_reads:
115 # If the database can't use chunked reads we need to make sure we
116 # evaluate the entire query up front.
117 result = list(self.cursor)
118 else:
119 result = self.cursor
120 return iter(result)
122 def __repr__(self):
123 return f"<{self.__class__.__name__}: {self}>"
125 @property
126 def params_type(self):
127 if self.params is None:
128 return None
129 return dict if isinstance(self.params, Mapping) else tuple
131 def __str__(self):
132 if self.params_type is None:
133 return self.sql
134 return self.sql % self.params_type(self.params)
136 def _execute_query(self):
137 connection = connections[self.using]
139 # Adapt parameters to the database, as much as possible considering
140 # that the target type isn't known. See #17755.
141 params_type = self.params_type
142 adapter = connection.ops.adapt_unknown_value
143 if params_type is tuple:
144 params = tuple(adapter(val) for val in self.params)
145 elif params_type is dict:
146 params = {key: adapter(val) for key, val in self.params.items()}
147 elif params_type is None:
148 params = None
149 else:
150 raise RuntimeError(f"Unexpected params type: {params_type}")
152 self.cursor = connection.cursor()
153 self.cursor.execute(self.sql, params)
156ExplainInfo = namedtuple("ExplainInfo", ("format", "options"))
159class Query(BaseExpression):
160 """A single SQL query."""
162 alias_prefix = "T"
163 empty_result_set_value = None
164 subq_aliases = frozenset([alias_prefix])
166 compiler = "SQLCompiler"
168 base_table_class = BaseTable
169 join_class = Join
171 default_cols = True
172 default_ordering = True
173 standard_ordering = True
175 filter_is_sticky = False
176 subquery = False
178 # SQL-related attributes.
179 # Select and related select clauses are expressions to use in the SELECT
180 # clause of the query. The select is used for cases where we want to set up
181 # the select clause to contain other than default fields (values(),
182 # subqueries...). Note that annotations go to annotations dictionary.
183 select = ()
184 # The group_by attribute can have one of the following forms:
185 # - None: no group by at all in the query
186 # - A tuple of expressions: group by (at least) those expressions.
187 # String refs are also allowed for now.
188 # - True: group by all select fields of the model
189 # See compiler.get_group_by() for details.
190 group_by = None
191 order_by = ()
192 low_mark = 0 # Used for offset/limit.
193 high_mark = None # Used for offset/limit.
194 distinct = False
195 distinct_fields = ()
196 select_for_update = False
197 select_for_update_nowait = False
198 select_for_update_skip_locked = False
199 select_for_update_of = ()
200 select_for_no_key_update = False
201 select_related = False
202 has_select_fields = False
203 # Arbitrary limit for select_related to prevents infinite recursion.
204 max_depth = 5
205 # Holds the selects defined by a call to values() or values_list()
206 # excluding annotation_select and extra_select.
207 values_select = ()
209 # SQL annotation-related attributes.
210 annotation_select_mask = None
211 _annotation_select_cache = None
213 # Set combination attributes.
214 combinator = None
215 combinator_all = False
216 combined_queries = ()
218 # These are for extensions. The contents are more or less appended verbatim
219 # to the appropriate clause.
220 extra_select_mask = None
221 _extra_select_cache = None
223 extra_tables = ()
224 extra_order_by = ()
226 # A tuple that is a set of model field names and either True, if these are
227 # the fields to defer, or False if these are the only fields to load.
228 deferred_loading = (frozenset(), True)
230 explain_info = None
232 def __init__(self, model, alias_cols=True):
233 self.model = model
234 self.alias_refcount = {}
235 # alias_map is the most important data structure regarding joins.
236 # It's used for recording which joins exist in the query and what
237 # types they are. The key is the alias of the joined table (possibly
238 # the table name) and the value is a Join-like object (see
239 # sql.datastructures.Join for more information).
240 self.alias_map = {}
241 # Whether to provide alias to columns during reference resolving.
242 self.alias_cols = alias_cols
243 # Sometimes the query contains references to aliases in outer queries (as
244 # a result of split_exclude). Correct alias quoting needs to know these
245 # aliases too.
246 # Map external tables to whether they are aliased.
247 self.external_aliases = {}
248 self.table_map = {} # Maps table names to list of aliases.
249 self.used_aliases = set()
251 self.where = WhereNode()
252 # Maps alias -> Annotation Expression.
253 self.annotations = {}
254 # These are for extensions. The contents are more or less appended
255 # verbatim to the appropriate clause.
256 self.extra = {} # Maps col_alias -> (col_sql, params).
258 self._filtered_relations = {}
260 @property
261 def output_field(self):
262 if len(self.select) == 1:
263 select = self.select[0]
264 return getattr(select, "target", None) or select.field
265 elif len(self.annotation_select) == 1:
266 return next(iter(self.annotation_select.values())).output_field
268 @cached_property
269 def base_table(self):
270 for alias in self.alias_map:
271 return alias
273 def __str__(self):
274 """
275 Return the query as a string of SQL with the parameter values
276 substituted in (use sql_with_params() to see the unsubstituted string).
278 Parameter values won't necessarily be quoted correctly, since that is
279 done by the database interface at execution time.
280 """
281 sql, params = self.sql_with_params()
282 return sql % params
284 def sql_with_params(self):
285 """
286 Return the query as an SQL string and the parameters that will be
287 substituted into the query.
288 """
289 return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()
291 def __deepcopy__(self, memo):
292 """Limit the amount of work when a Query is deepcopied."""
293 result = self.clone()
294 memo[id(self)] = result
295 return result
297 def get_compiler(self, using=None, connection=None, elide_empty=True):
298 if using is None and connection is None:
299 raise ValueError("Need either using or connection")
300 if using:
301 connection = connections[using]
302 return connection.ops.compiler(self.compiler)(
303 self, connection, using, elide_empty
304 )
306 def get_meta(self):
307 """
308 Return the Options instance (the model._meta) from which to start
309 processing. Normally, this is self.model._meta, but it can be changed
310 by subclasses.
311 """
312 if self.model:
313 return self.model._meta
315 def clone(self):
316 """
317 Return a copy of the current Query. A lightweight alternative to
318 deepcopy().
319 """
320 obj = Empty()
321 obj.__class__ = self.__class__
322 # Copy references to everything.
323 obj.__dict__ = self.__dict__.copy()
324 # Clone attributes that can't use shallow copy.
325 obj.alias_refcount = self.alias_refcount.copy()
326 obj.alias_map = self.alias_map.copy()
327 obj.external_aliases = self.external_aliases.copy()
328 obj.table_map = self.table_map.copy()
329 obj.where = self.where.clone()
330 obj.annotations = self.annotations.copy()
331 if self.annotation_select_mask is not None:
332 obj.annotation_select_mask = self.annotation_select_mask.copy()
333 if self.combined_queries:
334 obj.combined_queries = tuple(
335 [query.clone() for query in self.combined_queries]
336 )
337 # _annotation_select_cache cannot be copied, as doing so breaks the
338 # (necessary) state in which both annotations and
339 # _annotation_select_cache point to the same underlying objects.
340 # It will get re-populated in the cloned queryset the next time it's
341 # used.
342 obj._annotation_select_cache = None
343 obj.extra = self.extra.copy()
344 if self.extra_select_mask is not None:
345 obj.extra_select_mask = self.extra_select_mask.copy()
346 if self._extra_select_cache is not None:
347 obj._extra_select_cache = self._extra_select_cache.copy()
348 if self.select_related is not False:
349 # Use deepcopy because select_related stores fields in nested
350 # dicts.
351 obj.select_related = copy.deepcopy(obj.select_related)
352 if "subq_aliases" in self.__dict__:
353 obj.subq_aliases = self.subq_aliases.copy()
354 obj.used_aliases = self.used_aliases.copy()
355 obj._filtered_relations = self._filtered_relations.copy()
356 # Clear the cached_property, if it exists.
357 obj.__dict__.pop("base_table", None)
358 return obj
360 def chain(self, klass=None):
361 """
362 Return a copy of the current Query that's ready for another operation.
363 The klass argument changes the type of the Query, e.g. UpdateQuery.
364 """
365 obj = self.clone()
366 if klass and obj.__class__ != klass:
367 obj.__class__ = klass
368 if not obj.filter_is_sticky:
369 obj.used_aliases = set()
370 obj.filter_is_sticky = False
371 if hasattr(obj, "_setup_query"):
372 obj._setup_query()
373 return obj
375 def relabeled_clone(self, change_map):
376 clone = self.clone()
377 clone.change_aliases(change_map)
378 return clone
380 def _get_col(self, target, field, alias):
381 if not self.alias_cols:
382 alias = None
383 return target.get_col(alias, field)
385 def get_aggregation(self, using, aggregate_exprs):
386 """
387 Return the dictionary with the values of the existing aggregations.
388 """
389 if not aggregate_exprs:
390 return {}
391 aggregates = {}
392 for alias, aggregate_expr in aggregate_exprs.items():
393 self.check_alias(alias)
394 aggregate = aggregate_expr.resolve_expression(
395 self, allow_joins=True, reuse=None, summarize=True
396 )
397 if not aggregate.contains_aggregate:
398 raise TypeError(f"{alias} is not an aggregate expression")
399 aggregates[alias] = aggregate
400 # Existing usage of aggregation can be determined by the presence of
401 # selected aggregates but also by filters against aliased aggregates.
402 _, having, qualify = self.where.split_having_qualify()
403 has_existing_aggregation = (
404 any(
405 getattr(annotation, "contains_aggregate", True)
406 for annotation in self.annotations.values()
407 )
408 or having
409 )
410 # Decide if we need to use a subquery.
411 #
412 # Existing aggregations would cause incorrect results as
413 # get_aggregation() must produce just one result and thus must not use
414 # GROUP BY.
415 #
416 # If the query has limit or distinct, or uses set operations, then
417 # those operations must be done in a subquery so that the query
418 # aggregates on the limit and/or distinct results instead of applying
419 # the distinct and limit after the aggregation.
420 if (
421 isinstance(self.group_by, tuple)
422 or self.is_sliced
423 or has_existing_aggregation
424 or qualify
425 or self.distinct
426 or self.combinator
427 ):
428 from plain.models.sql.subqueries import AggregateQuery
430 inner_query = self.clone()
431 inner_query.subquery = True
432 outer_query = AggregateQuery(self.model, inner_query)
433 inner_query.select_for_update = False
434 inner_query.select_related = False
435 inner_query.set_annotation_mask(self.annotation_select)
436 # Queries with distinct_fields need ordering and when a limit is
437 # applied we must take the slice from the ordered query. Otherwise
438 # no need for ordering.
439 inner_query.clear_ordering(force=False)
440 if not inner_query.distinct:
441 # If the inner query uses default select and it has some
442 # aggregate annotations, then we must make sure the inner
443 # query is grouped by the main model's primary key. However,
444 # clearing the select clause can alter results if distinct is
445 # used.
446 if inner_query.default_cols and has_existing_aggregation:
447 inner_query.group_by = (
448 self.model._meta.pk.get_col(inner_query.get_initial_alias()),
449 )
450 inner_query.default_cols = False
451 if not qualify:
452 # Mask existing annotations that are not referenced by
453 # aggregates to be pushed to the outer query unless
454 # filtering against window functions is involved as it
455 # requires complex realising.
456 annotation_mask = set()
457 for aggregate in aggregates.values():
458 annotation_mask |= aggregate.get_refs()
459 inner_query.set_annotation_mask(annotation_mask)
461 # Add aggregates to the outer AggregateQuery. This requires making
462 # sure all columns referenced by the aggregates are selected in the
463 # inner query. It is achieved by retrieving all column references
464 # by the aggregates, explicitly selecting them in the inner query,
465 # and making sure the aggregates are repointed to them.
466 col_refs = {}
467 for alias, aggregate in aggregates.items():
468 replacements = {}
469 for col in self._gen_cols([aggregate], resolve_refs=False):
470 if not (col_ref := col_refs.get(col)):
471 index = len(col_refs) + 1
472 col_alias = f"__col{index}"
473 col_ref = Ref(col_alias, col)
474 col_refs[col] = col_ref
475 inner_query.annotations[col_alias] = col
476 inner_query.append_annotation_mask([col_alias])
477 replacements[col] = col_ref
478 outer_query.annotations[alias] = aggregate.replace_expressions(
479 replacements
480 )
481 if (
482 inner_query.select == ()
483 and not inner_query.default_cols
484 and not inner_query.annotation_select_mask
485 ):
486 # In case of Model.objects[0:3].count(), there would be no
487 # field selected in the inner query, yet we must use a subquery.
488 # So, make sure at least one field is selected.
489 inner_query.select = (
490 self.model._meta.pk.get_col(inner_query.get_initial_alias()),
491 )
492 else:
493 outer_query = self
494 self.select = ()
495 self.default_cols = False
496 self.extra = {}
497 if self.annotations:
498 # Inline reference to existing annotations and mask them as
499 # they are unnecessary given only the summarized aggregations
500 # are requested.
501 replacements = {
502 Ref(alias, annotation): annotation
503 for alias, annotation in self.annotations.items()
504 }
505 self.annotations = {
506 alias: aggregate.replace_expressions(replacements)
507 for alias, aggregate in aggregates.items()
508 }
509 else:
510 self.annotations = aggregates
511 self.set_annotation_mask(aggregates)
513 empty_set_result = [
514 expression.empty_result_set_value
515 for expression in outer_query.annotation_select.values()
516 ]
517 elide_empty = not any(result is NotImplemented for result in empty_set_result)
518 outer_query.clear_ordering(force=True)
519 outer_query.clear_limits()
520 outer_query.select_for_update = False
521 outer_query.select_related = False
522 compiler = outer_query.get_compiler(using, elide_empty=elide_empty)
523 result = compiler.execute_sql(SINGLE)
524 if result is None:
525 result = empty_set_result
526 else:
527 converters = compiler.get_converters(outer_query.annotation_select.values())
528 result = next(compiler.apply_converters((result,), converters))
530 return dict(zip(outer_query.annotation_select, result))
532 def get_count(self, using):
533 """
534 Perform a COUNT() query using the current filter constraints.
535 """
536 obj = self.clone()
537 return obj.get_aggregation(using, {"__count": Count("*")})["__count"]
539 def has_filters(self):
540 return self.where
542 def exists(self, limit=True):
543 q = self.clone()
544 if not (q.distinct and q.is_sliced):
545 if q.group_by is True:
546 q.add_fields(
547 (f.attname for f in self.model._meta.concrete_fields), False
548 )
549 # Disable GROUP BY aliases to avoid orphaning references to the
550 # SELECT clause which is about to be cleared.
551 q.set_group_by(allow_aliases=False)
552 q.clear_select_clause()
553 if q.combined_queries and q.combinator == "union":
554 q.combined_queries = tuple(
555 combined_query.exists(limit=False)
556 for combined_query in q.combined_queries
557 )
558 q.clear_ordering(force=True)
559 if limit:
560 q.set_limits(high=1)
561 q.add_annotation(Value(1), "a")
562 return q
564 def has_results(self, using):
565 q = self.exists(using)
566 compiler = q.get_compiler(using=using)
567 return compiler.has_results()
569 def explain(self, using, format=None, **options):
570 q = self.clone()
571 for option_name in options:
572 if (
573 not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name)
574 or "--" in option_name
575 ):
576 raise ValueError(f"Invalid option name: {option_name!r}.")
577 q.explain_info = ExplainInfo(format, options)
578 compiler = q.get_compiler(using=using)
579 return "\n".join(compiler.explain_query())
581 def combine(self, rhs, connector):
582 """
583 Merge the 'rhs' query into the current one (with any 'rhs' effects
584 being applied *after* (that is, "to the right of") anything in the
585 current query. 'rhs' is not modified during a call to this function.
587 The 'connector' parameter describes how to connect filters from the
588 'rhs' query.
589 """
590 if self.model != rhs.model:
591 raise TypeError("Cannot combine queries on two different base models.")
592 if self.is_sliced:
593 raise TypeError("Cannot combine queries once a slice has been taken.")
594 if self.distinct != rhs.distinct:
595 raise TypeError("Cannot combine a unique query with a non-unique query.")
596 if self.distinct_fields != rhs.distinct_fields:
597 raise TypeError("Cannot combine queries with different distinct fields.")
599 # If lhs and rhs shares the same alias prefix, it is possible to have
600 # conflicting alias changes like T4 -> T5, T5 -> T6, which might end up
601 # as T4 -> T6 while combining two querysets. To prevent this, change an
602 # alias prefix of the rhs and update current aliases accordingly,
603 # except if the alias is the base table since it must be present in the
604 # query on both sides.
605 initial_alias = self.get_initial_alias()
606 rhs.bump_prefix(self, exclude={initial_alias})
608 # Work out how to relabel the rhs aliases, if necessary.
609 change_map = {}
610 conjunction = connector == AND
612 # Determine which existing joins can be reused. When combining the
613 # query with AND we must recreate all joins for m2m filters. When
614 # combining with OR we can reuse joins. The reason is that in AND
615 # case a single row can't fulfill a condition like:
616 # revrel__col=1 & revrel__col=2
617 # But, there might be two different related rows matching this
618 # condition. In OR case a single True is enough, so single row is
619 # enough, too.
620 #
621 # Note that we will be creating duplicate joins for non-m2m joins in
622 # the AND case. The results will be correct but this creates too many
623 # joins. This is something that could be fixed later on.
624 reuse = set() if conjunction else set(self.alias_map)
625 joinpromoter = JoinPromoter(connector, 2, False)
626 joinpromoter.add_votes(
627 j for j in self.alias_map if self.alias_map[j].join_type == INNER
628 )
629 rhs_votes = set()
630 # Now, add the joins from rhs query into the new query (skipping base
631 # table).
632 rhs_tables = list(rhs.alias_map)[1:]
633 for alias in rhs_tables:
634 join = rhs.alias_map[alias]
635 # If the left side of the join was already relabeled, use the
636 # updated alias.
637 join = join.relabeled_clone(change_map)
638 new_alias = self.join(join, reuse=reuse)
639 if join.join_type == INNER:
640 rhs_votes.add(new_alias)
641 # We can't reuse the same join again in the query. If we have two
642 # distinct joins for the same connection in rhs query, then the
643 # combined query must have two joins, too.
644 reuse.discard(new_alias)
645 if alias != new_alias:
646 change_map[alias] = new_alias
647 if not rhs.alias_refcount[alias]:
648 # The alias was unused in the rhs query. Unref it so that it
649 # will be unused in the new query, too. We have to add and
650 # unref the alias so that join promotion has information of
651 # the join type for the unused alias.
652 self.unref_alias(new_alias)
653 joinpromoter.add_votes(rhs_votes)
654 joinpromoter.update_join_types(self)
656 # Combine subqueries aliases to ensure aliases relabelling properly
657 # handle subqueries when combining where and select clauses.
658 self.subq_aliases |= rhs.subq_aliases
660 # Now relabel a copy of the rhs where-clause and add it to the current
661 # one.
662 w = rhs.where.clone()
663 w.relabel_aliases(change_map)
664 self.where.add(w, connector)
666 # Selection columns and extra extensions are those provided by 'rhs'.
667 if rhs.select:
668 self.set_select([col.relabeled_clone(change_map) for col in rhs.select])
669 else:
670 self.select = ()
672 if connector == OR:
673 # It would be nice to be able to handle this, but the queries don't
674 # really make sense (or return consistent value sets). Not worth
675 # the extra complexity when you can write a real query instead.
676 if self.extra and rhs.extra:
677 raise ValueError(
678 "When merging querysets using 'or', you cannot have "
679 "extra(select=...) on both sides."
680 )
681 self.extra.update(rhs.extra)
682 extra_select_mask = set()
683 if self.extra_select_mask is not None:
684 extra_select_mask.update(self.extra_select_mask)
685 if rhs.extra_select_mask is not None:
686 extra_select_mask.update(rhs.extra_select_mask)
687 if extra_select_mask:
688 self.set_extra_mask(extra_select_mask)
689 self.extra_tables += rhs.extra_tables
691 # Ordering uses the 'rhs' ordering, unless it has none, in which case
692 # the current ordering is used.
693 self.order_by = rhs.order_by or self.order_by
694 self.extra_order_by = rhs.extra_order_by or self.extra_order_by
696 def _get_defer_select_mask(self, opts, mask, select_mask=None):
697 if select_mask is None:
698 select_mask = {}
699 select_mask[opts.pk] = {}
700 # All concrete fields that are not part of the defer mask must be
701 # loaded. If a relational field is encountered it gets added to the
702 # mask for it be considered if `select_related` and the cycle continues
703 # by recursively calling this function.
704 for field in opts.concrete_fields:
705 field_mask = mask.pop(field.name, None)
706 if field_mask is None:
707 select_mask.setdefault(field, {})
708 elif field_mask:
709 if not field.is_relation:
710 raise FieldError(next(iter(field_mask)))
711 field_select_mask = select_mask.setdefault(field, {})
712 related_model = field.remote_field.model._meta.concrete_model
713 self._get_defer_select_mask(
714 related_model._meta, field_mask, field_select_mask
715 )
716 # Remaining defer entries must be references to reverse relationships.
717 # The following code is expected to raise FieldError if it encounters
718 # a malformed defer entry.
719 for field_name, field_mask in mask.items():
720 if filtered_relation := self._filtered_relations.get(field_name):
721 relation = opts.get_field(filtered_relation.relation_name)
722 field_select_mask = select_mask.setdefault((field_name, relation), {})
723 field = relation.field
724 else:
725 field = opts.get_field(field_name).field
726 field_select_mask = select_mask.setdefault(field, {})
727 related_model = field.model._meta.concrete_model
728 self._get_defer_select_mask(
729 related_model._meta, field_mask, field_select_mask
730 )
731 return select_mask
733 def _get_only_select_mask(self, opts, mask, select_mask=None):
734 if select_mask is None:
735 select_mask = {}
736 select_mask[opts.pk] = {}
737 # Only include fields mentioned in the mask.
738 for field_name, field_mask in mask.items():
739 field = opts.get_field(field_name)
740 field_select_mask = select_mask.setdefault(field, {})
741 if field_mask:
742 if not field.is_relation:
743 raise FieldError(next(iter(field_mask)))
744 related_model = field.remote_field.model._meta.concrete_model
745 self._get_only_select_mask(
746 related_model._meta, field_mask, field_select_mask
747 )
748 return select_mask
750 def get_select_mask(self):
751 """
752 Convert the self.deferred_loading data structure to an alternate data
753 structure, describing the field that *will* be loaded. This is used to
754 compute the columns to select from the database and also by the
755 QuerySet class to work out which fields are being initialized on each
756 model. Models that have all their fields included aren't mentioned in
757 the result, only those that have field restrictions in place.
758 """
759 field_names, defer = self.deferred_loading
760 if not field_names:
761 return {}
762 mask = {}
763 for field_name in field_names:
764 part_mask = mask
765 for part in field_name.split(LOOKUP_SEP):
766 part_mask = part_mask.setdefault(part, {})
767 opts = self.get_meta()
768 if defer:
769 return self._get_defer_select_mask(opts, mask)
770 return self._get_only_select_mask(opts, mask)
772 def table_alias(self, table_name, create=False, filtered_relation=None):
773 """
774 Return a table alias for the given table_name and whether this is a
775 new alias or not.
777 If 'create' is true, a new alias is always created. Otherwise, the
778 most recently created alias for the table (if one exists) is reused.
779 """
780 alias_list = self.table_map.get(table_name)
781 if not create and alias_list:
782 alias = alias_list[0]
783 self.alias_refcount[alias] += 1
784 return alias, False
786 # Create a new alias for this table.
787 if alias_list:
788 alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1)
789 alias_list.append(alias)
790 else:
791 # The first occurrence of a table uses the table name directly.
792 alias = (
793 filtered_relation.alias if filtered_relation is not None else table_name
794 )
795 self.table_map[table_name] = [alias]
796 self.alias_refcount[alias] = 1
797 return alias, True
799 def ref_alias(self, alias):
800 """Increases the reference count for this alias."""
801 self.alias_refcount[alias] += 1
803 def unref_alias(self, alias, amount=1):
804 """Decreases the reference count for this alias."""
805 self.alias_refcount[alias] -= amount
807 def promote_joins(self, aliases):
808 """
809 Promote recursively the join type of given aliases and its children to
810 an outer join. If 'unconditional' is False, only promote the join if
811 it is nullable or the parent join is an outer join.
813 The children promotion is done to avoid join chains that contain a LOUTER
814 b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted,
815 then we must also promote b->c automatically, or otherwise the promotion
816 of a->b doesn't actually change anything in the query results.
817 """
818 aliases = list(aliases)
819 while aliases:
820 alias = aliases.pop(0)
821 if self.alias_map[alias].join_type is None:
822 # This is the base table (first FROM entry) - this table
823 # isn't really joined at all in the query, so we should not
824 # alter its join type.
825 continue
826 # Only the first alias (skipped above) should have None join_type
827 assert self.alias_map[alias].join_type is not None
828 parent_alias = self.alias_map[alias].parent_alias
829 parent_louter = (
830 parent_alias and self.alias_map[parent_alias].join_type == LOUTER
831 )
832 already_louter = self.alias_map[alias].join_type == LOUTER
833 if (self.alias_map[alias].nullable or parent_louter) and not already_louter:
834 self.alias_map[alias] = self.alias_map[alias].promote()
835 # Join type of 'alias' changed, so re-examine all aliases that
836 # refer to this one.
837 aliases.extend(
838 join
839 for join in self.alias_map
840 if self.alias_map[join].parent_alias == alias
841 and join not in aliases
842 )
844 def demote_joins(self, aliases):
845 """
846 Change join type from LOUTER to INNER for all joins in aliases.
848 Similarly to promote_joins(), this method must ensure no join chains
849 containing first an outer, then an inner join are generated. If we
850 are demoting b->c join in chain a LOUTER b LOUTER c then we must
851 demote a->b automatically, or otherwise the demotion of b->c doesn't
852 actually change anything in the query results. .
853 """
854 aliases = list(aliases)
855 while aliases:
856 alias = aliases.pop(0)
857 if self.alias_map[alias].join_type == LOUTER:
858 self.alias_map[alias] = self.alias_map[alias].demote()
859 parent_alias = self.alias_map[alias].parent_alias
860 if self.alias_map[parent_alias].join_type == INNER:
861 aliases.append(parent_alias)
863 def reset_refcounts(self, to_counts):
864 """
865 Reset reference counts for aliases so that they match the value passed
866 in `to_counts`.
867 """
868 for alias, cur_refcount in self.alias_refcount.copy().items():
869 unref_amount = cur_refcount - to_counts.get(alias, 0)
870 self.unref_alias(alias, unref_amount)
872 def change_aliases(self, change_map):
873 """
874 Change the aliases in change_map (which maps old-alias -> new-alias),
875 relabelling any references to them in select columns and the where
876 clause.
877 """
878 # If keys and values of change_map were to intersect, an alias might be
879 # updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending
880 # on their order in change_map.
881 assert set(change_map).isdisjoint(change_map.values())
883 # 1. Update references in "select" (normal columns plus aliases),
884 # "group by" and "where".
885 self.where.relabel_aliases(change_map)
886 if isinstance(self.group_by, tuple):
887 self.group_by = tuple(
888 [col.relabeled_clone(change_map) for col in self.group_by]
889 )
890 self.select = tuple([col.relabeled_clone(change_map) for col in self.select])
891 self.annotations = self.annotations and {
892 key: col.relabeled_clone(change_map)
893 for key, col in self.annotations.items()
894 }
896 # 2. Rename the alias in the internal table/alias datastructures.
897 for old_alias, new_alias in change_map.items():
898 if old_alias not in self.alias_map:
899 continue
900 alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
901 self.alias_map[new_alias] = alias_data
902 self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
903 del self.alias_refcount[old_alias]
904 del self.alias_map[old_alias]
906 table_aliases = self.table_map[alias_data.table_name]
907 for pos, alias in enumerate(table_aliases):
908 if alias == old_alias:
909 table_aliases[pos] = new_alias
910 break
911 self.external_aliases = {
912 # Table is aliased or it's being changed and thus is aliased.
913 change_map.get(alias, alias): (aliased or alias in change_map)
914 for alias, aliased in self.external_aliases.items()
915 }
917 def bump_prefix(self, other_query, exclude=None):
918 """
919 Change the alias prefix to the next letter in the alphabet in a way
920 that the other query's aliases and this query's aliases will not
921 conflict. Even tables that previously had no alias will get an alias
922 after this call. To prevent changing aliases use the exclude parameter.
923 """
925 def prefix_gen():
926 """
927 Generate a sequence of characters in alphabetical order:
928 -> 'A', 'B', 'C', ...
930 When the alphabet is finished, the sequence will continue with the
931 Cartesian product:
932 -> 'AA', 'AB', 'AC', ...
933 """
934 alphabet = ascii_uppercase
935 prefix = chr(ord(self.alias_prefix) + 1)
936 yield prefix
937 for n in count(1):
938 seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet
939 for s in product(seq, repeat=n):
940 yield "".join(s)
941 prefix = None
943 if self.alias_prefix != other_query.alias_prefix:
944 # No clashes between self and outer query should be possible.
945 return
947 # Explicitly avoid infinite loop. The constant divider is based on how
948 # much depth recursive subquery references add to the stack. This value
949 # might need to be adjusted when adding or removing function calls from
950 # the code path in charge of performing these operations.
951 local_recursion_limit = sys.getrecursionlimit() // 16
952 for pos, prefix in enumerate(prefix_gen()):
953 if prefix not in self.subq_aliases:
954 self.alias_prefix = prefix
955 break
956 if pos > local_recursion_limit:
957 raise RecursionError(
958 "Maximum recursion depth exceeded: too many subqueries."
959 )
960 self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
961 other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases)
962 if exclude is None:
963 exclude = {}
964 self.change_aliases(
965 {
966 alias: "%s%d" % (self.alias_prefix, pos)
967 for pos, alias in enumerate(self.alias_map)
968 if alias not in exclude
969 }
970 )
972 def get_initial_alias(self):
973 """
974 Return the first alias for this query, after increasing its reference
975 count.
976 """
977 if self.alias_map:
978 alias = self.base_table
979 self.ref_alias(alias)
980 elif self.model:
981 alias = self.join(self.base_table_class(self.get_meta().db_table, None))
982 else:
983 alias = None
984 return alias
986 def count_active_tables(self):
987 """
988 Return the number of tables in this query with a non-zero reference
989 count. After execution, the reference counts are zeroed, so tables
990 added in compiler will not be seen by this method.
991 """
992 return len([1 for count in self.alias_refcount.values() if count])
994 def join(self, join, reuse=None, reuse_with_filtered_relation=False):
995 """
996 Return an alias for the 'join', either reusing an existing alias for
997 that join or creating a new one. 'join' is either a base_table_class or
998 join_class.
1000 The 'reuse' parameter can be either None which means all joins are
1001 reusable, or it can be a set containing the aliases that can be reused.
1003 The 'reuse_with_filtered_relation' parameter is used when computing
1004 FilteredRelation instances.
1006 A join is always created as LOUTER if the lhs alias is LOUTER to make
1007 sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new
1008 joins are created as LOUTER if the join is nullable.
1009 """
1010 if reuse_with_filtered_relation and reuse:
1011 reuse_aliases = [
1012 a for a, j in self.alias_map.items() if a in reuse and j.equals(join)
1013 ]
1014 else:
1015 reuse_aliases = [
1016 a
1017 for a, j in self.alias_map.items()
1018 if (reuse is None or a in reuse) and j == join
1019 ]
1020 if reuse_aliases:
1021 if join.table_alias in reuse_aliases:
1022 reuse_alias = join.table_alias
1023 else:
1024 # Reuse the most recent alias of the joined table
1025 # (a many-to-many relation may be joined multiple times).
1026 reuse_alias = reuse_aliases[-1]
1027 self.ref_alias(reuse_alias)
1028 return reuse_alias
1030 # No reuse is possible, so we need a new alias.
1031 alias, _ = self.table_alias(
1032 join.table_name, create=True, filtered_relation=join.filtered_relation
1033 )
1034 if join.join_type:
1035 if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
1036 join_type = LOUTER
1037 else:
1038 join_type = INNER
1039 join.join_type = join_type
1040 join.table_alias = alias
1041 self.alias_map[alias] = join
1042 return alias
1044 def join_parent_model(self, opts, model, alias, seen):
1045 """
1046 Make sure the given 'model' is joined in the query. If 'model' isn't
1047 a parent of 'opts' or if it is None this method is a no-op.
1049 The 'alias' is the root alias for starting the join, 'seen' is a dict
1050 of model -> alias of existing joins. It must also contain a mapping
1051 of None -> some alias. This will be returned in the no-op case.
1052 """
1053 if model in seen:
1054 return seen[model]
1055 chain = opts.get_base_chain(model)
1056 if not chain:
1057 return alias
1058 curr_opts = opts
1059 for int_model in chain:
1060 if int_model in seen:
1061 curr_opts = int_model._meta
1062 alias = seen[int_model]
1063 continue
1064 # Proxy model have elements in base chain
1065 # with no parents, assign the new options
1066 # object and skip to the next base in that
1067 # case
1068 if not curr_opts.parents[int_model]:
1069 curr_opts = int_model._meta
1070 continue
1071 link_field = curr_opts.get_ancestor_link(int_model)
1072 join_info = self.setup_joins([link_field.name], curr_opts, alias)
1073 curr_opts = int_model._meta
1074 alias = seen[int_model] = join_info.joins[-1]
1075 return alias or seen[None]
1077 def check_alias(self, alias):
1078 if FORBIDDEN_ALIAS_PATTERN.search(alias):
1079 raise ValueError(
1080 "Column aliases cannot contain whitespace characters, quotation marks, "
1081 "semicolons, or SQL comments."
1082 )
1084 def add_annotation(self, annotation, alias, select=True):
1085 """Add a single annotation expression to the Query."""
1086 self.check_alias(alias)
1087 annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)
1088 if select:
1089 self.append_annotation_mask([alias])
1090 else:
1091 self.set_annotation_mask(set(self.annotation_select).difference({alias}))
1092 self.annotations[alias] = annotation
1094 def resolve_expression(self, query, *args, **kwargs):
1095 clone = self.clone()
1096 # Subqueries need to use a different set of aliases than the outer query.
1097 clone.bump_prefix(query)
1098 clone.subquery = True
1099 clone.where.resolve_expression(query, *args, **kwargs)
1100 # Resolve combined queries.
1101 if clone.combinator:
1102 clone.combined_queries = tuple(
1103 [
1104 combined_query.resolve_expression(query, *args, **kwargs)
1105 for combined_query in clone.combined_queries
1106 ]
1107 )
1108 for key, value in clone.annotations.items():
1109 resolved = value.resolve_expression(query, *args, **kwargs)
1110 if hasattr(resolved, "external_aliases"):
1111 resolved.external_aliases.update(clone.external_aliases)
1112 clone.annotations[key] = resolved
1113 # Outer query's aliases are considered external.
1114 for alias, table in query.alias_map.items():
1115 clone.external_aliases[alias] = (
1116 isinstance(table, Join)
1117 and table.join_field.related_model._meta.db_table != alias
1118 ) or (
1119 isinstance(table, BaseTable) and table.table_name != table.table_alias
1120 )
1121 return clone
1123 def get_external_cols(self):
1124 exprs = chain(self.annotations.values(), self.where.children)
1125 return [
1126 col
1127 for col in self._gen_cols(exprs, include_external=True)
1128 if col.alias in self.external_aliases
1129 ]
1131 def get_group_by_cols(self, wrapper=None):
1132 # If wrapper is referenced by an alias for an explicit GROUP BY through
1133 # values() a reference to this expression and not the self must be
1134 # returned to ensure external column references are not grouped against
1135 # as well.
1136 external_cols = self.get_external_cols()
1137 if any(col.possibly_multivalued for col in external_cols):
1138 return [wrapper or self]
1139 return external_cols
1141 def as_sql(self, compiler, connection):
1142 # Some backends (e.g. Oracle) raise an error when a subquery contains
1143 # unnecessary ORDER BY clause.
1144 if (
1145 self.subquery
1146 and not connection.features.ignores_unnecessary_order_by_in_subqueries
1147 ):
1148 self.clear_ordering(force=False)
1149 for query in self.combined_queries:
1150 query.clear_ordering(force=False)
1151 sql, params = self.get_compiler(connection=connection).as_sql()
1152 if self.subquery:
1153 sql = f"({sql})"
1154 return sql, params
1156 def resolve_lookup_value(self, value, can_reuse, allow_joins):
1157 if hasattr(value, "resolve_expression"):
1158 value = value.resolve_expression(
1159 self,
1160 reuse=can_reuse,
1161 allow_joins=allow_joins,
1162 )
1163 elif isinstance(value, list | tuple):
1164 # The items of the iterable may be expressions and therefore need
1165 # to be resolved independently.
1166 values = (
1167 self.resolve_lookup_value(sub_value, can_reuse, allow_joins)
1168 for sub_value in value
1169 )
1170 type_ = type(value)
1171 if hasattr(type_, "_make"): # namedtuple
1172 return type_(*values)
1173 return type_(values)
1174 return value
1176 def solve_lookup_type(self, lookup, summarize=False):
1177 """
1178 Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains').
1179 """
1180 lookup_splitted = lookup.split(LOOKUP_SEP)
1181 if self.annotations:
1182 annotation, expression_lookups = refs_expression(
1183 lookup_splitted, self.annotations
1184 )
1185 if annotation:
1186 expression = self.annotations[annotation]
1187 if summarize:
1188 expression = Ref(annotation, expression)
1189 return expression_lookups, (), expression
1190 _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
1191 field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)]
1192 if len(lookup_parts) > 1 and not field_parts:
1193 raise FieldError(
1194 f'Invalid lookup "{lookup}" for model {self.get_meta().model.__name__}".'
1195 )
1196 return lookup_parts, field_parts, False
1198 def check_query_object_type(self, value, opts, field):
1199 """
1200 Check whether the object passed while querying is of the correct type.
1201 If not, raise a ValueError specifying the wrong object.
1202 """
1203 if hasattr(value, "_meta"):
1204 if not check_rel_lookup_compatibility(value._meta.model, opts, field):
1205 raise ValueError(
1206 f'Cannot query "{value}": Must be "{opts.object_name}" instance.'
1207 )
1209 def check_related_objects(self, field, value, opts):
1210 """Check the type of object passed to query relations."""
1211 if field.is_relation:
1212 # Check that the field and the queryset use the same model in a
1213 # query like .filter(author=Author.objects.all()). For example, the
1214 # opts would be Author's (from the author field) and value.model
1215 # would be Author.objects.all() queryset's .model (Author also).
1216 # The field is the related field on the lhs side.
1217 if (
1218 isinstance(value, Query)
1219 and not value.has_select_fields
1220 and not check_rel_lookup_compatibility(value.model, opts, field)
1221 ):
1222 raise ValueError(
1223 f'Cannot use QuerySet for "{value.model._meta.object_name}": Use a QuerySet for "{opts.object_name}".'
1224 )
1225 elif hasattr(value, "_meta"):
1226 self.check_query_object_type(value, opts, field)
1227 elif hasattr(value, "__iter__"):
1228 for v in value:
1229 self.check_query_object_type(v, opts, field)
1231 def check_filterable(self, expression):
1232 """Raise an error if expression cannot be used in a WHERE clause."""
1233 if hasattr(expression, "resolve_expression") and not getattr(
1234 expression, "filterable", True
1235 ):
1236 raise NotSupportedError(
1237 expression.__class__.__name__ + " is disallowed in the filter "
1238 "clause."
1239 )
1240 if hasattr(expression, "get_source_expressions"):
1241 for expr in expression.get_source_expressions():
1242 self.check_filterable(expr)
1244 def build_lookup(self, lookups, lhs, rhs):
1245 """
1246 Try to extract transforms and lookup from given lhs.
1248 The lhs value is something that works like SQLExpression.
1249 The rhs value is what the lookup is going to compare against.
1250 The lookups is a list of names to extract using get_lookup()
1251 and get_transform().
1252 """
1253 # __exact is the default lookup if one isn't given.
1254 *transforms, lookup_name = lookups or ["exact"]
1255 for name in transforms:
1256 lhs = self.try_transform(lhs, name)
1257 # First try get_lookup() so that the lookup takes precedence if the lhs
1258 # supports both transform and lookup for the name.
1259 lookup_class = lhs.get_lookup(lookup_name)
1260 if not lookup_class:
1261 # A lookup wasn't found. Try to interpret the name as a transform
1262 # and do an Exact lookup against it.
1263 lhs = self.try_transform(lhs, lookup_name)
1264 lookup_name = "exact"
1265 lookup_class = lhs.get_lookup(lookup_name)
1266 if not lookup_class:
1267 return
1269 lookup = lookup_class(lhs, rhs)
1270 # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
1271 # uses of None as a query value unless the lookup supports it.
1272 if lookup.rhs is None and not lookup.can_use_none_as_rhs:
1273 if lookup_name not in ("exact", "iexact"):
1274 raise ValueError("Cannot use None as a query value")
1275 return lhs.get_lookup("isnull")(lhs, True)
1277 # For Oracle '' is equivalent to null. The check must be done at this
1278 # stage because join promotion can't be done in the compiler. Using
1279 # DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here.
1280 # A similar thing is done in is_nullable(), too.
1281 if (
1282 lookup_name == "exact"
1283 and lookup.rhs == ""
1284 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
1285 ):
1286 return lhs.get_lookup("isnull")(lhs, True)
1288 return lookup
1290 def try_transform(self, lhs, name):
1291 """
1292 Helper method for build_lookup(). Try to fetch and initialize
1293 a transform for name parameter from lhs.
1294 """
1295 transform_class = lhs.get_transform(name)
1296 if transform_class:
1297 return transform_class(lhs)
1298 else:
1299 output_field = lhs.output_field.__class__
1300 suggested_lookups = difflib.get_close_matches(
1301 name, output_field.get_lookups()
1302 )
1303 if suggested_lookups:
1304 suggestion = ", perhaps you meant {}?".format(
1305 " or ".join(suggested_lookups)
1306 )
1307 else:
1308 suggestion = "."
1309 raise FieldError(
1310 f"Unsupported lookup '{name}' for {output_field.__name__} or join on the field not "
1311 f"permitted{suggestion}"
1312 )
1314 def build_filter(
1315 self,
1316 filter_expr,
1317 branch_negated=False,
1318 current_negated=False,
1319 can_reuse=None,
1320 allow_joins=True,
1321 split_subq=True,
1322 reuse_with_filtered_relation=False,
1323 check_filterable=True,
1324 summarize=False,
1325 ):
1326 """
1327 Build a WhereNode for a single filter clause but don't add it
1328 to this Query. Query.add_q() will then add this filter to the where
1329 Node.
1331 The 'branch_negated' tells us if the current branch contains any
1332 negations. This will be used to determine if subqueries are needed.
1334 The 'current_negated' is used to determine if the current filter is
1335 negated or not and this will be used to determine if IS NULL filtering
1336 is needed.
1338 The difference between current_negated and branch_negated is that
1339 branch_negated is set on first negation, but current_negated is
1340 flipped for each negation.
1342 Note that add_filter will not do any negating itself, that is done
1343 upper in the code by add_q().
1345 The 'can_reuse' is a set of reusable joins for multijoins.
1347 If 'reuse_with_filtered_relation' is True, then only joins in can_reuse
1348 will be reused.
1350 The method will create a filter clause that can be added to the current
1351 query. However, if the filter isn't added to the query then the caller
1352 is responsible for unreffing the joins used.
1353 """
1354 if isinstance(filter_expr, dict):
1355 raise FieldError("Cannot parse keyword query as dict")
1356 if isinstance(filter_expr, Q):
1357 return self._add_q(
1358 filter_expr,
1359 branch_negated=branch_negated,
1360 current_negated=current_negated,
1361 used_aliases=can_reuse,
1362 allow_joins=allow_joins,
1363 split_subq=split_subq,
1364 check_filterable=check_filterable,
1365 summarize=summarize,
1366 )
1367 if hasattr(filter_expr, "resolve_expression"):
1368 if not getattr(filter_expr, "conditional", False):
1369 raise TypeError("Cannot filter against a non-conditional expression.")
1370 condition = filter_expr.resolve_expression(
1371 self, allow_joins=allow_joins, summarize=summarize
1372 )
1373 if not isinstance(condition, Lookup):
1374 condition = self.build_lookup(["exact"], condition, True)
1375 return WhereNode([condition], connector=AND), []
1376 arg, value = filter_expr
1377 if not arg:
1378 raise FieldError(f"Cannot parse keyword query {arg!r}")
1379 lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize)
1381 if check_filterable:
1382 self.check_filterable(reffed_expression)
1384 if not allow_joins and len(parts) > 1:
1385 raise FieldError("Joined field references are not permitted in this query")
1387 pre_joins = self.alias_refcount.copy()
1388 value = self.resolve_lookup_value(value, can_reuse, allow_joins)
1389 used_joins = {
1390 k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)
1391 }
1393 if check_filterable:
1394 self.check_filterable(value)
1396 if reffed_expression:
1397 condition = self.build_lookup(lookups, reffed_expression, value)
1398 return WhereNode([condition], connector=AND), []
1400 opts = self.get_meta()
1401 alias = self.get_initial_alias()
1402 allow_many = not branch_negated or not split_subq
1404 try:
1405 join_info = self.setup_joins(
1406 parts,
1407 opts,
1408 alias,
1409 can_reuse=can_reuse,
1410 allow_many=allow_many,
1411 reuse_with_filtered_relation=reuse_with_filtered_relation,
1412 )
1414 # Prevent iterator from being consumed by check_related_objects()
1415 if isinstance(value, Iterator):
1416 value = list(value)
1417 self.check_related_objects(join_info.final_field, value, join_info.opts)
1419 # split_exclude() needs to know which joins were generated for the
1420 # lookup parts
1421 self._lookup_joins = join_info.joins
1422 except MultiJoin as e:
1423 return self.split_exclude(filter_expr, can_reuse, e.names_with_path)
1425 # Update used_joins before trimming since they are reused to determine
1426 # which joins could be later promoted to INNER.
1427 used_joins.update(join_info.joins)
1428 targets, alias, join_list = self.trim_joins(
1429 join_info.targets, join_info.joins, join_info.path
1430 )
1431 if can_reuse is not None:
1432 can_reuse.update(join_list)
1434 if join_info.final_field.is_relation:
1435 if len(targets) == 1:
1436 col = self._get_col(targets[0], join_info.final_field, alias)
1437 else:
1438 col = MultiColSource(
1439 alias, targets, join_info.targets, join_info.final_field
1440 )
1441 else:
1442 col = self._get_col(targets[0], join_info.final_field, alias)
1444 condition = self.build_lookup(lookups, col, value)
1445 lookup_type = condition.lookup_name
1446 clause = WhereNode([condition], connector=AND)
1448 require_outer = (
1449 lookup_type == "isnull" and condition.rhs is True and not current_negated
1450 )
1451 if (
1452 current_negated
1453 and (lookup_type != "isnull" or condition.rhs is False)
1454 and condition.rhs is not None
1455 ):
1456 require_outer = True
1457 if lookup_type != "isnull":
1458 # The condition added here will be SQL like this:
1459 # NOT (col IS NOT NULL), where the first NOT is added in
1460 # upper layers of code. The reason for addition is that if col
1461 # is null, then col != someval will result in SQL "unknown"
1462 # which isn't the same as in Python. The Python None handling
1463 # is wanted, and it can be gotten by
1464 # (col IS NULL OR col != someval)
1465 # <=>
1466 # NOT (col IS NOT NULL AND col = someval).
1467 if (
1468 self.is_nullable(targets[0])
1469 or self.alias_map[join_list[-1]].join_type == LOUTER
1470 ):
1471 lookup_class = targets[0].get_lookup("isnull")
1472 col = self._get_col(targets[0], join_info.targets[0], alias)
1473 clause.add(lookup_class(col, False), AND)
1474 # If someval is a nullable column, someval IS NOT NULL is
1475 # added.
1476 if isinstance(value, Col) and self.is_nullable(value.target):
1477 lookup_class = value.target.get_lookup("isnull")
1478 clause.add(lookup_class(value, False), AND)
1479 return clause, used_joins if not require_outer else ()
1481 def add_filter(self, filter_lhs, filter_rhs):
1482 self.add_q(Q((filter_lhs, filter_rhs)))
1484 def add_q(self, q_object):
1485 """
1486 A preprocessor for the internal _add_q(). Responsible for doing final
1487 join promotion.
1488 """
1489 # For join promotion this case is doing an AND for the added q_object
1490 # and existing conditions. So, any existing inner join forces the join
1491 # type to remain inner. Existing outer joins can however be demoted.
1492 # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if
1493 # rel_a doesn't produce any rows, then the whole condition must fail.
1494 # So, demotion is OK.
1495 existing_inner = {
1496 a for a in self.alias_map if self.alias_map[a].join_type == INNER
1497 }
1498 clause, _ = self._add_q(q_object, self.used_aliases)
1499 if clause:
1500 self.where.add(clause, AND)
1501 self.demote_joins(existing_inner)
1503 def build_where(self, filter_expr):
1504 return self.build_filter(filter_expr, allow_joins=False)[0]
1506 def clear_where(self):
1507 self.where = WhereNode()
1509 def _add_q(
1510 self,
1511 q_object,
1512 used_aliases,
1513 branch_negated=False,
1514 current_negated=False,
1515 allow_joins=True,
1516 split_subq=True,
1517 check_filterable=True,
1518 summarize=False,
1519 ):
1520 """Add a Q-object to the current filter."""
1521 connector = q_object.connector
1522 current_negated ^= q_object.negated
1523 branch_negated = branch_negated or q_object.negated
1524 target_clause = WhereNode(connector=connector, negated=q_object.negated)
1525 joinpromoter = JoinPromoter(
1526 q_object.connector, len(q_object.children), current_negated
1527 )
1528 for child in q_object.children:
1529 child_clause, needed_inner = self.build_filter(
1530 child,
1531 can_reuse=used_aliases,
1532 branch_negated=branch_negated,
1533 current_negated=current_negated,
1534 allow_joins=allow_joins,
1535 split_subq=split_subq,
1536 check_filterable=check_filterable,
1537 summarize=summarize,
1538 )
1539 joinpromoter.add_votes(needed_inner)
1540 if child_clause:
1541 target_clause.add(child_clause, connector)
1542 needed_inner = joinpromoter.update_join_types(self)
1543 return target_clause, needed_inner
1545 def build_filtered_relation_q(
1546 self, q_object, reuse, branch_negated=False, current_negated=False
1547 ):
1548 """Add a FilteredRelation object to the current filter."""
1549 connector = q_object.connector
1550 current_negated ^= q_object.negated
1551 branch_negated = branch_negated or q_object.negated
1552 target_clause = WhereNode(connector=connector, negated=q_object.negated)
1553 for child in q_object.children:
1554 if isinstance(child, Node):
1555 child_clause = self.build_filtered_relation_q(
1556 child,
1557 reuse=reuse,
1558 branch_negated=branch_negated,
1559 current_negated=current_negated,
1560 )
1561 else:
1562 child_clause, _ = self.build_filter(
1563 child,
1564 can_reuse=reuse,
1565 branch_negated=branch_negated,
1566 current_negated=current_negated,
1567 allow_joins=True,
1568 split_subq=False,
1569 reuse_with_filtered_relation=True,
1570 )
1571 target_clause.add(child_clause, connector)
1572 return target_clause
1574 def add_filtered_relation(self, filtered_relation, alias):
1575 filtered_relation.alias = alias
1576 lookups = dict(get_children_from_q(filtered_relation.condition))
1577 relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type(
1578 filtered_relation.relation_name
1579 )
1580 if relation_lookup_parts:
1581 raise ValueError(
1582 "FilteredRelation's relation_name cannot contain lookups "
1583 f"(got {filtered_relation.relation_name!r})."
1584 )
1585 for lookup in chain(lookups):
1586 lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup)
1587 shift = 2 if not lookup_parts else 1
1588 lookup_field_path = lookup_field_parts[:-shift]
1589 for idx, lookup_field_part in enumerate(lookup_field_path):
1590 if len(relation_field_parts) > idx:
1591 if relation_field_parts[idx] != lookup_field_part:
1592 raise ValueError(
1593 "FilteredRelation's condition doesn't support "
1594 f"relations outside the {filtered_relation.relation_name!r} (got {lookup!r})."
1595 )
1596 else:
1597 raise ValueError(
1598 "FilteredRelation's condition doesn't support nested "
1599 f"relations deeper than the relation_name (got {lookup!r} for "
1600 f"{filtered_relation.relation_name!r})."
1601 )
1602 self._filtered_relations[filtered_relation.alias] = filtered_relation
1604 def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
1605 """
1606 Walk the list of names and turns them into PathInfo tuples. A single
1607 name in 'names' can generate multiple PathInfos (m2m, for example).
1609 'names' is the path of names to travel, 'opts' is the model Options we
1610 start the name resolving from, 'allow_many' is as for setup_joins().
1611 If fail_on_missing is set to True, then a name that can't be resolved
1612 will generate a FieldError.
1614 Return a list of PathInfo tuples. In addition return the final field
1615 (the last used join field) and target (which is a field guaranteed to
1616 contain the same value as the final field). Finally, return those names
1617 that weren't found (which are likely transforms and the final lookup).
1618 """
1619 path, names_with_path = [], []
1620 for pos, name in enumerate(names):
1621 cur_names_with_path = (name, [])
1622 if name == "pk":
1623 name = opts.pk.name
1625 field = None
1626 filtered_relation = None
1627 try:
1628 if opts is None:
1629 raise FieldDoesNotExist
1630 field = opts.get_field(name)
1631 except FieldDoesNotExist:
1632 if name in self.annotation_select:
1633 field = self.annotation_select[name].output_field
1634 elif name in self._filtered_relations and pos == 0:
1635 filtered_relation = self._filtered_relations[name]
1636 if LOOKUP_SEP in filtered_relation.relation_name:
1637 parts = filtered_relation.relation_name.split(LOOKUP_SEP)
1638 filtered_relation_path, field, _, _ = self.names_to_path(
1639 parts,
1640 opts,
1641 allow_many,
1642 fail_on_missing,
1643 )
1644 path.extend(filtered_relation_path[:-1])
1645 else:
1646 field = opts.get_field(filtered_relation.relation_name)
1647 if field is not None:
1648 # Fields that contain one-to-many relations with a generic
1649 # model (like a GenericForeignKey) cannot generate reverse
1650 # relations and therefore cannot be used for reverse querying.
1651 if field.is_relation and not field.related_model:
1652 raise FieldError(
1653 f"Field {name!r} does not generate an automatic reverse "
1654 "relation and therefore cannot be used for reverse "
1655 "querying. If it is a GenericForeignKey, consider "
1656 "adding a GenericRelation."
1657 )
1658 try:
1659 model = field.model._meta.concrete_model
1660 except AttributeError:
1661 # QuerySet.annotate() may introduce fields that aren't
1662 # attached to a model.
1663 model = None
1664 else:
1665 # We didn't find the current field, so move position back
1666 # one step.
1667 pos -= 1
1668 if pos == -1 or fail_on_missing:
1669 available = sorted(
1670 [
1671 *get_field_names_from_opts(opts),
1672 *self.annotation_select,
1673 *self._filtered_relations,
1674 ]
1675 )
1676 raise FieldError(
1677 "Cannot resolve keyword '{}' into field. "
1678 "Choices are: {}".format(name, ", ".join(available))
1679 )
1680 break
1681 # Check if we need any joins for concrete inheritance cases (the
1682 # field lives in parent, but we are currently in one of its
1683 # children)
1684 if opts is not None and model is not opts.model:
1685 path_to_parent = opts.get_path_to_parent(model)
1686 if path_to_parent:
1687 path.extend(path_to_parent)
1688 cur_names_with_path[1].extend(path_to_parent)
1689 opts = path_to_parent[-1].to_opts
1690 if hasattr(field, "path_infos"):
1691 if filtered_relation:
1692 pathinfos = field.get_path_info(filtered_relation)
1693 else:
1694 pathinfos = field.path_infos
1695 if not allow_many:
1696 for inner_pos, p in enumerate(pathinfos):
1697 if p.m2m:
1698 cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1])
1699 names_with_path.append(cur_names_with_path)
1700 raise MultiJoin(pos + 1, names_with_path)
1701 last = pathinfos[-1]
1702 path.extend(pathinfos)
1703 final_field = last.join_field
1704 opts = last.to_opts
1705 targets = last.target_fields
1706 cur_names_with_path[1].extend(pathinfos)
1707 names_with_path.append(cur_names_with_path)
1708 else:
1709 # Local non-relational field.
1710 final_field = field
1711 targets = (field,)
1712 if fail_on_missing and pos + 1 != len(names):
1713 raise FieldError(
1714 f"Cannot resolve keyword {names[pos + 1]!r} into field. Join on '{name}'"
1715 " not permitted."
1716 )
1717 break
1718 return path, final_field, targets, names[pos + 1 :]
1720 def setup_joins(
1721 self,
1722 names,
1723 opts,
1724 alias,
1725 can_reuse=None,
1726 allow_many=True,
1727 reuse_with_filtered_relation=False,
1728 ):
1729 """
1730 Compute the necessary table joins for the passage through the fields
1731 given in 'names'. 'opts' is the Options class for the current model
1732 (which gives the table we are starting from), 'alias' is the alias for
1733 the table to start the joining from.
1735 The 'can_reuse' defines the reverse foreign key joins we can reuse. It
1736 can be None in which case all joins are reusable or a set of aliases
1737 that can be reused. Note that non-reverse foreign keys are always
1738 reusable when using setup_joins().
1740 The 'reuse_with_filtered_relation' can be used to force 'can_reuse'
1741 parameter and force the relation on the given connections.
1743 If 'allow_many' is False, then any reverse foreign key seen will
1744 generate a MultiJoin exception.
1746 Return the final field involved in the joins, the target field (used
1747 for any 'where' constraint), the final 'opts' value, the joins, the
1748 field path traveled to generate the joins, and a transform function
1749 that takes a field and alias and is equivalent to `field.get_col(alias)`
1750 in the simple case but wraps field transforms if they were included in
1751 names.
1753 The target field is the field containing the concrete value. Final
1754 field can be something different, for example foreign key pointing to
1755 that value. Final field is needed for example in some value
1756 conversions (convert 'obj' in fk__id=obj to pk val using the foreign
1757 key field for example).
1758 """
1759 joins = [alias]
1760 # The transform can't be applied yet, as joins must be trimmed later.
1761 # To avoid making every caller of this method look up transforms
1762 # directly, compute transforms here and create a partial that converts
1763 # fields to the appropriate wrapped version.
1765 def final_transformer(field, alias):
1766 if not self.alias_cols:
1767 alias = None
1768 return field.get_col(alias)
1770 # Try resolving all the names as fields first. If there's an error,
1771 # treat trailing names as lookups until a field can be resolved.
1772 last_field_exception = None
1773 for pivot in range(len(names), 0, -1):
1774 try:
1775 path, final_field, targets, rest = self.names_to_path(
1776 names[:pivot],
1777 opts,
1778 allow_many,
1779 fail_on_missing=True,
1780 )
1781 except FieldError as exc:
1782 if pivot == 1:
1783 # The first item cannot be a lookup, so it's safe
1784 # to raise the field error here.
1785 raise
1786 else:
1787 last_field_exception = exc
1788 else:
1789 # The transforms are the remaining items that couldn't be
1790 # resolved into fields.
1791 transforms = names[pivot:]
1792 break
1793 for name in transforms:
1795 def transform(field, alias, *, name, previous):
1796 try:
1797 wrapped = previous(field, alias)
1798 return self.try_transform(wrapped, name)
1799 except FieldError:
1800 # FieldError is raised if the transform doesn't exist.
1801 if isinstance(final_field, Field) and last_field_exception:
1802 raise last_field_exception
1803 else:
1804 raise
1806 final_transformer = functools.partial(
1807 transform, name=name, previous=final_transformer
1808 )
1809 final_transformer.has_transforms = True
1810 # Then, add the path to the query's joins. Note that we can't trim
1811 # joins at this stage - we will need the information about join type
1812 # of the trimmed joins.
1813 for join in path:
1814 if join.filtered_relation:
1815 filtered_relation = join.filtered_relation.clone()
1816 table_alias = filtered_relation.alias
1817 else:
1818 filtered_relation = None
1819 table_alias = None
1820 opts = join.to_opts
1821 if join.direct:
1822 nullable = self.is_nullable(join.join_field)
1823 else:
1824 nullable = True
1825 connection = self.join_class(
1826 opts.db_table,
1827 alias,
1828 table_alias,
1829 INNER,
1830 join.join_field,
1831 nullable,
1832 filtered_relation=filtered_relation,
1833 )
1834 reuse = can_reuse if join.m2m or reuse_with_filtered_relation else None
1835 alias = self.join(
1836 connection,
1837 reuse=reuse,
1838 reuse_with_filtered_relation=reuse_with_filtered_relation,
1839 )
1840 joins.append(alias)
1841 if filtered_relation:
1842 filtered_relation.path = joins[:]
1843 return JoinInfo(final_field, targets, opts, joins, path, final_transformer)
1845 def trim_joins(self, targets, joins, path):
1846 """
1847 The 'target' parameter is the final field being joined to, 'joins'
1848 is the full list of join aliases. The 'path' contain the PathInfos
1849 used to create the joins.
1851 Return the final target field and table alias and the new active
1852 joins.
1854 Always trim any direct join if the target column is already in the
1855 previous table. Can't trim reverse joins as it's unknown if there's
1856 anything on the other side of the join.
1857 """
1858 joins = joins[:]
1859 for pos, info in enumerate(reversed(path)):
1860 if len(joins) == 1 or not info.direct:
1861 break
1862 if info.filtered_relation:
1863 break
1864 join_targets = {t.column for t in info.join_field.foreign_related_fields}
1865 cur_targets = {t.column for t in targets}
1866 if not cur_targets.issubset(join_targets):
1867 break
1868 targets_dict = {
1869 r[1].column: r[0]
1870 for r in info.join_field.related_fields
1871 if r[1].column in cur_targets
1872 }
1873 targets = tuple(targets_dict[t.column] for t in targets)
1874 self.unref_alias(joins.pop())
1875 return targets, joins[-1], joins
1877 @classmethod
1878 def _gen_cols(cls, exprs, include_external=False, resolve_refs=True):
1879 for expr in exprs:
1880 if isinstance(expr, Col):
1881 yield expr
1882 elif include_external and callable(
1883 getattr(expr, "get_external_cols", None)
1884 ):
1885 yield from expr.get_external_cols()
1886 elif hasattr(expr, "get_source_expressions"):
1887 if not resolve_refs and isinstance(expr, Ref):
1888 continue
1889 yield from cls._gen_cols(
1890 expr.get_source_expressions(),
1891 include_external=include_external,
1892 resolve_refs=resolve_refs,
1893 )
1895 @classmethod
1896 def _gen_col_aliases(cls, exprs):
1897 yield from (expr.alias for expr in cls._gen_cols(exprs))
1899 def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
1900 annotation = self.annotations.get(name)
1901 if annotation is not None:
1902 if not allow_joins:
1903 for alias in self._gen_col_aliases([annotation]):
1904 if isinstance(self.alias_map[alias], Join):
1905 raise FieldError(
1906 "Joined field references are not permitted in this query"
1907 )
1908 if summarize:
1909 # Summarize currently means we are doing an aggregate() query
1910 # which is executed as a wrapped subquery if any of the
1911 # aggregate() elements reference an existing annotation. In
1912 # that case we need to return a Ref to the subquery's annotation.
1913 if name not in self.annotation_select:
1914 raise FieldError(
1915 f"Cannot aggregate over the '{name}' alias. Use annotate() "
1916 "to promote it."
1917 )
1918 return Ref(name, self.annotation_select[name])
1919 else:
1920 return annotation
1921 else:
1922 field_list = name.split(LOOKUP_SEP)
1923 annotation = self.annotations.get(field_list[0])
1924 if annotation is not None:
1925 for transform in field_list[1:]:
1926 annotation = self.try_transform(annotation, transform)
1927 return annotation
1928 join_info = self.setup_joins(
1929 field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse
1930 )
1931 targets, final_alias, join_list = self.trim_joins(
1932 join_info.targets, join_info.joins, join_info.path
1933 )
1934 if not allow_joins and len(join_list) > 1:
1935 raise FieldError(
1936 "Joined field references are not permitted in this query"
1937 )
1938 if len(targets) > 1:
1939 raise FieldError(
1940 "Referencing multicolumn fields with F() objects isn't supported"
1941 )
1942 # Verify that the last lookup in name is a field or a transform:
1943 # transform_function() raises FieldError if not.
1944 transform = join_info.transform_function(targets[0], final_alias)
1945 if reuse is not None:
1946 reuse.update(join_list)
1947 return transform
1949 def split_exclude(self, filter_expr, can_reuse, names_with_path):
1950 """
1951 When doing an exclude against any kind of N-to-many relation, we need
1952 to use a subquery. This method constructs the nested query, given the
1953 original exclude filter (filter_expr) and the portion up to the first
1954 N-to-many relation field.
1956 For example, if the origin filter is ~Q(child__name='foo'), filter_expr
1957 is ('child__name', 'foo') and can_reuse is a set of joins usable for
1958 filters in the original query.
1960 We will turn this into equivalent of:
1961 WHERE NOT EXISTS(
1962 SELECT 1
1963 FROM child
1964 WHERE name = 'foo' AND child.parent_id = parent.id
1965 LIMIT 1
1966 )
1967 """
1968 # Generate the inner query.
1969 query = self.__class__(self.model)
1970 query._filtered_relations = self._filtered_relations
1971 filter_lhs, filter_rhs = filter_expr
1972 if isinstance(filter_rhs, OuterRef):
1973 filter_rhs = OuterRef(filter_rhs)
1974 elif isinstance(filter_rhs, F):
1975 filter_rhs = OuterRef(filter_rhs.name)
1976 query.add_filter(filter_lhs, filter_rhs)
1977 query.clear_ordering(force=True)
1978 # Try to have as simple as possible subquery -> trim leading joins from
1979 # the subquery.
1980 trimmed_prefix, contains_louter = query.trim_start(names_with_path)
1982 col = query.select[0]
1983 select_field = col.target
1984 alias = col.alias
1985 if alias in can_reuse:
1986 pk = select_field.model._meta.pk
1987 # Need to add a restriction so that outer query's filters are in effect for
1988 # the subquery, too.
1989 query.bump_prefix(self)
1990 lookup_class = select_field.get_lookup("exact")
1991 # Note that the query.select[0].alias is different from alias
1992 # due to bump_prefix above.
1993 lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias))
1994 query.where.add(lookup, AND)
1995 query.external_aliases[alias] = True
1997 lookup_class = select_field.get_lookup("exact")
1998 lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix))
1999 query.where.add(lookup, AND)
2000 condition, needed_inner = self.build_filter(Exists(query))
2002 if contains_louter:
2003 or_null_condition, _ = self.build_filter(
2004 (f"{trimmed_prefix}__isnull", True),
2005 current_negated=True,
2006 branch_negated=True,
2007 can_reuse=can_reuse,
2008 )
2009 condition.add(or_null_condition, OR)
2010 # Note that the end result will be:
2011 # (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL.
2012 # This might look crazy but due to how IN works, this seems to be
2013 # correct. If the IS NOT NULL check is removed then outercol NOT
2014 # IN will return UNKNOWN. If the IS NULL check is removed, then if
2015 # outercol IS NULL we will not match the row.
2016 return condition, needed_inner
2018 def set_empty(self):
2019 self.where.add(NothingNode(), AND)
2020 for query in self.combined_queries:
2021 query.set_empty()
2023 def is_empty(self):
2024 return any(isinstance(c, NothingNode) for c in self.where.children)
2026 def set_limits(self, low=None, high=None):
2027 """
2028 Adjust the limits on the rows retrieved. Use low/high to set these,
2029 as it makes it more Pythonic to read and write. When the SQL query is
2030 created, convert them to the appropriate offset and limit values.
2032 Apply any limits passed in here to the existing constraints. Add low
2033 to the current low value and clamp both to any existing high value.
2034 """
2035 if high is not None:
2036 if self.high_mark is not None:
2037 self.high_mark = min(self.high_mark, self.low_mark + high)
2038 else:
2039 self.high_mark = self.low_mark + high
2040 if low is not None:
2041 if self.high_mark is not None:
2042 self.low_mark = min(self.high_mark, self.low_mark + low)
2043 else:
2044 self.low_mark = self.low_mark + low
2046 if self.low_mark == self.high_mark:
2047 self.set_empty()
2049 def clear_limits(self):
2050 """Clear any existing limits."""
2051 self.low_mark, self.high_mark = 0, None
2053 @property
2054 def is_sliced(self):
2055 return self.low_mark != 0 or self.high_mark is not None
2057 def has_limit_one(self):
2058 return self.high_mark is not None and (self.high_mark - self.low_mark) == 1
2060 def can_filter(self):
2061 """
2062 Return True if adding filters to this instance is still possible.
2064 Typically, this means no limits or offsets have been put on the results.
2065 """
2066 return not self.is_sliced
2068 def clear_select_clause(self):
2069 """Remove all fields from SELECT clause."""
2070 self.select = ()
2071 self.default_cols = False
2072 self.select_related = False
2073 self.set_extra_mask(())
2074 self.set_annotation_mask(())
2076 def clear_select_fields(self):
2077 """
2078 Clear the list of fields to select (but not extra_select columns).
2079 Some queryset types completely replace any existing list of select
2080 columns.
2081 """
2082 self.select = ()
2083 self.values_select = ()
2085 def add_select_col(self, col, name):
2086 self.select += (col,)
2087 self.values_select += (name,)
2089 def set_select(self, cols):
2090 self.default_cols = False
2091 self.select = tuple(cols)
2093 def add_distinct_fields(self, *field_names):
2094 """
2095 Add and resolve the given fields to the query's "distinct on" clause.
2096 """
2097 self.distinct_fields = field_names
2098 self.distinct = True
2100 def add_fields(self, field_names, allow_m2m=True):
2101 """
2102 Add the given (model) fields to the select set. Add the field names in
2103 the order specified.
2104 """
2105 alias = self.get_initial_alias()
2106 opts = self.get_meta()
2108 try:
2109 cols = []
2110 for name in field_names:
2111 # Join promotion note - we must not remove any rows here, so
2112 # if there is no existing joins, use outer join.
2113 join_info = self.setup_joins(
2114 name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m
2115 )
2116 targets, final_alias, joins = self.trim_joins(
2117 join_info.targets,
2118 join_info.joins,
2119 join_info.path,
2120 )
2121 for target in targets:
2122 cols.append(join_info.transform_function(target, final_alias))
2123 if cols:
2124 self.set_select(cols)
2125 except MultiJoin:
2126 raise FieldError(f"Invalid field name: '{name}'")
2127 except FieldError:
2128 if LOOKUP_SEP in name:
2129 # For lookups spanning over relationships, show the error
2130 # from the model on which the lookup failed.
2131 raise
2132 elif name in self.annotations:
2133 raise FieldError(
2134 f"Cannot select the '{name}' alias. Use annotate() to promote "
2135 "it."
2136 )
2137 else:
2138 names = sorted(
2139 [
2140 *get_field_names_from_opts(opts),
2141 *self.extra,
2142 *self.annotation_select,
2143 *self._filtered_relations,
2144 ]
2145 )
2146 raise FieldError(
2147 "Cannot resolve keyword {!r} into field. Choices are: {}".format(
2148 name, ", ".join(names)
2149 )
2150 )
2152 def add_ordering(self, *ordering):
2153 """
2154 Add items from the 'ordering' sequence to the query's "order by"
2155 clause. These items are either field names (not column names) --
2156 possibly with a direction prefix ('-' or '?') -- or OrderBy
2157 expressions.
2159 If 'ordering' is empty, clear all ordering from the query.
2160 """
2161 errors = []
2162 for item in ordering:
2163 if isinstance(item, str):
2164 if item == "?":
2165 continue
2166 item = item.removeprefix("-")
2167 if item in self.annotations:
2168 continue
2169 if self.extra and item in self.extra:
2170 continue
2171 # names_to_path() validates the lookup. A descriptive
2172 # FieldError will be raise if it's not.
2173 self.names_to_path(item.split(LOOKUP_SEP), self.model._meta)
2174 elif not hasattr(item, "resolve_expression"):
2175 errors.append(item)
2176 if getattr(item, "contains_aggregate", False):
2177 raise FieldError(
2178 "Using an aggregate in order_by() without also including "
2179 f"it in annotate() is not allowed: {item}"
2180 )
2181 if errors:
2182 raise FieldError(f"Invalid order_by arguments: {errors}")
2183 if ordering:
2184 self.order_by += ordering
2185 else:
2186 self.default_ordering = False
2188 def clear_ordering(self, force=False, clear_default=True):
2189 """
2190 Remove any ordering settings if the current query allows it without
2191 side effects, set 'force' to True to clear the ordering regardless.
2192 If 'clear_default' is True, there will be no ordering in the resulting
2193 query (not even the model's default).
2194 """
2195 if not force and (
2196 self.is_sliced or self.distinct_fields or self.select_for_update
2197 ):
2198 return
2199 self.order_by = ()
2200 self.extra_order_by = ()
2201 if clear_default:
2202 self.default_ordering = False
2204 def set_group_by(self, allow_aliases=True):
2205 """
2206 Expand the GROUP BY clause required by the query.
2208 This will usually be the set of all non-aggregate fields in the
2209 return data. If the database backend supports grouping by the
2210 primary key, and the query would be equivalent, the optimization
2211 will be made automatically.
2212 """
2213 if allow_aliases and self.values_select:
2214 # If grouping by aliases is allowed assign selected value aliases
2215 # by moving them to annotations.
2216 group_by_annotations = {}
2217 values_select = {}
2218 for alias, expr in zip(self.values_select, self.select):
2219 if isinstance(expr, Col):
2220 values_select[alias] = expr
2221 else:
2222 group_by_annotations[alias] = expr
2223 self.annotations = {**group_by_annotations, **self.annotations}
2224 self.append_annotation_mask(group_by_annotations)
2225 self.select = tuple(values_select.values())
2226 self.values_select = tuple(values_select)
2227 group_by = list(self.select)
2228 for alias, annotation in self.annotation_select.items():
2229 if not (group_by_cols := annotation.get_group_by_cols()):
2230 continue
2231 if allow_aliases and not annotation.contains_aggregate:
2232 group_by.append(Ref(alias, annotation))
2233 else:
2234 group_by.extend(group_by_cols)
2235 self.group_by = tuple(group_by)
2237 def add_select_related(self, fields):
2238 """
2239 Set up the select_related data structure so that we only select
2240 certain related models (as opposed to all models, when
2241 self.select_related=True).
2242 """
2243 if isinstance(self.select_related, bool):
2244 field_dict = {}
2245 else:
2246 field_dict = self.select_related
2247 for field in fields:
2248 d = field_dict
2249 for part in field.split(LOOKUP_SEP):
2250 d = d.setdefault(part, {})
2251 self.select_related = field_dict
2253 def add_extra(self, select, select_params, where, params, tables, order_by):
2254 """
2255 Add data to the various extra_* attributes for user-created additions
2256 to the query.
2257 """
2258 if select:
2259 # We need to pair any placeholder markers in the 'select'
2260 # dictionary with their parameters in 'select_params' so that
2261 # subsequent updates to the select dictionary also adjust the
2262 # parameters appropriately.
2263 select_pairs = {}
2264 if select_params:
2265 param_iter = iter(select_params)
2266 else:
2267 param_iter = iter([])
2268 for name, entry in select.items():
2269 self.check_alias(name)
2270 entry = str(entry)
2271 entry_params = []
2272 pos = entry.find("%s")
2273 while pos != -1:
2274 if pos == 0 or entry[pos - 1] != "%":
2275 entry_params.append(next(param_iter))
2276 pos = entry.find("%s", pos + 2)
2277 select_pairs[name] = (entry, entry_params)
2278 self.extra.update(select_pairs)
2279 if where or params:
2280 self.where.add(ExtraWhere(where, params), AND)
2281 if tables:
2282 self.extra_tables += tuple(tables)
2283 if order_by:
2284 self.extra_order_by = order_by
2286 def clear_deferred_loading(self):
2287 """Remove any fields from the deferred loading set."""
2288 self.deferred_loading = (frozenset(), True)
2290 def add_deferred_loading(self, field_names):
2291 """
2292 Add the given list of model field names to the set of fields to
2293 exclude from loading from the database when automatic column selection
2294 is done. Add the new field names to any existing field names that
2295 are deferred (or removed from any existing field names that are marked
2296 as the only ones for immediate loading).
2297 """
2298 # Fields on related models are stored in the literal double-underscore
2299 # format, so that we can use a set datastructure. We do the foo__bar
2300 # splitting and handling when computing the SQL column names (as part of
2301 # get_columns()).
2302 existing, defer = self.deferred_loading
2303 if defer:
2304 # Add to existing deferred names.
2305 self.deferred_loading = existing.union(field_names), True
2306 else:
2307 # Remove names from the set of any existing "immediate load" names.
2308 if new_existing := existing.difference(field_names):
2309 self.deferred_loading = new_existing, False
2310 else:
2311 self.clear_deferred_loading()
2312 if new_only := set(field_names).difference(existing):
2313 self.deferred_loading = new_only, True
2315 def add_immediate_loading(self, field_names):
2316 """
2317 Add the given list of model field names to the set of fields to
2318 retrieve when the SQL is executed ("immediate loading" fields). The
2319 field names replace any existing immediate loading field names. If
2320 there are field names already specified for deferred loading, remove
2321 those names from the new field_names before storing the new names
2322 for immediate loading. (That is, immediate loading overrides any
2323 existing immediate values, but respects existing deferrals.)
2324 """
2325 existing, defer = self.deferred_loading
2326 field_names = set(field_names)
2327 if "pk" in field_names:
2328 field_names.remove("pk")
2329 field_names.add(self.get_meta().pk.name)
2331 if defer:
2332 # Remove any existing deferred names from the current set before
2333 # setting the new names.
2334 self.deferred_loading = field_names.difference(existing), False
2335 else:
2336 # Replace any existing "immediate load" field names.
2337 self.deferred_loading = frozenset(field_names), False
2339 def set_annotation_mask(self, names):
2340 """Set the mask of annotations that will be returned by the SELECT."""
2341 if names is None:
2342 self.annotation_select_mask = None
2343 else:
2344 self.annotation_select_mask = set(names)
2345 self._annotation_select_cache = None
2347 def append_annotation_mask(self, names):
2348 if self.annotation_select_mask is not None:
2349 self.set_annotation_mask(self.annotation_select_mask.union(names))
2351 def set_extra_mask(self, names):
2352 """
2353 Set the mask of extra select items that will be returned by SELECT.
2354 Don't remove them from the Query since they might be used later.
2355 """
2356 if names is None:
2357 self.extra_select_mask = None
2358 else:
2359 self.extra_select_mask = set(names)
2360 self._extra_select_cache = None
2362 def set_values(self, fields):
2363 self.select_related = False
2364 self.clear_deferred_loading()
2365 self.clear_select_fields()
2366 self.has_select_fields = True
2368 if fields:
2369 field_names = []
2370 extra_names = []
2371 annotation_names = []
2372 if not self.extra and not self.annotations:
2373 # Shortcut - if there are no extra or annotations, then
2374 # the values() clause must be just field names.
2375 field_names = list(fields)
2376 else:
2377 self.default_cols = False
2378 for f in fields:
2379 if f in self.extra_select:
2380 extra_names.append(f)
2381 elif f in self.annotation_select:
2382 annotation_names.append(f)
2383 else:
2384 field_names.append(f)
2385 self.set_extra_mask(extra_names)
2386 self.set_annotation_mask(annotation_names)
2387 selected = frozenset(field_names + extra_names + annotation_names)
2388 else:
2389 field_names = [f.attname for f in self.model._meta.concrete_fields]
2390 selected = frozenset(field_names)
2391 # Selected annotations must be known before setting the GROUP BY
2392 # clause.
2393 if self.group_by is True:
2394 self.add_fields(
2395 (f.attname for f in self.model._meta.concrete_fields), False
2396 )
2397 # Disable GROUP BY aliases to avoid orphaning references to the
2398 # SELECT clause which is about to be cleared.
2399 self.set_group_by(allow_aliases=False)
2400 self.clear_select_fields()
2401 elif self.group_by:
2402 # Resolve GROUP BY annotation references if they are not part of
2403 # the selected fields anymore.
2404 group_by = []
2405 for expr in self.group_by:
2406 if isinstance(expr, Ref) and expr.refs not in selected:
2407 expr = self.annotations[expr.refs]
2408 group_by.append(expr)
2409 self.group_by = tuple(group_by)
2411 self.values_select = tuple(field_names)
2412 self.add_fields(field_names, True)
2414 @property
2415 def annotation_select(self):
2416 """
2417 Return the dictionary of aggregate columns that are not masked and
2418 should be used in the SELECT clause. Cache this result for performance.
2419 """
2420 if self._annotation_select_cache is not None:
2421 return self._annotation_select_cache
2422 elif not self.annotations:
2423 return {}
2424 elif self.annotation_select_mask is not None:
2425 self._annotation_select_cache = {
2426 k: v
2427 for k, v in self.annotations.items()
2428 if k in self.annotation_select_mask
2429 }
2430 return self._annotation_select_cache
2431 else:
2432 return self.annotations
2434 @property
2435 def extra_select(self):
2436 if self._extra_select_cache is not None:
2437 return self._extra_select_cache
2438 if not self.extra:
2439 return {}
2440 elif self.extra_select_mask is not None:
2441 self._extra_select_cache = {
2442 k: v for k, v in self.extra.items() if k in self.extra_select_mask
2443 }
2444 return self._extra_select_cache
2445 else:
2446 return self.extra
2448 def trim_start(self, names_with_path):
2449 """
2450 Trim joins from the start of the join path. The candidates for trim
2451 are the PathInfos in names_with_path structure that are m2m joins.
2453 Also set the select column so the start matches the join.
2455 This method is meant to be used for generating the subquery joins &
2456 cols in split_exclude().
2458 Return a lookup usable for doing outerq.filter(lookup=self) and a
2459 boolean indicating if the joins in the prefix contain a LEFT OUTER join.
2460 _"""
2461 all_paths = []
2462 for _, paths in names_with_path:
2463 all_paths.extend(paths)
2464 contains_louter = False
2465 # Trim and operate only on tables that were generated for
2466 # the lookup part of the query. That is, avoid trimming
2467 # joins generated for F() expressions.
2468 lookup_tables = [
2469 t for t in self.alias_map if t in self._lookup_joins or t == self.base_table
2470 ]
2471 for trimmed_paths, path in enumerate(all_paths):
2472 if path.m2m:
2473 break
2474 if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
2475 contains_louter = True
2476 alias = lookup_tables[trimmed_paths]
2477 self.unref_alias(alias)
2478 # The path.join_field is a Rel, lets get the other side's field
2479 join_field = path.join_field.field
2480 # Build the filter prefix.
2481 paths_in_prefix = trimmed_paths
2482 trimmed_prefix = []
2483 for name, path in names_with_path:
2484 if paths_in_prefix - len(path) < 0:
2485 break
2486 trimmed_prefix.append(name)
2487 paths_in_prefix -= len(path)
2488 trimmed_prefix.append(join_field.foreign_related_fields[0].name)
2489 trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
2490 # Lets still see if we can trim the first join from the inner query
2491 # (that is, self). We can't do this for:
2492 # - LEFT JOINs because we would miss those rows that have nothing on
2493 # the outer side,
2494 # - INNER JOINs from filtered relations because we would miss their
2495 # filters.
2496 first_join = self.alias_map[lookup_tables[trimmed_paths + 1]]
2497 if first_join.join_type != LOUTER and not first_join.filtered_relation:
2498 select_fields = [r[0] for r in join_field.related_fields]
2499 select_alias = lookup_tables[trimmed_paths + 1]
2500 self.unref_alias(lookup_tables[trimmed_paths])
2501 extra_restriction = join_field.get_extra_restriction(
2502 None, lookup_tables[trimmed_paths + 1]
2503 )
2504 if extra_restriction:
2505 self.where.add(extra_restriction, AND)
2506 else:
2507 # TODO: It might be possible to trim more joins from the start of the
2508 # inner query if it happens to have a longer join chain containing the
2509 # values in select_fields. Lets punt this one for now.
2510 select_fields = [r[1] for r in join_field.related_fields]
2511 select_alias = lookup_tables[trimmed_paths]
2512 # The found starting point is likely a join_class instead of a
2513 # base_table_class reference. But the first entry in the query's FROM
2514 # clause must not be a JOIN.
2515 for table in self.alias_map:
2516 if self.alias_refcount[table] > 0:
2517 self.alias_map[table] = self.base_table_class(
2518 self.alias_map[table].table_name,
2519 table,
2520 )
2521 break
2522 self.set_select([f.get_col(select_alias) for f in select_fields])
2523 return trimmed_prefix, contains_louter
2525 def is_nullable(self, field):
2526 """
2527 Check if the given field should be treated as nullable.
2529 Some backends treat '' as null and Plain treats such fields as
2530 nullable for those backends. In such situations field.null can be
2531 False even if we should treat the field as nullable.
2532 """
2533 # We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have
2534 # (nor should it have) knowledge of which connection is going to be
2535 # used. The proper fix would be to defer all decisions where
2536 # is_nullable() is needed to the compiler stage, but that is not easy
2537 # to do currently.
2538 return field.null or (
2539 field.empty_strings_allowed
2540 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
2541 )
2544def get_order_dir(field, default="ASC"):
2545 """
2546 Return the field name and direction for an order specification. For
2547 example, '-foo' is returned as ('foo', 'DESC').
2549 The 'default' param is used to indicate which way no prefix (or a '+'
2550 prefix) should sort. The '-' prefix always sorts the opposite way.
2551 """
2552 dirn = ORDER_DIR[default]
2553 if field[0] == "-":
2554 return field[1:], dirn[1]
2555 return field, dirn[0]
2558class JoinPromoter:
2559 """
2560 A class to abstract away join promotion problems for complex filter
2561 conditions.
2562 """
2564 def __init__(self, connector, num_children, negated):
2565 self.connector = connector
2566 self.negated = negated
2567 if self.negated:
2568 if connector == AND:
2569 self.effective_connector = OR
2570 else:
2571 self.effective_connector = AND
2572 else:
2573 self.effective_connector = self.connector
2574 self.num_children = num_children
2575 # Maps of table alias to how many times it is seen as required for
2576 # inner and/or outer joins.
2577 self.votes = Counter()
2579 def __repr__(self):
2580 return (
2581 f"{self.__class__.__qualname__}(connector={self.connector!r}, "
2582 f"num_children={self.num_children!r}, negated={self.negated!r})"
2583 )
2585 def add_votes(self, votes):
2586 """
2587 Add single vote per item to self.votes. Parameter can be any
2588 iterable.
2589 """
2590 self.votes.update(votes)
2592 def update_join_types(self, query):
2593 """
2594 Change join types so that the generated query is as efficient as
2595 possible, but still correct. So, change as many joins as possible
2596 to INNER, but don't make OUTER joins INNER if that could remove
2597 results from the query.
2598 """
2599 to_promote = set()
2600 to_demote = set()
2601 # The effective_connector is used so that NOT (a AND b) is treated
2602 # similarly to (a OR b) for join promotion.
2603 for table, votes in self.votes.items():
2604 # We must use outer joins in OR case when the join isn't contained
2605 # in all of the joins. Otherwise the INNER JOIN itself could remove
2606 # valid results. Consider the case where a model with rel_a and
2607 # rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now,
2608 # if rel_a join doesn't produce any results is null (for example
2609 # reverse foreign key or null value in direct foreign key), and
2610 # there is a matching row in rel_b with col=2, then an INNER join
2611 # to rel_a would remove a valid match from the query. So, we need
2612 # to promote any existing INNER to LOUTER (it is possible this
2613 # promotion in turn will be demoted later on).
2614 if self.effective_connector == OR and votes < self.num_children:
2615 to_promote.add(table)
2616 # If connector is AND and there is a filter that can match only
2617 # when there is a joinable row, then use INNER. For example, in
2618 # rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL
2619 # as join output, then the col=1 or col=2 can't match (as
2620 # NULL=anything is always false).
2621 # For the OR case, if all children voted for a join to be inner,
2622 # then we can use INNER for the join. For example:
2623 # (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell)
2624 # then if rel_a doesn't produce any rows, the whole condition
2625 # can't match. Hence we can safely use INNER join.
2626 if self.effective_connector == AND or (
2627 self.effective_connector == OR and votes == self.num_children
2628 ):
2629 to_demote.add(table)
2630 # Finally, what happens in cases where we have:
2631 # (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0
2632 # Now, we first generate the OR clause, and promote joins for it
2633 # in the first if branch above. Both rel_a and rel_b are promoted
2634 # to LOUTER joins. After that we do the AND case. The OR case
2635 # voted no inner joins but the rel_a__col__gte=0 votes inner join
2636 # for rel_a. We demote it back to INNER join (in AND case a single
2637 # vote is enough). The demotion is OK, if rel_a doesn't produce
2638 # rows, then the rel_a__col__gte=0 clause can't be true, and thus
2639 # the whole clause must be false. So, it is safe to use INNER
2640 # join.
2641 # Note that in this example we could just as well have the __gte
2642 # clause and the OR clause swapped. Or we could replace the __gte
2643 # clause with an OR clause containing rel_a__col=1|rel_a__col=2,
2644 # and again we could safely demote to INNER.
2645 query.promote_joins(to_promote)
2646 query.demote_joins(to_demote)
2647 return to_demote