Coverage for /Users/davegaeddert/Developer/dropseed/plain/plain-models/plain/models/sql/query.py: 46%

1276 statements  

« prev     ^ index     » next       coverage.py v7.6.9, created at 2024-12-23 11:16 -0600

1""" 

2Create SQL statements for QuerySets. 

3 

4The code in here encapsulates all of the SQL construction so that QuerySets 

5themselves do not have to (and could be backed by things other than SQL 

6databases). The abstraction barrier only works one way: this module has to know 

7all about the internals of models in order to get the information it needs. 

8""" 

9 

10import copy 

11import difflib 

12import functools 

13import sys 

14from collections import Counter, namedtuple 

15from collections.abc import Iterator, Mapping 

16from itertools import chain, count, product 

17from string import ascii_uppercase 

18 

19from plain.exceptions import FieldDoesNotExist, FieldError 

20from plain.models.aggregates import Count 

21from plain.models.constants import LOOKUP_SEP 

22from plain.models.db import DEFAULT_DB_ALIAS, NotSupportedError, connections 

23from plain.models.expressions import ( 

24 BaseExpression, 

25 Col, 

26 Exists, 

27 F, 

28 OuterRef, 

29 Ref, 

30 ResolvedOuterRef, 

31 Value, 

32) 

33from plain.models.fields import Field 

34from plain.models.fields.related_lookups import MultiColSource 

35from plain.models.lookups import Lookup 

36from plain.models.query_utils import ( 

37 Q, 

38 check_rel_lookup_compatibility, 

39 refs_expression, 

40) 

41from plain.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE 

42from plain.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin 

43from plain.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode 

44from plain.utils.functional import cached_property 

45from plain.utils.regex_helper import _lazy_re_compile 

46from plain.utils.tree import Node 

47 

48__all__ = ["Query", "RawQuery"] 

49 

50# Quotation marks ('"`[]), whitespace characters, semicolons, or inline 

51# SQL comments are forbidden in column aliases. 

52FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/") 

53 

54# Inspired from 

55# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS 

56EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w\-]+") 

57 

58 

59def get_field_names_from_opts(opts): 

60 if opts is None: 

61 return set() 

62 return set( 

63 chain.from_iterable( 

64 (f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields() 

65 ) 

66 ) 

67 

68 

69def get_children_from_q(q): 

70 for child in q.children: 

71 if isinstance(child, Node): 

72 yield from get_children_from_q(child) 

73 else: 

74 yield child 

75 

76 

77JoinInfo = namedtuple( 

78 "JoinInfo", 

79 ("final_field", "targets", "opts", "joins", "path", "transform_function"), 

80) 

81 

82 

83class RawQuery: 

84 """A single raw SQL query.""" 

85 

86 def __init__(self, sql, using, params=()): 

87 self.params = params 

88 self.sql = sql 

89 self.using = using 

90 self.cursor = None 

91 

92 # Mirror some properties of a normal query so that 

93 # the compiler can be used to process results. 

94 self.low_mark, self.high_mark = 0, None # Used for offset/limit 

95 self.extra_select = {} 

96 self.annotation_select = {} 

97 

98 def chain(self, using): 

99 return self.clone(using) 

100 

101 def clone(self, using): 

102 return RawQuery(self.sql, using, params=self.params) 

103 

104 def get_columns(self): 

105 if self.cursor is None: 

106 self._execute_query() 

107 converter = connections[self.using].introspection.identifier_converter 

108 return [converter(column_meta[0]) for column_meta in self.cursor.description] 

109 

110 def __iter__(self): 

111 # Always execute a new query for a new iterator. 

112 # This could be optimized with a cache at the expense of RAM. 

113 self._execute_query() 

114 if not connections[self.using].features.can_use_chunked_reads: 

115 # If the database can't use chunked reads we need to make sure we 

116 # evaluate the entire query up front. 

117 result = list(self.cursor) 

118 else: 

119 result = self.cursor 

120 return iter(result) 

121 

122 def __repr__(self): 

123 return f"<{self.__class__.__name__}: {self}>" 

124 

125 @property 

126 def params_type(self): 

127 if self.params is None: 

128 return None 

129 return dict if isinstance(self.params, Mapping) else tuple 

130 

131 def __str__(self): 

132 if self.params_type is None: 

133 return self.sql 

134 return self.sql % self.params_type(self.params) 

135 

136 def _execute_query(self): 

137 connection = connections[self.using] 

138 

139 # Adapt parameters to the database, as much as possible considering 

140 # that the target type isn't known. See #17755. 

141 params_type = self.params_type 

142 adapter = connection.ops.adapt_unknown_value 

143 if params_type is tuple: 

144 params = tuple(adapter(val) for val in self.params) 

145 elif params_type is dict: 

146 params = {key: adapter(val) for key, val in self.params.items()} 

147 elif params_type is None: 

148 params = None 

149 else: 

150 raise RuntimeError(f"Unexpected params type: {params_type}") 

151 

152 self.cursor = connection.cursor() 

153 self.cursor.execute(self.sql, params) 

154 

155 

156ExplainInfo = namedtuple("ExplainInfo", ("format", "options")) 

157 

158 

159class Query(BaseExpression): 

160 """A single SQL query.""" 

161 

162 alias_prefix = "T" 

163 empty_result_set_value = None 

164 subq_aliases = frozenset([alias_prefix]) 

165 

166 compiler = "SQLCompiler" 

167 

168 base_table_class = BaseTable 

169 join_class = Join 

170 

171 default_cols = True 

172 default_ordering = True 

173 standard_ordering = True 

174 

175 filter_is_sticky = False 

176 subquery = False 

177 

178 # SQL-related attributes. 

179 # Select and related select clauses are expressions to use in the SELECT 

180 # clause of the query. The select is used for cases where we want to set up 

181 # the select clause to contain other than default fields (values(), 

182 # subqueries...). Note that annotations go to annotations dictionary. 

183 select = () 

184 # The group_by attribute can have one of the following forms: 

185 # - None: no group by at all in the query 

186 # - A tuple of expressions: group by (at least) those expressions. 

187 # String refs are also allowed for now. 

188 # - True: group by all select fields of the model 

189 # See compiler.get_group_by() for details. 

190 group_by = None 

191 order_by = () 

192 low_mark = 0 # Used for offset/limit. 

193 high_mark = None # Used for offset/limit. 

194 distinct = False 

195 distinct_fields = () 

196 select_for_update = False 

197 select_for_update_nowait = False 

198 select_for_update_skip_locked = False 

199 select_for_update_of = () 

200 select_for_no_key_update = False 

201 select_related = False 

202 has_select_fields = False 

203 # Arbitrary limit for select_related to prevents infinite recursion. 

204 max_depth = 5 

205 # Holds the selects defined by a call to values() or values_list() 

206 # excluding annotation_select and extra_select. 

207 values_select = () 

208 

209 # SQL annotation-related attributes. 

210 annotation_select_mask = None 

211 _annotation_select_cache = None 

212 

213 # Set combination attributes. 

214 combinator = None 

215 combinator_all = False 

216 combined_queries = () 

217 

218 # These are for extensions. The contents are more or less appended verbatim 

219 # to the appropriate clause. 

220 extra_select_mask = None 

221 _extra_select_cache = None 

222 

223 extra_tables = () 

224 extra_order_by = () 

225 

226 # A tuple that is a set of model field names and either True, if these are 

227 # the fields to defer, or False if these are the only fields to load. 

228 deferred_loading = (frozenset(), True) 

229 

230 explain_info = None 

231 

232 def __init__(self, model, alias_cols=True): 

233 self.model = model 

234 self.alias_refcount = {} 

235 # alias_map is the most important data structure regarding joins. 

236 # It's used for recording which joins exist in the query and what 

237 # types they are. The key is the alias of the joined table (possibly 

238 # the table name) and the value is a Join-like object (see 

239 # sql.datastructures.Join for more information). 

240 self.alias_map = {} 

241 # Whether to provide alias to columns during reference resolving. 

242 self.alias_cols = alias_cols 

243 # Sometimes the query contains references to aliases in outer queries (as 

244 # a result of split_exclude). Correct alias quoting needs to know these 

245 # aliases too. 

246 # Map external tables to whether they are aliased. 

247 self.external_aliases = {} 

248 self.table_map = {} # Maps table names to list of aliases. 

249 self.used_aliases = set() 

250 

251 self.where = WhereNode() 

252 # Maps alias -> Annotation Expression. 

253 self.annotations = {} 

254 # These are for extensions. The contents are more or less appended 

255 # verbatim to the appropriate clause. 

256 self.extra = {} # Maps col_alias -> (col_sql, params). 

257 

258 self._filtered_relations = {} 

259 

260 @property 

261 def output_field(self): 

262 if len(self.select) == 1: 

263 select = self.select[0] 

264 return getattr(select, "target", None) or select.field 

265 elif len(self.annotation_select) == 1: 

266 return next(iter(self.annotation_select.values())).output_field 

267 

268 @cached_property 

269 def base_table(self): 

270 for alias in self.alias_map: 

271 return alias 

272 

273 def __str__(self): 

274 """ 

275 Return the query as a string of SQL with the parameter values 

276 substituted in (use sql_with_params() to see the unsubstituted string). 

277 

278 Parameter values won't necessarily be quoted correctly, since that is 

279 done by the database interface at execution time. 

280 """ 

281 sql, params = self.sql_with_params() 

282 return sql % params 

283 

284 def sql_with_params(self): 

285 """ 

286 Return the query as an SQL string and the parameters that will be 

287 substituted into the query. 

288 """ 

289 return self.get_compiler(DEFAULT_DB_ALIAS).as_sql() 

290 

291 def __deepcopy__(self, memo): 

292 """Limit the amount of work when a Query is deepcopied.""" 

293 result = self.clone() 

294 memo[id(self)] = result 

295 return result 

296 

297 def get_compiler(self, using=None, connection=None, elide_empty=True): 

298 if using is None and connection is None: 

299 raise ValueError("Need either using or connection") 

300 if using: 

301 connection = connections[using] 

302 return connection.ops.compiler(self.compiler)( 

303 self, connection, using, elide_empty 

304 ) 

305 

306 def get_meta(self): 

307 """ 

308 Return the Options instance (the model._meta) from which to start 

309 processing. Normally, this is self.model._meta, but it can be changed 

310 by subclasses. 

311 """ 

312 if self.model: 

313 return self.model._meta 

314 

315 def clone(self): 

316 """ 

317 Return a copy of the current Query. A lightweight alternative to 

318 deepcopy(). 

319 """ 

320 obj = Empty() 

321 obj.__class__ = self.__class__ 

322 # Copy references to everything. 

323 obj.__dict__ = self.__dict__.copy() 

324 # Clone attributes that can't use shallow copy. 

325 obj.alias_refcount = self.alias_refcount.copy() 

326 obj.alias_map = self.alias_map.copy() 

327 obj.external_aliases = self.external_aliases.copy() 

328 obj.table_map = self.table_map.copy() 

329 obj.where = self.where.clone() 

330 obj.annotations = self.annotations.copy() 

331 if self.annotation_select_mask is not None: 

332 obj.annotation_select_mask = self.annotation_select_mask.copy() 

333 if self.combined_queries: 

334 obj.combined_queries = tuple( 

335 [query.clone() for query in self.combined_queries] 

336 ) 

337 # _annotation_select_cache cannot be copied, as doing so breaks the 

338 # (necessary) state in which both annotations and 

339 # _annotation_select_cache point to the same underlying objects. 

340 # It will get re-populated in the cloned queryset the next time it's 

341 # used. 

342 obj._annotation_select_cache = None 

343 obj.extra = self.extra.copy() 

344 if self.extra_select_mask is not None: 

345 obj.extra_select_mask = self.extra_select_mask.copy() 

346 if self._extra_select_cache is not None: 

347 obj._extra_select_cache = self._extra_select_cache.copy() 

348 if self.select_related is not False: 

349 # Use deepcopy because select_related stores fields in nested 

350 # dicts. 

351 obj.select_related = copy.deepcopy(obj.select_related) 

352 if "subq_aliases" in self.__dict__: 

353 obj.subq_aliases = self.subq_aliases.copy() 

354 obj.used_aliases = self.used_aliases.copy() 

355 obj._filtered_relations = self._filtered_relations.copy() 

356 # Clear the cached_property, if it exists. 

357 obj.__dict__.pop("base_table", None) 

358 return obj 

359 

360 def chain(self, klass=None): 

361 """ 

362 Return a copy of the current Query that's ready for another operation. 

363 The klass argument changes the type of the Query, e.g. UpdateQuery. 

364 """ 

365 obj = self.clone() 

366 if klass and obj.__class__ != klass: 

367 obj.__class__ = klass 

368 if not obj.filter_is_sticky: 

369 obj.used_aliases = set() 

370 obj.filter_is_sticky = False 

371 if hasattr(obj, "_setup_query"): 

372 obj._setup_query() 

373 return obj 

374 

375 def relabeled_clone(self, change_map): 

376 clone = self.clone() 

377 clone.change_aliases(change_map) 

378 return clone 

379 

380 def _get_col(self, target, field, alias): 

381 if not self.alias_cols: 

382 alias = None 

383 return target.get_col(alias, field) 

384 

385 def get_aggregation(self, using, aggregate_exprs): 

386 """ 

387 Return the dictionary with the values of the existing aggregations. 

388 """ 

389 if not aggregate_exprs: 

390 return {} 

391 aggregates = {} 

392 for alias, aggregate_expr in aggregate_exprs.items(): 

393 self.check_alias(alias) 

394 aggregate = aggregate_expr.resolve_expression( 

395 self, allow_joins=True, reuse=None, summarize=True 

396 ) 

397 if not aggregate.contains_aggregate: 

398 raise TypeError(f"{alias} is not an aggregate expression") 

399 aggregates[alias] = aggregate 

400 # Existing usage of aggregation can be determined by the presence of 

401 # selected aggregates but also by filters against aliased aggregates. 

402 _, having, qualify = self.where.split_having_qualify() 

403 has_existing_aggregation = ( 

404 any( 

405 getattr(annotation, "contains_aggregate", True) 

406 for annotation in self.annotations.values() 

407 ) 

408 or having 

409 ) 

410 # Decide if we need to use a subquery. 

411 # 

412 # Existing aggregations would cause incorrect results as 

413 # get_aggregation() must produce just one result and thus must not use 

414 # GROUP BY. 

415 # 

416 # If the query has limit or distinct, or uses set operations, then 

417 # those operations must be done in a subquery so that the query 

418 # aggregates on the limit and/or distinct results instead of applying 

419 # the distinct and limit after the aggregation. 

420 if ( 

421 isinstance(self.group_by, tuple) 

422 or self.is_sliced 

423 or has_existing_aggregation 

424 or qualify 

425 or self.distinct 

426 or self.combinator 

427 ): 

428 from plain.models.sql.subqueries import AggregateQuery 

429 

430 inner_query = self.clone() 

431 inner_query.subquery = True 

432 outer_query = AggregateQuery(self.model, inner_query) 

433 inner_query.select_for_update = False 

434 inner_query.select_related = False 

435 inner_query.set_annotation_mask(self.annotation_select) 

436 # Queries with distinct_fields need ordering and when a limit is 

437 # applied we must take the slice from the ordered query. Otherwise 

438 # no need for ordering. 

439 inner_query.clear_ordering(force=False) 

440 if not inner_query.distinct: 

441 # If the inner query uses default select and it has some 

442 # aggregate annotations, then we must make sure the inner 

443 # query is grouped by the main model's primary key. However, 

444 # clearing the select clause can alter results if distinct is 

445 # used. 

446 if inner_query.default_cols and has_existing_aggregation: 

447 inner_query.group_by = ( 

448 self.model._meta.pk.get_col(inner_query.get_initial_alias()), 

449 ) 

450 inner_query.default_cols = False 

451 if not qualify: 

452 # Mask existing annotations that are not referenced by 

453 # aggregates to be pushed to the outer query unless 

454 # filtering against window functions is involved as it 

455 # requires complex realising. 

456 annotation_mask = set() 

457 for aggregate in aggregates.values(): 

458 annotation_mask |= aggregate.get_refs() 

459 inner_query.set_annotation_mask(annotation_mask) 

460 

461 # Add aggregates to the outer AggregateQuery. This requires making 

462 # sure all columns referenced by the aggregates are selected in the 

463 # inner query. It is achieved by retrieving all column references 

464 # by the aggregates, explicitly selecting them in the inner query, 

465 # and making sure the aggregates are repointed to them. 

466 col_refs = {} 

467 for alias, aggregate in aggregates.items(): 

468 replacements = {} 

469 for col in self._gen_cols([aggregate], resolve_refs=False): 

470 if not (col_ref := col_refs.get(col)): 

471 index = len(col_refs) + 1 

472 col_alias = f"__col{index}" 

473 col_ref = Ref(col_alias, col) 

474 col_refs[col] = col_ref 

475 inner_query.annotations[col_alias] = col 

476 inner_query.append_annotation_mask([col_alias]) 

477 replacements[col] = col_ref 

478 outer_query.annotations[alias] = aggregate.replace_expressions( 

479 replacements 

480 ) 

481 if ( 

482 inner_query.select == () 

483 and not inner_query.default_cols 

484 and not inner_query.annotation_select_mask 

485 ): 

486 # In case of Model.objects[0:3].count(), there would be no 

487 # field selected in the inner query, yet we must use a subquery. 

488 # So, make sure at least one field is selected. 

489 inner_query.select = ( 

490 self.model._meta.pk.get_col(inner_query.get_initial_alias()), 

491 ) 

492 else: 

493 outer_query = self 

494 self.select = () 

495 self.default_cols = False 

496 self.extra = {} 

497 if self.annotations: 

498 # Inline reference to existing annotations and mask them as 

499 # they are unnecessary given only the summarized aggregations 

500 # are requested. 

501 replacements = { 

502 Ref(alias, annotation): annotation 

503 for alias, annotation in self.annotations.items() 

504 } 

505 self.annotations = { 

506 alias: aggregate.replace_expressions(replacements) 

507 for alias, aggregate in aggregates.items() 

508 } 

509 else: 

510 self.annotations = aggregates 

511 self.set_annotation_mask(aggregates) 

512 

513 empty_set_result = [ 

514 expression.empty_result_set_value 

515 for expression in outer_query.annotation_select.values() 

516 ] 

517 elide_empty = not any(result is NotImplemented for result in empty_set_result) 

518 outer_query.clear_ordering(force=True) 

519 outer_query.clear_limits() 

520 outer_query.select_for_update = False 

521 outer_query.select_related = False 

522 compiler = outer_query.get_compiler(using, elide_empty=elide_empty) 

523 result = compiler.execute_sql(SINGLE) 

524 if result is None: 

525 result = empty_set_result 

526 else: 

527 converters = compiler.get_converters(outer_query.annotation_select.values()) 

528 result = next(compiler.apply_converters((result,), converters)) 

529 

530 return dict(zip(outer_query.annotation_select, result)) 

531 

532 def get_count(self, using): 

533 """ 

534 Perform a COUNT() query using the current filter constraints. 

535 """ 

536 obj = self.clone() 

537 return obj.get_aggregation(using, {"__count": Count("*")})["__count"] 

538 

539 def has_filters(self): 

540 return self.where 

541 

542 def exists(self, limit=True): 

543 q = self.clone() 

544 if not (q.distinct and q.is_sliced): 

545 if q.group_by is True: 

546 q.add_fields( 

547 (f.attname for f in self.model._meta.concrete_fields), False 

548 ) 

549 # Disable GROUP BY aliases to avoid orphaning references to the 

550 # SELECT clause which is about to be cleared. 

551 q.set_group_by(allow_aliases=False) 

552 q.clear_select_clause() 

553 if q.combined_queries and q.combinator == "union": 

554 q.combined_queries = tuple( 

555 combined_query.exists(limit=False) 

556 for combined_query in q.combined_queries 

557 ) 

558 q.clear_ordering(force=True) 

559 if limit: 

560 q.set_limits(high=1) 

561 q.add_annotation(Value(1), "a") 

562 return q 

563 

564 def has_results(self, using): 

565 q = self.exists(using) 

566 compiler = q.get_compiler(using=using) 

567 return compiler.has_results() 

568 

569 def explain(self, using, format=None, **options): 

570 q = self.clone() 

571 for option_name in options: 

572 if ( 

573 not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name) 

574 or "--" in option_name 

575 ): 

576 raise ValueError(f"Invalid option name: {option_name!r}.") 

577 q.explain_info = ExplainInfo(format, options) 

578 compiler = q.get_compiler(using=using) 

579 return "\n".join(compiler.explain_query()) 

580 

581 def combine(self, rhs, connector): 

582 """ 

583 Merge the 'rhs' query into the current one (with any 'rhs' effects 

584 being applied *after* (that is, "to the right of") anything in the 

585 current query. 'rhs' is not modified during a call to this function. 

586 

587 The 'connector' parameter describes how to connect filters from the 

588 'rhs' query. 

589 """ 

590 if self.model != rhs.model: 

591 raise TypeError("Cannot combine queries on two different base models.") 

592 if self.is_sliced: 

593 raise TypeError("Cannot combine queries once a slice has been taken.") 

594 if self.distinct != rhs.distinct: 

595 raise TypeError("Cannot combine a unique query with a non-unique query.") 

596 if self.distinct_fields != rhs.distinct_fields: 

597 raise TypeError("Cannot combine queries with different distinct fields.") 

598 

599 # If lhs and rhs shares the same alias prefix, it is possible to have 

600 # conflicting alias changes like T4 -> T5, T5 -> T6, which might end up 

601 # as T4 -> T6 while combining two querysets. To prevent this, change an 

602 # alias prefix of the rhs and update current aliases accordingly, 

603 # except if the alias is the base table since it must be present in the 

604 # query on both sides. 

605 initial_alias = self.get_initial_alias() 

606 rhs.bump_prefix(self, exclude={initial_alias}) 

607 

608 # Work out how to relabel the rhs aliases, if necessary. 

609 change_map = {} 

610 conjunction = connector == AND 

611 

612 # Determine which existing joins can be reused. When combining the 

613 # query with AND we must recreate all joins for m2m filters. When 

614 # combining with OR we can reuse joins. The reason is that in AND 

615 # case a single row can't fulfill a condition like: 

616 # revrel__col=1 & revrel__col=2 

617 # But, there might be two different related rows matching this 

618 # condition. In OR case a single True is enough, so single row is 

619 # enough, too. 

620 # 

621 # Note that we will be creating duplicate joins for non-m2m joins in 

622 # the AND case. The results will be correct but this creates too many 

623 # joins. This is something that could be fixed later on. 

624 reuse = set() if conjunction else set(self.alias_map) 

625 joinpromoter = JoinPromoter(connector, 2, False) 

626 joinpromoter.add_votes( 

627 j for j in self.alias_map if self.alias_map[j].join_type == INNER 

628 ) 

629 rhs_votes = set() 

630 # Now, add the joins from rhs query into the new query (skipping base 

631 # table). 

632 rhs_tables = list(rhs.alias_map)[1:] 

633 for alias in rhs_tables: 

634 join = rhs.alias_map[alias] 

635 # If the left side of the join was already relabeled, use the 

636 # updated alias. 

637 join = join.relabeled_clone(change_map) 

638 new_alias = self.join(join, reuse=reuse) 

639 if join.join_type == INNER: 

640 rhs_votes.add(new_alias) 

641 # We can't reuse the same join again in the query. If we have two 

642 # distinct joins for the same connection in rhs query, then the 

643 # combined query must have two joins, too. 

644 reuse.discard(new_alias) 

645 if alias != new_alias: 

646 change_map[alias] = new_alias 

647 if not rhs.alias_refcount[alias]: 

648 # The alias was unused in the rhs query. Unref it so that it 

649 # will be unused in the new query, too. We have to add and 

650 # unref the alias so that join promotion has information of 

651 # the join type for the unused alias. 

652 self.unref_alias(new_alias) 

653 joinpromoter.add_votes(rhs_votes) 

654 joinpromoter.update_join_types(self) 

655 

656 # Combine subqueries aliases to ensure aliases relabelling properly 

657 # handle subqueries when combining where and select clauses. 

658 self.subq_aliases |= rhs.subq_aliases 

659 

660 # Now relabel a copy of the rhs where-clause and add it to the current 

661 # one. 

662 w = rhs.where.clone() 

663 w.relabel_aliases(change_map) 

664 self.where.add(w, connector) 

665 

666 # Selection columns and extra extensions are those provided by 'rhs'. 

667 if rhs.select: 

668 self.set_select([col.relabeled_clone(change_map) for col in rhs.select]) 

669 else: 

670 self.select = () 

671 

672 if connector == OR: 

673 # It would be nice to be able to handle this, but the queries don't 

674 # really make sense (or return consistent value sets). Not worth 

675 # the extra complexity when you can write a real query instead. 

676 if self.extra and rhs.extra: 

677 raise ValueError( 

678 "When merging querysets using 'or', you cannot have " 

679 "extra(select=...) on both sides." 

680 ) 

681 self.extra.update(rhs.extra) 

682 extra_select_mask = set() 

683 if self.extra_select_mask is not None: 

684 extra_select_mask.update(self.extra_select_mask) 

685 if rhs.extra_select_mask is not None: 

686 extra_select_mask.update(rhs.extra_select_mask) 

687 if extra_select_mask: 

688 self.set_extra_mask(extra_select_mask) 

689 self.extra_tables += rhs.extra_tables 

690 

691 # Ordering uses the 'rhs' ordering, unless it has none, in which case 

692 # the current ordering is used. 

693 self.order_by = rhs.order_by or self.order_by 

694 self.extra_order_by = rhs.extra_order_by or self.extra_order_by 

695 

696 def _get_defer_select_mask(self, opts, mask, select_mask=None): 

697 if select_mask is None: 

698 select_mask = {} 

699 select_mask[opts.pk] = {} 

700 # All concrete fields that are not part of the defer mask must be 

701 # loaded. If a relational field is encountered it gets added to the 

702 # mask for it be considered if `select_related` and the cycle continues 

703 # by recursively calling this function. 

704 for field in opts.concrete_fields: 

705 field_mask = mask.pop(field.name, None) 

706 if field_mask is None: 

707 select_mask.setdefault(field, {}) 

708 elif field_mask: 

709 if not field.is_relation: 

710 raise FieldError(next(iter(field_mask))) 

711 field_select_mask = select_mask.setdefault(field, {}) 

712 related_model = field.remote_field.model._meta.concrete_model 

713 self._get_defer_select_mask( 

714 related_model._meta, field_mask, field_select_mask 

715 ) 

716 # Remaining defer entries must be references to reverse relationships. 

717 # The following code is expected to raise FieldError if it encounters 

718 # a malformed defer entry. 

719 for field_name, field_mask in mask.items(): 

720 if filtered_relation := self._filtered_relations.get(field_name): 

721 relation = opts.get_field(filtered_relation.relation_name) 

722 field_select_mask = select_mask.setdefault((field_name, relation), {}) 

723 field = relation.field 

724 else: 

725 field = opts.get_field(field_name).field 

726 field_select_mask = select_mask.setdefault(field, {}) 

727 related_model = field.model._meta.concrete_model 

728 self._get_defer_select_mask( 

729 related_model._meta, field_mask, field_select_mask 

730 ) 

731 return select_mask 

732 

733 def _get_only_select_mask(self, opts, mask, select_mask=None): 

734 if select_mask is None: 

735 select_mask = {} 

736 select_mask[opts.pk] = {} 

737 # Only include fields mentioned in the mask. 

738 for field_name, field_mask in mask.items(): 

739 field = opts.get_field(field_name) 

740 field_select_mask = select_mask.setdefault(field, {}) 

741 if field_mask: 

742 if not field.is_relation: 

743 raise FieldError(next(iter(field_mask))) 

744 related_model = field.remote_field.model._meta.concrete_model 

745 self._get_only_select_mask( 

746 related_model._meta, field_mask, field_select_mask 

747 ) 

748 return select_mask 

749 

750 def get_select_mask(self): 

751 """ 

752 Convert the self.deferred_loading data structure to an alternate data 

753 structure, describing the field that *will* be loaded. This is used to 

754 compute the columns to select from the database and also by the 

755 QuerySet class to work out which fields are being initialized on each 

756 model. Models that have all their fields included aren't mentioned in 

757 the result, only those that have field restrictions in place. 

758 """ 

759 field_names, defer = self.deferred_loading 

760 if not field_names: 

761 return {} 

762 mask = {} 

763 for field_name in field_names: 

764 part_mask = mask 

765 for part in field_name.split(LOOKUP_SEP): 

766 part_mask = part_mask.setdefault(part, {}) 

767 opts = self.get_meta() 

768 if defer: 

769 return self._get_defer_select_mask(opts, mask) 

770 return self._get_only_select_mask(opts, mask) 

771 

772 def table_alias(self, table_name, create=False, filtered_relation=None): 

773 """ 

774 Return a table alias for the given table_name and whether this is a 

775 new alias or not. 

776 

777 If 'create' is true, a new alias is always created. Otherwise, the 

778 most recently created alias for the table (if one exists) is reused. 

779 """ 

780 alias_list = self.table_map.get(table_name) 

781 if not create and alias_list: 

782 alias = alias_list[0] 

783 self.alias_refcount[alias] += 1 

784 return alias, False 

785 

786 # Create a new alias for this table. 

787 if alias_list: 

788 alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1) 

789 alias_list.append(alias) 

790 else: 

791 # The first occurrence of a table uses the table name directly. 

792 alias = ( 

793 filtered_relation.alias if filtered_relation is not None else table_name 

794 ) 

795 self.table_map[table_name] = [alias] 

796 self.alias_refcount[alias] = 1 

797 return alias, True 

798 

799 def ref_alias(self, alias): 

800 """Increases the reference count for this alias.""" 

801 self.alias_refcount[alias] += 1 

802 

803 def unref_alias(self, alias, amount=1): 

804 """Decreases the reference count for this alias.""" 

805 self.alias_refcount[alias] -= amount 

806 

807 def promote_joins(self, aliases): 

808 """ 

809 Promote recursively the join type of given aliases and its children to 

810 an outer join. If 'unconditional' is False, only promote the join if 

811 it is nullable or the parent join is an outer join. 

812 

813 The children promotion is done to avoid join chains that contain a LOUTER 

814 b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted, 

815 then we must also promote b->c automatically, or otherwise the promotion 

816 of a->b doesn't actually change anything in the query results. 

817 """ 

818 aliases = list(aliases) 

819 while aliases: 

820 alias = aliases.pop(0) 

821 if self.alias_map[alias].join_type is None: 

822 # This is the base table (first FROM entry) - this table 

823 # isn't really joined at all in the query, so we should not 

824 # alter its join type. 

825 continue 

826 # Only the first alias (skipped above) should have None join_type 

827 assert self.alias_map[alias].join_type is not None 

828 parent_alias = self.alias_map[alias].parent_alias 

829 parent_louter = ( 

830 parent_alias and self.alias_map[parent_alias].join_type == LOUTER 

831 ) 

832 already_louter = self.alias_map[alias].join_type == LOUTER 

833 if (self.alias_map[alias].nullable or parent_louter) and not already_louter: 

834 self.alias_map[alias] = self.alias_map[alias].promote() 

835 # Join type of 'alias' changed, so re-examine all aliases that 

836 # refer to this one. 

837 aliases.extend( 

838 join 

839 for join in self.alias_map 

840 if self.alias_map[join].parent_alias == alias 

841 and join not in aliases 

842 ) 

843 

844 def demote_joins(self, aliases): 

845 """ 

846 Change join type from LOUTER to INNER for all joins in aliases. 

847 

848 Similarly to promote_joins(), this method must ensure no join chains 

849 containing first an outer, then an inner join are generated. If we 

850 are demoting b->c join in chain a LOUTER b LOUTER c then we must 

851 demote a->b automatically, or otherwise the demotion of b->c doesn't 

852 actually change anything in the query results. . 

853 """ 

854 aliases = list(aliases) 

855 while aliases: 

856 alias = aliases.pop(0) 

857 if self.alias_map[alias].join_type == LOUTER: 

858 self.alias_map[alias] = self.alias_map[alias].demote() 

859 parent_alias = self.alias_map[alias].parent_alias 

860 if self.alias_map[parent_alias].join_type == INNER: 

861 aliases.append(parent_alias) 

862 

863 def reset_refcounts(self, to_counts): 

864 """ 

865 Reset reference counts for aliases so that they match the value passed 

866 in `to_counts`. 

867 """ 

868 for alias, cur_refcount in self.alias_refcount.copy().items(): 

869 unref_amount = cur_refcount - to_counts.get(alias, 0) 

870 self.unref_alias(alias, unref_amount) 

871 

872 def change_aliases(self, change_map): 

873 """ 

874 Change the aliases in change_map (which maps old-alias -> new-alias), 

875 relabelling any references to them in select columns and the where 

876 clause. 

877 """ 

878 # If keys and values of change_map were to intersect, an alias might be 

879 # updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending 

880 # on their order in change_map. 

881 assert set(change_map).isdisjoint(change_map.values()) 

882 

883 # 1. Update references in "select" (normal columns plus aliases), 

884 # "group by" and "where". 

885 self.where.relabel_aliases(change_map) 

886 if isinstance(self.group_by, tuple): 

887 self.group_by = tuple( 

888 [col.relabeled_clone(change_map) for col in self.group_by] 

889 ) 

890 self.select = tuple([col.relabeled_clone(change_map) for col in self.select]) 

891 self.annotations = self.annotations and { 

892 key: col.relabeled_clone(change_map) 

893 for key, col in self.annotations.items() 

894 } 

895 

896 # 2. Rename the alias in the internal table/alias datastructures. 

897 for old_alias, new_alias in change_map.items(): 

898 if old_alias not in self.alias_map: 

899 continue 

900 alias_data = self.alias_map[old_alias].relabeled_clone(change_map) 

901 self.alias_map[new_alias] = alias_data 

902 self.alias_refcount[new_alias] = self.alias_refcount[old_alias] 

903 del self.alias_refcount[old_alias] 

904 del self.alias_map[old_alias] 

905 

906 table_aliases = self.table_map[alias_data.table_name] 

907 for pos, alias in enumerate(table_aliases): 

908 if alias == old_alias: 

909 table_aliases[pos] = new_alias 

910 break 

911 self.external_aliases = { 

912 # Table is aliased or it's being changed and thus is aliased. 

913 change_map.get(alias, alias): (aliased or alias in change_map) 

914 for alias, aliased in self.external_aliases.items() 

915 } 

916 

917 def bump_prefix(self, other_query, exclude=None): 

918 """ 

919 Change the alias prefix to the next letter in the alphabet in a way 

920 that the other query's aliases and this query's aliases will not 

921 conflict. Even tables that previously had no alias will get an alias 

922 after this call. To prevent changing aliases use the exclude parameter. 

923 """ 

924 

925 def prefix_gen(): 

926 """ 

927 Generate a sequence of characters in alphabetical order: 

928 -> 'A', 'B', 'C', ... 

929 

930 When the alphabet is finished, the sequence will continue with the 

931 Cartesian product: 

932 -> 'AA', 'AB', 'AC', ... 

933 """ 

934 alphabet = ascii_uppercase 

935 prefix = chr(ord(self.alias_prefix) + 1) 

936 yield prefix 

937 for n in count(1): 

938 seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet 

939 for s in product(seq, repeat=n): 

940 yield "".join(s) 

941 prefix = None 

942 

943 if self.alias_prefix != other_query.alias_prefix: 

944 # No clashes between self and outer query should be possible. 

945 return 

946 

947 # Explicitly avoid infinite loop. The constant divider is based on how 

948 # much depth recursive subquery references add to the stack. This value 

949 # might need to be adjusted when adding or removing function calls from 

950 # the code path in charge of performing these operations. 

951 local_recursion_limit = sys.getrecursionlimit() // 16 

952 for pos, prefix in enumerate(prefix_gen()): 

953 if prefix not in self.subq_aliases: 

954 self.alias_prefix = prefix 

955 break 

956 if pos > local_recursion_limit: 

957 raise RecursionError( 

958 "Maximum recursion depth exceeded: too many subqueries." 

959 ) 

960 self.subq_aliases = self.subq_aliases.union([self.alias_prefix]) 

961 other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases) 

962 if exclude is None: 

963 exclude = {} 

964 self.change_aliases( 

965 { 

966 alias: "%s%d" % (self.alias_prefix, pos) 

967 for pos, alias in enumerate(self.alias_map) 

968 if alias not in exclude 

969 } 

970 ) 

971 

972 def get_initial_alias(self): 

973 """ 

974 Return the first alias for this query, after increasing its reference 

975 count. 

976 """ 

977 if self.alias_map: 

978 alias = self.base_table 

979 self.ref_alias(alias) 

980 elif self.model: 

981 alias = self.join(self.base_table_class(self.get_meta().db_table, None)) 

982 else: 

983 alias = None 

984 return alias 

985 

986 def count_active_tables(self): 

987 """ 

988 Return the number of tables in this query with a non-zero reference 

989 count. After execution, the reference counts are zeroed, so tables 

990 added in compiler will not be seen by this method. 

991 """ 

992 return len([1 for count in self.alias_refcount.values() if count]) 

993 

994 def join(self, join, reuse=None, reuse_with_filtered_relation=False): 

995 """ 

996 Return an alias for the 'join', either reusing an existing alias for 

997 that join or creating a new one. 'join' is either a base_table_class or 

998 join_class. 

999 

1000 The 'reuse' parameter can be either None which means all joins are 

1001 reusable, or it can be a set containing the aliases that can be reused. 

1002 

1003 The 'reuse_with_filtered_relation' parameter is used when computing 

1004 FilteredRelation instances. 

1005 

1006 A join is always created as LOUTER if the lhs alias is LOUTER to make 

1007 sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new 

1008 joins are created as LOUTER if the join is nullable. 

1009 """ 

1010 if reuse_with_filtered_relation and reuse: 

1011 reuse_aliases = [ 

1012 a for a, j in self.alias_map.items() if a in reuse and j.equals(join) 

1013 ] 

1014 else: 

1015 reuse_aliases = [ 

1016 a 

1017 for a, j in self.alias_map.items() 

1018 if (reuse is None or a in reuse) and j == join 

1019 ] 

1020 if reuse_aliases: 

1021 if join.table_alias in reuse_aliases: 

1022 reuse_alias = join.table_alias 

1023 else: 

1024 # Reuse the most recent alias of the joined table 

1025 # (a many-to-many relation may be joined multiple times). 

1026 reuse_alias = reuse_aliases[-1] 

1027 self.ref_alias(reuse_alias) 

1028 return reuse_alias 

1029 

1030 # No reuse is possible, so we need a new alias. 

1031 alias, _ = self.table_alias( 

1032 join.table_name, create=True, filtered_relation=join.filtered_relation 

1033 ) 

1034 if join.join_type: 

1035 if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable: 

1036 join_type = LOUTER 

1037 else: 

1038 join_type = INNER 

1039 join.join_type = join_type 

1040 join.table_alias = alias 

1041 self.alias_map[alias] = join 

1042 return alias 

1043 

1044 def join_parent_model(self, opts, model, alias, seen): 

1045 """ 

1046 Make sure the given 'model' is joined in the query. If 'model' isn't 

1047 a parent of 'opts' or if it is None this method is a no-op. 

1048 

1049 The 'alias' is the root alias for starting the join, 'seen' is a dict 

1050 of model -> alias of existing joins. It must also contain a mapping 

1051 of None -> some alias. This will be returned in the no-op case. 

1052 """ 

1053 if model in seen: 

1054 return seen[model] 

1055 chain = opts.get_base_chain(model) 

1056 if not chain: 

1057 return alias 

1058 curr_opts = opts 

1059 for int_model in chain: 

1060 if int_model in seen: 

1061 curr_opts = int_model._meta 

1062 alias = seen[int_model] 

1063 continue 

1064 # Proxy model have elements in base chain 

1065 # with no parents, assign the new options 

1066 # object and skip to the next base in that 

1067 # case 

1068 if not curr_opts.parents[int_model]: 

1069 curr_opts = int_model._meta 

1070 continue 

1071 link_field = curr_opts.get_ancestor_link(int_model) 

1072 join_info = self.setup_joins([link_field.name], curr_opts, alias) 

1073 curr_opts = int_model._meta 

1074 alias = seen[int_model] = join_info.joins[-1] 

1075 return alias or seen[None] 

1076 

1077 def check_alias(self, alias): 

1078 if FORBIDDEN_ALIAS_PATTERN.search(alias): 

1079 raise ValueError( 

1080 "Column aliases cannot contain whitespace characters, quotation marks, " 

1081 "semicolons, or SQL comments." 

1082 ) 

1083 

1084 def add_annotation(self, annotation, alias, select=True): 

1085 """Add a single annotation expression to the Query.""" 

1086 self.check_alias(alias) 

1087 annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None) 

1088 if select: 

1089 self.append_annotation_mask([alias]) 

1090 else: 

1091 self.set_annotation_mask(set(self.annotation_select).difference({alias})) 

1092 self.annotations[alias] = annotation 

1093 

1094 def resolve_expression(self, query, *args, **kwargs): 

1095 clone = self.clone() 

1096 # Subqueries need to use a different set of aliases than the outer query. 

1097 clone.bump_prefix(query) 

1098 clone.subquery = True 

1099 clone.where.resolve_expression(query, *args, **kwargs) 

1100 # Resolve combined queries. 

1101 if clone.combinator: 

1102 clone.combined_queries = tuple( 

1103 [ 

1104 combined_query.resolve_expression(query, *args, **kwargs) 

1105 for combined_query in clone.combined_queries 

1106 ] 

1107 ) 

1108 for key, value in clone.annotations.items(): 

1109 resolved = value.resolve_expression(query, *args, **kwargs) 

1110 if hasattr(resolved, "external_aliases"): 

1111 resolved.external_aliases.update(clone.external_aliases) 

1112 clone.annotations[key] = resolved 

1113 # Outer query's aliases are considered external. 

1114 for alias, table in query.alias_map.items(): 

1115 clone.external_aliases[alias] = ( 

1116 isinstance(table, Join) 

1117 and table.join_field.related_model._meta.db_table != alias 

1118 ) or ( 

1119 isinstance(table, BaseTable) and table.table_name != table.table_alias 

1120 ) 

1121 return clone 

1122 

1123 def get_external_cols(self): 

1124 exprs = chain(self.annotations.values(), self.where.children) 

1125 return [ 

1126 col 

1127 for col in self._gen_cols(exprs, include_external=True) 

1128 if col.alias in self.external_aliases 

1129 ] 

1130 

1131 def get_group_by_cols(self, wrapper=None): 

1132 # If wrapper is referenced by an alias for an explicit GROUP BY through 

1133 # values() a reference to this expression and not the self must be 

1134 # returned to ensure external column references are not grouped against 

1135 # as well. 

1136 external_cols = self.get_external_cols() 

1137 if any(col.possibly_multivalued for col in external_cols): 

1138 return [wrapper or self] 

1139 return external_cols 

1140 

1141 def as_sql(self, compiler, connection): 

1142 # Some backends (e.g. Oracle) raise an error when a subquery contains 

1143 # unnecessary ORDER BY clause. 

1144 if ( 

1145 self.subquery 

1146 and not connection.features.ignores_unnecessary_order_by_in_subqueries 

1147 ): 

1148 self.clear_ordering(force=False) 

1149 for query in self.combined_queries: 

1150 query.clear_ordering(force=False) 

1151 sql, params = self.get_compiler(connection=connection).as_sql() 

1152 if self.subquery: 

1153 sql = f"({sql})" 

1154 return sql, params 

1155 

1156 def resolve_lookup_value(self, value, can_reuse, allow_joins): 

1157 if hasattr(value, "resolve_expression"): 

1158 value = value.resolve_expression( 

1159 self, 

1160 reuse=can_reuse, 

1161 allow_joins=allow_joins, 

1162 ) 

1163 elif isinstance(value, list | tuple): 

1164 # The items of the iterable may be expressions and therefore need 

1165 # to be resolved independently. 

1166 values = ( 

1167 self.resolve_lookup_value(sub_value, can_reuse, allow_joins) 

1168 for sub_value in value 

1169 ) 

1170 type_ = type(value) 

1171 if hasattr(type_, "_make"): # namedtuple 

1172 return type_(*values) 

1173 return type_(values) 

1174 return value 

1175 

1176 def solve_lookup_type(self, lookup, summarize=False): 

1177 """ 

1178 Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains'). 

1179 """ 

1180 lookup_splitted = lookup.split(LOOKUP_SEP) 

1181 if self.annotations: 

1182 annotation, expression_lookups = refs_expression( 

1183 lookup_splitted, self.annotations 

1184 ) 

1185 if annotation: 

1186 expression = self.annotations[annotation] 

1187 if summarize: 

1188 expression = Ref(annotation, expression) 

1189 return expression_lookups, (), expression 

1190 _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta()) 

1191 field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)] 

1192 if len(lookup_parts) > 1 and not field_parts: 

1193 raise FieldError( 

1194 f'Invalid lookup "{lookup}" for model {self.get_meta().model.__name__}".' 

1195 ) 

1196 return lookup_parts, field_parts, False 

1197 

1198 def check_query_object_type(self, value, opts, field): 

1199 """ 

1200 Check whether the object passed while querying is of the correct type. 

1201 If not, raise a ValueError specifying the wrong object. 

1202 """ 

1203 if hasattr(value, "_meta"): 

1204 if not check_rel_lookup_compatibility(value._meta.model, opts, field): 

1205 raise ValueError( 

1206 f'Cannot query "{value}": Must be "{opts.object_name}" instance.' 

1207 ) 

1208 

1209 def check_related_objects(self, field, value, opts): 

1210 """Check the type of object passed to query relations.""" 

1211 if field.is_relation: 

1212 # Check that the field and the queryset use the same model in a 

1213 # query like .filter(author=Author.objects.all()). For example, the 

1214 # opts would be Author's (from the author field) and value.model 

1215 # would be Author.objects.all() queryset's .model (Author also). 

1216 # The field is the related field on the lhs side. 

1217 if ( 

1218 isinstance(value, Query) 

1219 and not value.has_select_fields 

1220 and not check_rel_lookup_compatibility(value.model, opts, field) 

1221 ): 

1222 raise ValueError( 

1223 f'Cannot use QuerySet for "{value.model._meta.object_name}": Use a QuerySet for "{opts.object_name}".' 

1224 ) 

1225 elif hasattr(value, "_meta"): 

1226 self.check_query_object_type(value, opts, field) 

1227 elif hasattr(value, "__iter__"): 

1228 for v in value: 

1229 self.check_query_object_type(v, opts, field) 

1230 

1231 def check_filterable(self, expression): 

1232 """Raise an error if expression cannot be used in a WHERE clause.""" 

1233 if hasattr(expression, "resolve_expression") and not getattr( 

1234 expression, "filterable", True 

1235 ): 

1236 raise NotSupportedError( 

1237 expression.__class__.__name__ + " is disallowed in the filter " 

1238 "clause." 

1239 ) 

1240 if hasattr(expression, "get_source_expressions"): 

1241 for expr in expression.get_source_expressions(): 

1242 self.check_filterable(expr) 

1243 

1244 def build_lookup(self, lookups, lhs, rhs): 

1245 """ 

1246 Try to extract transforms and lookup from given lhs. 

1247 

1248 The lhs value is something that works like SQLExpression. 

1249 The rhs value is what the lookup is going to compare against. 

1250 The lookups is a list of names to extract using get_lookup() 

1251 and get_transform(). 

1252 """ 

1253 # __exact is the default lookup if one isn't given. 

1254 *transforms, lookup_name = lookups or ["exact"] 

1255 for name in transforms: 

1256 lhs = self.try_transform(lhs, name) 

1257 # First try get_lookup() so that the lookup takes precedence if the lhs 

1258 # supports both transform and lookup for the name. 

1259 lookup_class = lhs.get_lookup(lookup_name) 

1260 if not lookup_class: 

1261 # A lookup wasn't found. Try to interpret the name as a transform 

1262 # and do an Exact lookup against it. 

1263 lhs = self.try_transform(lhs, lookup_name) 

1264 lookup_name = "exact" 

1265 lookup_class = lhs.get_lookup(lookup_name) 

1266 if not lookup_class: 

1267 return 

1268 

1269 lookup = lookup_class(lhs, rhs) 

1270 # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all 

1271 # uses of None as a query value unless the lookup supports it. 

1272 if lookup.rhs is None and not lookup.can_use_none_as_rhs: 

1273 if lookup_name not in ("exact", "iexact"): 

1274 raise ValueError("Cannot use None as a query value") 

1275 return lhs.get_lookup("isnull")(lhs, True) 

1276 

1277 # For Oracle '' is equivalent to null. The check must be done at this 

1278 # stage because join promotion can't be done in the compiler. Using 

1279 # DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here. 

1280 # A similar thing is done in is_nullable(), too. 

1281 if ( 

1282 lookup_name == "exact" 

1283 and lookup.rhs == "" 

1284 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls 

1285 ): 

1286 return lhs.get_lookup("isnull")(lhs, True) 

1287 

1288 return lookup 

1289 

1290 def try_transform(self, lhs, name): 

1291 """ 

1292 Helper method for build_lookup(). Try to fetch and initialize 

1293 a transform for name parameter from lhs. 

1294 """ 

1295 transform_class = lhs.get_transform(name) 

1296 if transform_class: 

1297 return transform_class(lhs) 

1298 else: 

1299 output_field = lhs.output_field.__class__ 

1300 suggested_lookups = difflib.get_close_matches( 

1301 name, output_field.get_lookups() 

1302 ) 

1303 if suggested_lookups: 

1304 suggestion = ", perhaps you meant {}?".format( 

1305 " or ".join(suggested_lookups) 

1306 ) 

1307 else: 

1308 suggestion = "." 

1309 raise FieldError( 

1310 f"Unsupported lookup '{name}' for {output_field.__name__} or join on the field not " 

1311 f"permitted{suggestion}" 

1312 ) 

1313 

1314 def build_filter( 

1315 self, 

1316 filter_expr, 

1317 branch_negated=False, 

1318 current_negated=False, 

1319 can_reuse=None, 

1320 allow_joins=True, 

1321 split_subq=True, 

1322 reuse_with_filtered_relation=False, 

1323 check_filterable=True, 

1324 summarize=False, 

1325 ): 

1326 """ 

1327 Build a WhereNode for a single filter clause but don't add it 

1328 to this Query. Query.add_q() will then add this filter to the where 

1329 Node. 

1330 

1331 The 'branch_negated' tells us if the current branch contains any 

1332 negations. This will be used to determine if subqueries are needed. 

1333 

1334 The 'current_negated' is used to determine if the current filter is 

1335 negated or not and this will be used to determine if IS NULL filtering 

1336 is needed. 

1337 

1338 The difference between current_negated and branch_negated is that 

1339 branch_negated is set on first negation, but current_negated is 

1340 flipped for each negation. 

1341 

1342 Note that add_filter will not do any negating itself, that is done 

1343 upper in the code by add_q(). 

1344 

1345 The 'can_reuse' is a set of reusable joins for multijoins. 

1346 

1347 If 'reuse_with_filtered_relation' is True, then only joins in can_reuse 

1348 will be reused. 

1349 

1350 The method will create a filter clause that can be added to the current 

1351 query. However, if the filter isn't added to the query then the caller 

1352 is responsible for unreffing the joins used. 

1353 """ 

1354 if isinstance(filter_expr, dict): 

1355 raise FieldError("Cannot parse keyword query as dict") 

1356 if isinstance(filter_expr, Q): 

1357 return self._add_q( 

1358 filter_expr, 

1359 branch_negated=branch_negated, 

1360 current_negated=current_negated, 

1361 used_aliases=can_reuse, 

1362 allow_joins=allow_joins, 

1363 split_subq=split_subq, 

1364 check_filterable=check_filterable, 

1365 summarize=summarize, 

1366 ) 

1367 if hasattr(filter_expr, "resolve_expression"): 

1368 if not getattr(filter_expr, "conditional", False): 

1369 raise TypeError("Cannot filter against a non-conditional expression.") 

1370 condition = filter_expr.resolve_expression( 

1371 self, allow_joins=allow_joins, summarize=summarize 

1372 ) 

1373 if not isinstance(condition, Lookup): 

1374 condition = self.build_lookup(["exact"], condition, True) 

1375 return WhereNode([condition], connector=AND), [] 

1376 arg, value = filter_expr 

1377 if not arg: 

1378 raise FieldError(f"Cannot parse keyword query {arg!r}") 

1379 lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize) 

1380 

1381 if check_filterable: 

1382 self.check_filterable(reffed_expression) 

1383 

1384 if not allow_joins and len(parts) > 1: 

1385 raise FieldError("Joined field references are not permitted in this query") 

1386 

1387 pre_joins = self.alias_refcount.copy() 

1388 value = self.resolve_lookup_value(value, can_reuse, allow_joins) 

1389 used_joins = { 

1390 k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0) 

1391 } 

1392 

1393 if check_filterable: 

1394 self.check_filterable(value) 

1395 

1396 if reffed_expression: 

1397 condition = self.build_lookup(lookups, reffed_expression, value) 

1398 return WhereNode([condition], connector=AND), [] 

1399 

1400 opts = self.get_meta() 

1401 alias = self.get_initial_alias() 

1402 allow_many = not branch_negated or not split_subq 

1403 

1404 try: 

1405 join_info = self.setup_joins( 

1406 parts, 

1407 opts, 

1408 alias, 

1409 can_reuse=can_reuse, 

1410 allow_many=allow_many, 

1411 reuse_with_filtered_relation=reuse_with_filtered_relation, 

1412 ) 

1413 

1414 # Prevent iterator from being consumed by check_related_objects() 

1415 if isinstance(value, Iterator): 

1416 value = list(value) 

1417 self.check_related_objects(join_info.final_field, value, join_info.opts) 

1418 

1419 # split_exclude() needs to know which joins were generated for the 

1420 # lookup parts 

1421 self._lookup_joins = join_info.joins 

1422 except MultiJoin as e: 

1423 return self.split_exclude(filter_expr, can_reuse, e.names_with_path) 

1424 

1425 # Update used_joins before trimming since they are reused to determine 

1426 # which joins could be later promoted to INNER. 

1427 used_joins.update(join_info.joins) 

1428 targets, alias, join_list = self.trim_joins( 

1429 join_info.targets, join_info.joins, join_info.path 

1430 ) 

1431 if can_reuse is not None: 

1432 can_reuse.update(join_list) 

1433 

1434 if join_info.final_field.is_relation: 

1435 if len(targets) == 1: 

1436 col = self._get_col(targets[0], join_info.final_field, alias) 

1437 else: 

1438 col = MultiColSource( 

1439 alias, targets, join_info.targets, join_info.final_field 

1440 ) 

1441 else: 

1442 col = self._get_col(targets[0], join_info.final_field, alias) 

1443 

1444 condition = self.build_lookup(lookups, col, value) 

1445 lookup_type = condition.lookup_name 

1446 clause = WhereNode([condition], connector=AND) 

1447 

1448 require_outer = ( 

1449 lookup_type == "isnull" and condition.rhs is True and not current_negated 

1450 ) 

1451 if ( 

1452 current_negated 

1453 and (lookup_type != "isnull" or condition.rhs is False) 

1454 and condition.rhs is not None 

1455 ): 

1456 require_outer = True 

1457 if lookup_type != "isnull": 

1458 # The condition added here will be SQL like this: 

1459 # NOT (col IS NOT NULL), where the first NOT is added in 

1460 # upper layers of code. The reason for addition is that if col 

1461 # is null, then col != someval will result in SQL "unknown" 

1462 # which isn't the same as in Python. The Python None handling 

1463 # is wanted, and it can be gotten by 

1464 # (col IS NULL OR col != someval) 

1465 # <=> 

1466 # NOT (col IS NOT NULL AND col = someval). 

1467 if ( 

1468 self.is_nullable(targets[0]) 

1469 or self.alias_map[join_list[-1]].join_type == LOUTER 

1470 ): 

1471 lookup_class = targets[0].get_lookup("isnull") 

1472 col = self._get_col(targets[0], join_info.targets[0], alias) 

1473 clause.add(lookup_class(col, False), AND) 

1474 # If someval is a nullable column, someval IS NOT NULL is 

1475 # added. 

1476 if isinstance(value, Col) and self.is_nullable(value.target): 

1477 lookup_class = value.target.get_lookup("isnull") 

1478 clause.add(lookup_class(value, False), AND) 

1479 return clause, used_joins if not require_outer else () 

1480 

1481 def add_filter(self, filter_lhs, filter_rhs): 

1482 self.add_q(Q((filter_lhs, filter_rhs))) 

1483 

1484 def add_q(self, q_object): 

1485 """ 

1486 A preprocessor for the internal _add_q(). Responsible for doing final 

1487 join promotion. 

1488 """ 

1489 # For join promotion this case is doing an AND for the added q_object 

1490 # and existing conditions. So, any existing inner join forces the join 

1491 # type to remain inner. Existing outer joins can however be demoted. 

1492 # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if 

1493 # rel_a doesn't produce any rows, then the whole condition must fail. 

1494 # So, demotion is OK. 

1495 existing_inner = { 

1496 a for a in self.alias_map if self.alias_map[a].join_type == INNER 

1497 } 

1498 clause, _ = self._add_q(q_object, self.used_aliases) 

1499 if clause: 

1500 self.where.add(clause, AND) 

1501 self.demote_joins(existing_inner) 

1502 

1503 def build_where(self, filter_expr): 

1504 return self.build_filter(filter_expr, allow_joins=False)[0] 

1505 

1506 def clear_where(self): 

1507 self.where = WhereNode() 

1508 

1509 def _add_q( 

1510 self, 

1511 q_object, 

1512 used_aliases, 

1513 branch_negated=False, 

1514 current_negated=False, 

1515 allow_joins=True, 

1516 split_subq=True, 

1517 check_filterable=True, 

1518 summarize=False, 

1519 ): 

1520 """Add a Q-object to the current filter.""" 

1521 connector = q_object.connector 

1522 current_negated ^= q_object.negated 

1523 branch_negated = branch_negated or q_object.negated 

1524 target_clause = WhereNode(connector=connector, negated=q_object.negated) 

1525 joinpromoter = JoinPromoter( 

1526 q_object.connector, len(q_object.children), current_negated 

1527 ) 

1528 for child in q_object.children: 

1529 child_clause, needed_inner = self.build_filter( 

1530 child, 

1531 can_reuse=used_aliases, 

1532 branch_negated=branch_negated, 

1533 current_negated=current_negated, 

1534 allow_joins=allow_joins, 

1535 split_subq=split_subq, 

1536 check_filterable=check_filterable, 

1537 summarize=summarize, 

1538 ) 

1539 joinpromoter.add_votes(needed_inner) 

1540 if child_clause: 

1541 target_clause.add(child_clause, connector) 

1542 needed_inner = joinpromoter.update_join_types(self) 

1543 return target_clause, needed_inner 

1544 

1545 def build_filtered_relation_q( 

1546 self, q_object, reuse, branch_negated=False, current_negated=False 

1547 ): 

1548 """Add a FilteredRelation object to the current filter.""" 

1549 connector = q_object.connector 

1550 current_negated ^= q_object.negated 

1551 branch_negated = branch_negated or q_object.negated 

1552 target_clause = WhereNode(connector=connector, negated=q_object.negated) 

1553 for child in q_object.children: 

1554 if isinstance(child, Node): 

1555 child_clause = self.build_filtered_relation_q( 

1556 child, 

1557 reuse=reuse, 

1558 branch_negated=branch_negated, 

1559 current_negated=current_negated, 

1560 ) 

1561 else: 

1562 child_clause, _ = self.build_filter( 

1563 child, 

1564 can_reuse=reuse, 

1565 branch_negated=branch_negated, 

1566 current_negated=current_negated, 

1567 allow_joins=True, 

1568 split_subq=False, 

1569 reuse_with_filtered_relation=True, 

1570 ) 

1571 target_clause.add(child_clause, connector) 

1572 return target_clause 

1573 

1574 def add_filtered_relation(self, filtered_relation, alias): 

1575 filtered_relation.alias = alias 

1576 lookups = dict(get_children_from_q(filtered_relation.condition)) 

1577 relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type( 

1578 filtered_relation.relation_name 

1579 ) 

1580 if relation_lookup_parts: 

1581 raise ValueError( 

1582 "FilteredRelation's relation_name cannot contain lookups " 

1583 f"(got {filtered_relation.relation_name!r})." 

1584 ) 

1585 for lookup in chain(lookups): 

1586 lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup) 

1587 shift = 2 if not lookup_parts else 1 

1588 lookup_field_path = lookup_field_parts[:-shift] 

1589 for idx, lookup_field_part in enumerate(lookup_field_path): 

1590 if len(relation_field_parts) > idx: 

1591 if relation_field_parts[idx] != lookup_field_part: 

1592 raise ValueError( 

1593 "FilteredRelation's condition doesn't support " 

1594 f"relations outside the {filtered_relation.relation_name!r} (got {lookup!r})." 

1595 ) 

1596 else: 

1597 raise ValueError( 

1598 "FilteredRelation's condition doesn't support nested " 

1599 f"relations deeper than the relation_name (got {lookup!r} for " 

1600 f"{filtered_relation.relation_name!r})." 

1601 ) 

1602 self._filtered_relations[filtered_relation.alias] = filtered_relation 

1603 

1604 def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False): 

1605 """ 

1606 Walk the list of names and turns them into PathInfo tuples. A single 

1607 name in 'names' can generate multiple PathInfos (m2m, for example). 

1608 

1609 'names' is the path of names to travel, 'opts' is the model Options we 

1610 start the name resolving from, 'allow_many' is as for setup_joins(). 

1611 If fail_on_missing is set to True, then a name that can't be resolved 

1612 will generate a FieldError. 

1613 

1614 Return a list of PathInfo tuples. In addition return the final field 

1615 (the last used join field) and target (which is a field guaranteed to 

1616 contain the same value as the final field). Finally, return those names 

1617 that weren't found (which are likely transforms and the final lookup). 

1618 """ 

1619 path, names_with_path = [], [] 

1620 for pos, name in enumerate(names): 

1621 cur_names_with_path = (name, []) 

1622 if name == "pk": 

1623 name = opts.pk.name 

1624 

1625 field = None 

1626 filtered_relation = None 

1627 try: 

1628 if opts is None: 

1629 raise FieldDoesNotExist 

1630 field = opts.get_field(name) 

1631 except FieldDoesNotExist: 

1632 if name in self.annotation_select: 

1633 field = self.annotation_select[name].output_field 

1634 elif name in self._filtered_relations and pos == 0: 

1635 filtered_relation = self._filtered_relations[name] 

1636 if LOOKUP_SEP in filtered_relation.relation_name: 

1637 parts = filtered_relation.relation_name.split(LOOKUP_SEP) 

1638 filtered_relation_path, field, _, _ = self.names_to_path( 

1639 parts, 

1640 opts, 

1641 allow_many, 

1642 fail_on_missing, 

1643 ) 

1644 path.extend(filtered_relation_path[:-1]) 

1645 else: 

1646 field = opts.get_field(filtered_relation.relation_name) 

1647 if field is not None: 

1648 # Fields that contain one-to-many relations with a generic 

1649 # model (like a GenericForeignKey) cannot generate reverse 

1650 # relations and therefore cannot be used for reverse querying. 

1651 if field.is_relation and not field.related_model: 

1652 raise FieldError( 

1653 f"Field {name!r} does not generate an automatic reverse " 

1654 "relation and therefore cannot be used for reverse " 

1655 "querying. If it is a GenericForeignKey, consider " 

1656 "adding a GenericRelation." 

1657 ) 

1658 try: 

1659 model = field.model._meta.concrete_model 

1660 except AttributeError: 

1661 # QuerySet.annotate() may introduce fields that aren't 

1662 # attached to a model. 

1663 model = None 

1664 else: 

1665 # We didn't find the current field, so move position back 

1666 # one step. 

1667 pos -= 1 

1668 if pos == -1 or fail_on_missing: 

1669 available = sorted( 

1670 [ 

1671 *get_field_names_from_opts(opts), 

1672 *self.annotation_select, 

1673 *self._filtered_relations, 

1674 ] 

1675 ) 

1676 raise FieldError( 

1677 "Cannot resolve keyword '{}' into field. " 

1678 "Choices are: {}".format(name, ", ".join(available)) 

1679 ) 

1680 break 

1681 # Check if we need any joins for concrete inheritance cases (the 

1682 # field lives in parent, but we are currently in one of its 

1683 # children) 

1684 if opts is not None and model is not opts.model: 

1685 path_to_parent = opts.get_path_to_parent(model) 

1686 if path_to_parent: 

1687 path.extend(path_to_parent) 

1688 cur_names_with_path[1].extend(path_to_parent) 

1689 opts = path_to_parent[-1].to_opts 

1690 if hasattr(field, "path_infos"): 

1691 if filtered_relation: 

1692 pathinfos = field.get_path_info(filtered_relation) 

1693 else: 

1694 pathinfos = field.path_infos 

1695 if not allow_many: 

1696 for inner_pos, p in enumerate(pathinfos): 

1697 if p.m2m: 

1698 cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1]) 

1699 names_with_path.append(cur_names_with_path) 

1700 raise MultiJoin(pos + 1, names_with_path) 

1701 last = pathinfos[-1] 

1702 path.extend(pathinfos) 

1703 final_field = last.join_field 

1704 opts = last.to_opts 

1705 targets = last.target_fields 

1706 cur_names_with_path[1].extend(pathinfos) 

1707 names_with_path.append(cur_names_with_path) 

1708 else: 

1709 # Local non-relational field. 

1710 final_field = field 

1711 targets = (field,) 

1712 if fail_on_missing and pos + 1 != len(names): 

1713 raise FieldError( 

1714 f"Cannot resolve keyword {names[pos + 1]!r} into field. Join on '{name}'" 

1715 " not permitted." 

1716 ) 

1717 break 

1718 return path, final_field, targets, names[pos + 1 :] 

1719 

1720 def setup_joins( 

1721 self, 

1722 names, 

1723 opts, 

1724 alias, 

1725 can_reuse=None, 

1726 allow_many=True, 

1727 reuse_with_filtered_relation=False, 

1728 ): 

1729 """ 

1730 Compute the necessary table joins for the passage through the fields 

1731 given in 'names'. 'opts' is the Options class for the current model 

1732 (which gives the table we are starting from), 'alias' is the alias for 

1733 the table to start the joining from. 

1734 

1735 The 'can_reuse' defines the reverse foreign key joins we can reuse. It 

1736 can be None in which case all joins are reusable or a set of aliases 

1737 that can be reused. Note that non-reverse foreign keys are always 

1738 reusable when using setup_joins(). 

1739 

1740 The 'reuse_with_filtered_relation' can be used to force 'can_reuse' 

1741 parameter and force the relation on the given connections. 

1742 

1743 If 'allow_many' is False, then any reverse foreign key seen will 

1744 generate a MultiJoin exception. 

1745 

1746 Return the final field involved in the joins, the target field (used 

1747 for any 'where' constraint), the final 'opts' value, the joins, the 

1748 field path traveled to generate the joins, and a transform function 

1749 that takes a field and alias and is equivalent to `field.get_col(alias)` 

1750 in the simple case but wraps field transforms if they were included in 

1751 names. 

1752 

1753 The target field is the field containing the concrete value. Final 

1754 field can be something different, for example foreign key pointing to 

1755 that value. Final field is needed for example in some value 

1756 conversions (convert 'obj' in fk__id=obj to pk val using the foreign 

1757 key field for example). 

1758 """ 

1759 joins = [alias] 

1760 # The transform can't be applied yet, as joins must be trimmed later. 

1761 # To avoid making every caller of this method look up transforms 

1762 # directly, compute transforms here and create a partial that converts 

1763 # fields to the appropriate wrapped version. 

1764 

1765 def final_transformer(field, alias): 

1766 if not self.alias_cols: 

1767 alias = None 

1768 return field.get_col(alias) 

1769 

1770 # Try resolving all the names as fields first. If there's an error, 

1771 # treat trailing names as lookups until a field can be resolved. 

1772 last_field_exception = None 

1773 for pivot in range(len(names), 0, -1): 

1774 try: 

1775 path, final_field, targets, rest = self.names_to_path( 

1776 names[:pivot], 

1777 opts, 

1778 allow_many, 

1779 fail_on_missing=True, 

1780 ) 

1781 except FieldError as exc: 

1782 if pivot == 1: 

1783 # The first item cannot be a lookup, so it's safe 

1784 # to raise the field error here. 

1785 raise 

1786 else: 

1787 last_field_exception = exc 

1788 else: 

1789 # The transforms are the remaining items that couldn't be 

1790 # resolved into fields. 

1791 transforms = names[pivot:] 

1792 break 

1793 for name in transforms: 

1794 

1795 def transform(field, alias, *, name, previous): 

1796 try: 

1797 wrapped = previous(field, alias) 

1798 return self.try_transform(wrapped, name) 

1799 except FieldError: 

1800 # FieldError is raised if the transform doesn't exist. 

1801 if isinstance(final_field, Field) and last_field_exception: 

1802 raise last_field_exception 

1803 else: 

1804 raise 

1805 

1806 final_transformer = functools.partial( 

1807 transform, name=name, previous=final_transformer 

1808 ) 

1809 final_transformer.has_transforms = True 

1810 # Then, add the path to the query's joins. Note that we can't trim 

1811 # joins at this stage - we will need the information about join type 

1812 # of the trimmed joins. 

1813 for join in path: 

1814 if join.filtered_relation: 

1815 filtered_relation = join.filtered_relation.clone() 

1816 table_alias = filtered_relation.alias 

1817 else: 

1818 filtered_relation = None 

1819 table_alias = None 

1820 opts = join.to_opts 

1821 if join.direct: 

1822 nullable = self.is_nullable(join.join_field) 

1823 else: 

1824 nullable = True 

1825 connection = self.join_class( 

1826 opts.db_table, 

1827 alias, 

1828 table_alias, 

1829 INNER, 

1830 join.join_field, 

1831 nullable, 

1832 filtered_relation=filtered_relation, 

1833 ) 

1834 reuse = can_reuse if join.m2m or reuse_with_filtered_relation else None 

1835 alias = self.join( 

1836 connection, 

1837 reuse=reuse, 

1838 reuse_with_filtered_relation=reuse_with_filtered_relation, 

1839 ) 

1840 joins.append(alias) 

1841 if filtered_relation: 

1842 filtered_relation.path = joins[:] 

1843 return JoinInfo(final_field, targets, opts, joins, path, final_transformer) 

1844 

1845 def trim_joins(self, targets, joins, path): 

1846 """ 

1847 The 'target' parameter is the final field being joined to, 'joins' 

1848 is the full list of join aliases. The 'path' contain the PathInfos 

1849 used to create the joins. 

1850 

1851 Return the final target field and table alias and the new active 

1852 joins. 

1853 

1854 Always trim any direct join if the target column is already in the 

1855 previous table. Can't trim reverse joins as it's unknown if there's 

1856 anything on the other side of the join. 

1857 """ 

1858 joins = joins[:] 

1859 for pos, info in enumerate(reversed(path)): 

1860 if len(joins) == 1 or not info.direct: 

1861 break 

1862 if info.filtered_relation: 

1863 break 

1864 join_targets = {t.column for t in info.join_field.foreign_related_fields} 

1865 cur_targets = {t.column for t in targets} 

1866 if not cur_targets.issubset(join_targets): 

1867 break 

1868 targets_dict = { 

1869 r[1].column: r[0] 

1870 for r in info.join_field.related_fields 

1871 if r[1].column in cur_targets 

1872 } 

1873 targets = tuple(targets_dict[t.column] for t in targets) 

1874 self.unref_alias(joins.pop()) 

1875 return targets, joins[-1], joins 

1876 

1877 @classmethod 

1878 def _gen_cols(cls, exprs, include_external=False, resolve_refs=True): 

1879 for expr in exprs: 

1880 if isinstance(expr, Col): 

1881 yield expr 

1882 elif include_external and callable( 

1883 getattr(expr, "get_external_cols", None) 

1884 ): 

1885 yield from expr.get_external_cols() 

1886 elif hasattr(expr, "get_source_expressions"): 

1887 if not resolve_refs and isinstance(expr, Ref): 

1888 continue 

1889 yield from cls._gen_cols( 

1890 expr.get_source_expressions(), 

1891 include_external=include_external, 

1892 resolve_refs=resolve_refs, 

1893 ) 

1894 

1895 @classmethod 

1896 def _gen_col_aliases(cls, exprs): 

1897 yield from (expr.alias for expr in cls._gen_cols(exprs)) 

1898 

1899 def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False): 

1900 annotation = self.annotations.get(name) 

1901 if annotation is not None: 

1902 if not allow_joins: 

1903 for alias in self._gen_col_aliases([annotation]): 

1904 if isinstance(self.alias_map[alias], Join): 

1905 raise FieldError( 

1906 "Joined field references are not permitted in this query" 

1907 ) 

1908 if summarize: 

1909 # Summarize currently means we are doing an aggregate() query 

1910 # which is executed as a wrapped subquery if any of the 

1911 # aggregate() elements reference an existing annotation. In 

1912 # that case we need to return a Ref to the subquery's annotation. 

1913 if name not in self.annotation_select: 

1914 raise FieldError( 

1915 f"Cannot aggregate over the '{name}' alias. Use annotate() " 

1916 "to promote it." 

1917 ) 

1918 return Ref(name, self.annotation_select[name]) 

1919 else: 

1920 return annotation 

1921 else: 

1922 field_list = name.split(LOOKUP_SEP) 

1923 annotation = self.annotations.get(field_list[0]) 

1924 if annotation is not None: 

1925 for transform in field_list[1:]: 

1926 annotation = self.try_transform(annotation, transform) 

1927 return annotation 

1928 join_info = self.setup_joins( 

1929 field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse 

1930 ) 

1931 targets, final_alias, join_list = self.trim_joins( 

1932 join_info.targets, join_info.joins, join_info.path 

1933 ) 

1934 if not allow_joins and len(join_list) > 1: 

1935 raise FieldError( 

1936 "Joined field references are not permitted in this query" 

1937 ) 

1938 if len(targets) > 1: 

1939 raise FieldError( 

1940 "Referencing multicolumn fields with F() objects isn't supported" 

1941 ) 

1942 # Verify that the last lookup in name is a field or a transform: 

1943 # transform_function() raises FieldError if not. 

1944 transform = join_info.transform_function(targets[0], final_alias) 

1945 if reuse is not None: 

1946 reuse.update(join_list) 

1947 return transform 

1948 

1949 def split_exclude(self, filter_expr, can_reuse, names_with_path): 

1950 """ 

1951 When doing an exclude against any kind of N-to-many relation, we need 

1952 to use a subquery. This method constructs the nested query, given the 

1953 original exclude filter (filter_expr) and the portion up to the first 

1954 N-to-many relation field. 

1955 

1956 For example, if the origin filter is ~Q(child__name='foo'), filter_expr 

1957 is ('child__name', 'foo') and can_reuse is a set of joins usable for 

1958 filters in the original query. 

1959 

1960 We will turn this into equivalent of: 

1961 WHERE NOT EXISTS( 

1962 SELECT 1 

1963 FROM child 

1964 WHERE name = 'foo' AND child.parent_id = parent.id 

1965 LIMIT 1 

1966 ) 

1967 """ 

1968 # Generate the inner query. 

1969 query = self.__class__(self.model) 

1970 query._filtered_relations = self._filtered_relations 

1971 filter_lhs, filter_rhs = filter_expr 

1972 if isinstance(filter_rhs, OuterRef): 

1973 filter_rhs = OuterRef(filter_rhs) 

1974 elif isinstance(filter_rhs, F): 

1975 filter_rhs = OuterRef(filter_rhs.name) 

1976 query.add_filter(filter_lhs, filter_rhs) 

1977 query.clear_ordering(force=True) 

1978 # Try to have as simple as possible subquery -> trim leading joins from 

1979 # the subquery. 

1980 trimmed_prefix, contains_louter = query.trim_start(names_with_path) 

1981 

1982 col = query.select[0] 

1983 select_field = col.target 

1984 alias = col.alias 

1985 if alias in can_reuse: 

1986 pk = select_field.model._meta.pk 

1987 # Need to add a restriction so that outer query's filters are in effect for 

1988 # the subquery, too. 

1989 query.bump_prefix(self) 

1990 lookup_class = select_field.get_lookup("exact") 

1991 # Note that the query.select[0].alias is different from alias 

1992 # due to bump_prefix above. 

1993 lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias)) 

1994 query.where.add(lookup, AND) 

1995 query.external_aliases[alias] = True 

1996 

1997 lookup_class = select_field.get_lookup("exact") 

1998 lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix)) 

1999 query.where.add(lookup, AND) 

2000 condition, needed_inner = self.build_filter(Exists(query)) 

2001 

2002 if contains_louter: 

2003 or_null_condition, _ = self.build_filter( 

2004 (f"{trimmed_prefix}__isnull", True), 

2005 current_negated=True, 

2006 branch_negated=True, 

2007 can_reuse=can_reuse, 

2008 ) 

2009 condition.add(or_null_condition, OR) 

2010 # Note that the end result will be: 

2011 # (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL. 

2012 # This might look crazy but due to how IN works, this seems to be 

2013 # correct. If the IS NOT NULL check is removed then outercol NOT 

2014 # IN will return UNKNOWN. If the IS NULL check is removed, then if 

2015 # outercol IS NULL we will not match the row. 

2016 return condition, needed_inner 

2017 

2018 def set_empty(self): 

2019 self.where.add(NothingNode(), AND) 

2020 for query in self.combined_queries: 

2021 query.set_empty() 

2022 

2023 def is_empty(self): 

2024 return any(isinstance(c, NothingNode) for c in self.where.children) 

2025 

2026 def set_limits(self, low=None, high=None): 

2027 """ 

2028 Adjust the limits on the rows retrieved. Use low/high to set these, 

2029 as it makes it more Pythonic to read and write. When the SQL query is 

2030 created, convert them to the appropriate offset and limit values. 

2031 

2032 Apply any limits passed in here to the existing constraints. Add low 

2033 to the current low value and clamp both to any existing high value. 

2034 """ 

2035 if high is not None: 

2036 if self.high_mark is not None: 

2037 self.high_mark = min(self.high_mark, self.low_mark + high) 

2038 else: 

2039 self.high_mark = self.low_mark + high 

2040 if low is not None: 

2041 if self.high_mark is not None: 

2042 self.low_mark = min(self.high_mark, self.low_mark + low) 

2043 else: 

2044 self.low_mark = self.low_mark + low 

2045 

2046 if self.low_mark == self.high_mark: 

2047 self.set_empty() 

2048 

2049 def clear_limits(self): 

2050 """Clear any existing limits.""" 

2051 self.low_mark, self.high_mark = 0, None 

2052 

2053 @property 

2054 def is_sliced(self): 

2055 return self.low_mark != 0 or self.high_mark is not None 

2056 

2057 def has_limit_one(self): 

2058 return self.high_mark is not None and (self.high_mark - self.low_mark) == 1 

2059 

2060 def can_filter(self): 

2061 """ 

2062 Return True if adding filters to this instance is still possible. 

2063 

2064 Typically, this means no limits or offsets have been put on the results. 

2065 """ 

2066 return not self.is_sliced 

2067 

2068 def clear_select_clause(self): 

2069 """Remove all fields from SELECT clause.""" 

2070 self.select = () 

2071 self.default_cols = False 

2072 self.select_related = False 

2073 self.set_extra_mask(()) 

2074 self.set_annotation_mask(()) 

2075 

2076 def clear_select_fields(self): 

2077 """ 

2078 Clear the list of fields to select (but not extra_select columns). 

2079 Some queryset types completely replace any existing list of select 

2080 columns. 

2081 """ 

2082 self.select = () 

2083 self.values_select = () 

2084 

2085 def add_select_col(self, col, name): 

2086 self.select += (col,) 

2087 self.values_select += (name,) 

2088 

2089 def set_select(self, cols): 

2090 self.default_cols = False 

2091 self.select = tuple(cols) 

2092 

2093 def add_distinct_fields(self, *field_names): 

2094 """ 

2095 Add and resolve the given fields to the query's "distinct on" clause. 

2096 """ 

2097 self.distinct_fields = field_names 

2098 self.distinct = True 

2099 

2100 def add_fields(self, field_names, allow_m2m=True): 

2101 """ 

2102 Add the given (model) fields to the select set. Add the field names in 

2103 the order specified. 

2104 """ 

2105 alias = self.get_initial_alias() 

2106 opts = self.get_meta() 

2107 

2108 try: 

2109 cols = [] 

2110 for name in field_names: 

2111 # Join promotion note - we must not remove any rows here, so 

2112 # if there is no existing joins, use outer join. 

2113 join_info = self.setup_joins( 

2114 name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m 

2115 ) 

2116 targets, final_alias, joins = self.trim_joins( 

2117 join_info.targets, 

2118 join_info.joins, 

2119 join_info.path, 

2120 ) 

2121 for target in targets: 

2122 cols.append(join_info.transform_function(target, final_alias)) 

2123 if cols: 

2124 self.set_select(cols) 

2125 except MultiJoin: 

2126 raise FieldError(f"Invalid field name: '{name}'") 

2127 except FieldError: 

2128 if LOOKUP_SEP in name: 

2129 # For lookups spanning over relationships, show the error 

2130 # from the model on which the lookup failed. 

2131 raise 

2132 elif name in self.annotations: 

2133 raise FieldError( 

2134 f"Cannot select the '{name}' alias. Use annotate() to promote " 

2135 "it." 

2136 ) 

2137 else: 

2138 names = sorted( 

2139 [ 

2140 *get_field_names_from_opts(opts), 

2141 *self.extra, 

2142 *self.annotation_select, 

2143 *self._filtered_relations, 

2144 ] 

2145 ) 

2146 raise FieldError( 

2147 "Cannot resolve keyword {!r} into field. Choices are: {}".format( 

2148 name, ", ".join(names) 

2149 ) 

2150 ) 

2151 

2152 def add_ordering(self, *ordering): 

2153 """ 

2154 Add items from the 'ordering' sequence to the query's "order by" 

2155 clause. These items are either field names (not column names) -- 

2156 possibly with a direction prefix ('-' or '?') -- or OrderBy 

2157 expressions. 

2158 

2159 If 'ordering' is empty, clear all ordering from the query. 

2160 """ 

2161 errors = [] 

2162 for item in ordering: 

2163 if isinstance(item, str): 

2164 if item == "?": 

2165 continue 

2166 item = item.removeprefix("-") 

2167 if item in self.annotations: 

2168 continue 

2169 if self.extra and item in self.extra: 

2170 continue 

2171 # names_to_path() validates the lookup. A descriptive 

2172 # FieldError will be raise if it's not. 

2173 self.names_to_path(item.split(LOOKUP_SEP), self.model._meta) 

2174 elif not hasattr(item, "resolve_expression"): 

2175 errors.append(item) 

2176 if getattr(item, "contains_aggregate", False): 

2177 raise FieldError( 

2178 "Using an aggregate in order_by() without also including " 

2179 f"it in annotate() is not allowed: {item}" 

2180 ) 

2181 if errors: 

2182 raise FieldError(f"Invalid order_by arguments: {errors}") 

2183 if ordering: 

2184 self.order_by += ordering 

2185 else: 

2186 self.default_ordering = False 

2187 

2188 def clear_ordering(self, force=False, clear_default=True): 

2189 """ 

2190 Remove any ordering settings if the current query allows it without 

2191 side effects, set 'force' to True to clear the ordering regardless. 

2192 If 'clear_default' is True, there will be no ordering in the resulting 

2193 query (not even the model's default). 

2194 """ 

2195 if not force and ( 

2196 self.is_sliced or self.distinct_fields or self.select_for_update 

2197 ): 

2198 return 

2199 self.order_by = () 

2200 self.extra_order_by = () 

2201 if clear_default: 

2202 self.default_ordering = False 

2203 

2204 def set_group_by(self, allow_aliases=True): 

2205 """ 

2206 Expand the GROUP BY clause required by the query. 

2207 

2208 This will usually be the set of all non-aggregate fields in the 

2209 return data. If the database backend supports grouping by the 

2210 primary key, and the query would be equivalent, the optimization 

2211 will be made automatically. 

2212 """ 

2213 if allow_aliases and self.values_select: 

2214 # If grouping by aliases is allowed assign selected value aliases 

2215 # by moving them to annotations. 

2216 group_by_annotations = {} 

2217 values_select = {} 

2218 for alias, expr in zip(self.values_select, self.select): 

2219 if isinstance(expr, Col): 

2220 values_select[alias] = expr 

2221 else: 

2222 group_by_annotations[alias] = expr 

2223 self.annotations = {**group_by_annotations, **self.annotations} 

2224 self.append_annotation_mask(group_by_annotations) 

2225 self.select = tuple(values_select.values()) 

2226 self.values_select = tuple(values_select) 

2227 group_by = list(self.select) 

2228 for alias, annotation in self.annotation_select.items(): 

2229 if not (group_by_cols := annotation.get_group_by_cols()): 

2230 continue 

2231 if allow_aliases and not annotation.contains_aggregate: 

2232 group_by.append(Ref(alias, annotation)) 

2233 else: 

2234 group_by.extend(group_by_cols) 

2235 self.group_by = tuple(group_by) 

2236 

2237 def add_select_related(self, fields): 

2238 """ 

2239 Set up the select_related data structure so that we only select 

2240 certain related models (as opposed to all models, when 

2241 self.select_related=True). 

2242 """ 

2243 if isinstance(self.select_related, bool): 

2244 field_dict = {} 

2245 else: 

2246 field_dict = self.select_related 

2247 for field in fields: 

2248 d = field_dict 

2249 for part in field.split(LOOKUP_SEP): 

2250 d = d.setdefault(part, {}) 

2251 self.select_related = field_dict 

2252 

2253 def add_extra(self, select, select_params, where, params, tables, order_by): 

2254 """ 

2255 Add data to the various extra_* attributes for user-created additions 

2256 to the query. 

2257 """ 

2258 if select: 

2259 # We need to pair any placeholder markers in the 'select' 

2260 # dictionary with their parameters in 'select_params' so that 

2261 # subsequent updates to the select dictionary also adjust the 

2262 # parameters appropriately. 

2263 select_pairs = {} 

2264 if select_params: 

2265 param_iter = iter(select_params) 

2266 else: 

2267 param_iter = iter([]) 

2268 for name, entry in select.items(): 

2269 self.check_alias(name) 

2270 entry = str(entry) 

2271 entry_params = [] 

2272 pos = entry.find("%s") 

2273 while pos != -1: 

2274 if pos == 0 or entry[pos - 1] != "%": 

2275 entry_params.append(next(param_iter)) 

2276 pos = entry.find("%s", pos + 2) 

2277 select_pairs[name] = (entry, entry_params) 

2278 self.extra.update(select_pairs) 

2279 if where or params: 

2280 self.where.add(ExtraWhere(where, params), AND) 

2281 if tables: 

2282 self.extra_tables += tuple(tables) 

2283 if order_by: 

2284 self.extra_order_by = order_by 

2285 

2286 def clear_deferred_loading(self): 

2287 """Remove any fields from the deferred loading set.""" 

2288 self.deferred_loading = (frozenset(), True) 

2289 

2290 def add_deferred_loading(self, field_names): 

2291 """ 

2292 Add the given list of model field names to the set of fields to 

2293 exclude from loading from the database when automatic column selection 

2294 is done. Add the new field names to any existing field names that 

2295 are deferred (or removed from any existing field names that are marked 

2296 as the only ones for immediate loading). 

2297 """ 

2298 # Fields on related models are stored in the literal double-underscore 

2299 # format, so that we can use a set datastructure. We do the foo__bar 

2300 # splitting and handling when computing the SQL column names (as part of 

2301 # get_columns()). 

2302 existing, defer = self.deferred_loading 

2303 if defer: 

2304 # Add to existing deferred names. 

2305 self.deferred_loading = existing.union(field_names), True 

2306 else: 

2307 # Remove names from the set of any existing "immediate load" names. 

2308 if new_existing := existing.difference(field_names): 

2309 self.deferred_loading = new_existing, False 

2310 else: 

2311 self.clear_deferred_loading() 

2312 if new_only := set(field_names).difference(existing): 

2313 self.deferred_loading = new_only, True 

2314 

2315 def add_immediate_loading(self, field_names): 

2316 """ 

2317 Add the given list of model field names to the set of fields to 

2318 retrieve when the SQL is executed ("immediate loading" fields). The 

2319 field names replace any existing immediate loading field names. If 

2320 there are field names already specified for deferred loading, remove 

2321 those names from the new field_names before storing the new names 

2322 for immediate loading. (That is, immediate loading overrides any 

2323 existing immediate values, but respects existing deferrals.) 

2324 """ 

2325 existing, defer = self.deferred_loading 

2326 field_names = set(field_names) 

2327 if "pk" in field_names: 

2328 field_names.remove("pk") 

2329 field_names.add(self.get_meta().pk.name) 

2330 

2331 if defer: 

2332 # Remove any existing deferred names from the current set before 

2333 # setting the new names. 

2334 self.deferred_loading = field_names.difference(existing), False 

2335 else: 

2336 # Replace any existing "immediate load" field names. 

2337 self.deferred_loading = frozenset(field_names), False 

2338 

2339 def set_annotation_mask(self, names): 

2340 """Set the mask of annotations that will be returned by the SELECT.""" 

2341 if names is None: 

2342 self.annotation_select_mask = None 

2343 else: 

2344 self.annotation_select_mask = set(names) 

2345 self._annotation_select_cache = None 

2346 

2347 def append_annotation_mask(self, names): 

2348 if self.annotation_select_mask is not None: 

2349 self.set_annotation_mask(self.annotation_select_mask.union(names)) 

2350 

2351 def set_extra_mask(self, names): 

2352 """ 

2353 Set the mask of extra select items that will be returned by SELECT. 

2354 Don't remove them from the Query since they might be used later. 

2355 """ 

2356 if names is None: 

2357 self.extra_select_mask = None 

2358 else: 

2359 self.extra_select_mask = set(names) 

2360 self._extra_select_cache = None 

2361 

2362 def set_values(self, fields): 

2363 self.select_related = False 

2364 self.clear_deferred_loading() 

2365 self.clear_select_fields() 

2366 self.has_select_fields = True 

2367 

2368 if fields: 

2369 field_names = [] 

2370 extra_names = [] 

2371 annotation_names = [] 

2372 if not self.extra and not self.annotations: 

2373 # Shortcut - if there are no extra or annotations, then 

2374 # the values() clause must be just field names. 

2375 field_names = list(fields) 

2376 else: 

2377 self.default_cols = False 

2378 for f in fields: 

2379 if f in self.extra_select: 

2380 extra_names.append(f) 

2381 elif f in self.annotation_select: 

2382 annotation_names.append(f) 

2383 else: 

2384 field_names.append(f) 

2385 self.set_extra_mask(extra_names) 

2386 self.set_annotation_mask(annotation_names) 

2387 selected = frozenset(field_names + extra_names + annotation_names) 

2388 else: 

2389 field_names = [f.attname for f in self.model._meta.concrete_fields] 

2390 selected = frozenset(field_names) 

2391 # Selected annotations must be known before setting the GROUP BY 

2392 # clause. 

2393 if self.group_by is True: 

2394 self.add_fields( 

2395 (f.attname for f in self.model._meta.concrete_fields), False 

2396 ) 

2397 # Disable GROUP BY aliases to avoid orphaning references to the 

2398 # SELECT clause which is about to be cleared. 

2399 self.set_group_by(allow_aliases=False) 

2400 self.clear_select_fields() 

2401 elif self.group_by: 

2402 # Resolve GROUP BY annotation references if they are not part of 

2403 # the selected fields anymore. 

2404 group_by = [] 

2405 for expr in self.group_by: 

2406 if isinstance(expr, Ref) and expr.refs not in selected: 

2407 expr = self.annotations[expr.refs] 

2408 group_by.append(expr) 

2409 self.group_by = tuple(group_by) 

2410 

2411 self.values_select = tuple(field_names) 

2412 self.add_fields(field_names, True) 

2413 

2414 @property 

2415 def annotation_select(self): 

2416 """ 

2417 Return the dictionary of aggregate columns that are not masked and 

2418 should be used in the SELECT clause. Cache this result for performance. 

2419 """ 

2420 if self._annotation_select_cache is not None: 

2421 return self._annotation_select_cache 

2422 elif not self.annotations: 

2423 return {} 

2424 elif self.annotation_select_mask is not None: 

2425 self._annotation_select_cache = { 

2426 k: v 

2427 for k, v in self.annotations.items() 

2428 if k in self.annotation_select_mask 

2429 } 

2430 return self._annotation_select_cache 

2431 else: 

2432 return self.annotations 

2433 

2434 @property 

2435 def extra_select(self): 

2436 if self._extra_select_cache is not None: 

2437 return self._extra_select_cache 

2438 if not self.extra: 

2439 return {} 

2440 elif self.extra_select_mask is not None: 

2441 self._extra_select_cache = { 

2442 k: v for k, v in self.extra.items() if k in self.extra_select_mask 

2443 } 

2444 return self._extra_select_cache 

2445 else: 

2446 return self.extra 

2447 

2448 def trim_start(self, names_with_path): 

2449 """ 

2450 Trim joins from the start of the join path. The candidates for trim 

2451 are the PathInfos in names_with_path structure that are m2m joins. 

2452 

2453 Also set the select column so the start matches the join. 

2454 

2455 This method is meant to be used for generating the subquery joins & 

2456 cols in split_exclude(). 

2457 

2458 Return a lookup usable for doing outerq.filter(lookup=self) and a 

2459 boolean indicating if the joins in the prefix contain a LEFT OUTER join. 

2460 _""" 

2461 all_paths = [] 

2462 for _, paths in names_with_path: 

2463 all_paths.extend(paths) 

2464 contains_louter = False 

2465 # Trim and operate only on tables that were generated for 

2466 # the lookup part of the query. That is, avoid trimming 

2467 # joins generated for F() expressions. 

2468 lookup_tables = [ 

2469 t for t in self.alias_map if t in self._lookup_joins or t == self.base_table 

2470 ] 

2471 for trimmed_paths, path in enumerate(all_paths): 

2472 if path.m2m: 

2473 break 

2474 if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER: 

2475 contains_louter = True 

2476 alias = lookup_tables[trimmed_paths] 

2477 self.unref_alias(alias) 

2478 # The path.join_field is a Rel, lets get the other side's field 

2479 join_field = path.join_field.field 

2480 # Build the filter prefix. 

2481 paths_in_prefix = trimmed_paths 

2482 trimmed_prefix = [] 

2483 for name, path in names_with_path: 

2484 if paths_in_prefix - len(path) < 0: 

2485 break 

2486 trimmed_prefix.append(name) 

2487 paths_in_prefix -= len(path) 

2488 trimmed_prefix.append(join_field.foreign_related_fields[0].name) 

2489 trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix) 

2490 # Lets still see if we can trim the first join from the inner query 

2491 # (that is, self). We can't do this for: 

2492 # - LEFT JOINs because we would miss those rows that have nothing on 

2493 # the outer side, 

2494 # - INNER JOINs from filtered relations because we would miss their 

2495 # filters. 

2496 first_join = self.alias_map[lookup_tables[trimmed_paths + 1]] 

2497 if first_join.join_type != LOUTER and not first_join.filtered_relation: 

2498 select_fields = [r[0] for r in join_field.related_fields] 

2499 select_alias = lookup_tables[trimmed_paths + 1] 

2500 self.unref_alias(lookup_tables[trimmed_paths]) 

2501 extra_restriction = join_field.get_extra_restriction( 

2502 None, lookup_tables[trimmed_paths + 1] 

2503 ) 

2504 if extra_restriction: 

2505 self.where.add(extra_restriction, AND) 

2506 else: 

2507 # TODO: It might be possible to trim more joins from the start of the 

2508 # inner query if it happens to have a longer join chain containing the 

2509 # values in select_fields. Lets punt this one for now. 

2510 select_fields = [r[1] for r in join_field.related_fields] 

2511 select_alias = lookup_tables[trimmed_paths] 

2512 # The found starting point is likely a join_class instead of a 

2513 # base_table_class reference. But the first entry in the query's FROM 

2514 # clause must not be a JOIN. 

2515 for table in self.alias_map: 

2516 if self.alias_refcount[table] > 0: 

2517 self.alias_map[table] = self.base_table_class( 

2518 self.alias_map[table].table_name, 

2519 table, 

2520 ) 

2521 break 

2522 self.set_select([f.get_col(select_alias) for f in select_fields]) 

2523 return trimmed_prefix, contains_louter 

2524 

2525 def is_nullable(self, field): 

2526 """ 

2527 Check if the given field should be treated as nullable. 

2528 

2529 Some backends treat '' as null and Plain treats such fields as 

2530 nullable for those backends. In such situations field.null can be 

2531 False even if we should treat the field as nullable. 

2532 """ 

2533 # We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have 

2534 # (nor should it have) knowledge of which connection is going to be 

2535 # used. The proper fix would be to defer all decisions where 

2536 # is_nullable() is needed to the compiler stage, but that is not easy 

2537 # to do currently. 

2538 return field.null or ( 

2539 field.empty_strings_allowed 

2540 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls 

2541 ) 

2542 

2543 

2544def get_order_dir(field, default="ASC"): 

2545 """ 

2546 Return the field name and direction for an order specification. For 

2547 example, '-foo' is returned as ('foo', 'DESC'). 

2548 

2549 The 'default' param is used to indicate which way no prefix (or a '+' 

2550 prefix) should sort. The '-' prefix always sorts the opposite way. 

2551 """ 

2552 dirn = ORDER_DIR[default] 

2553 if field[0] == "-": 

2554 return field[1:], dirn[1] 

2555 return field, dirn[0] 

2556 

2557 

2558class JoinPromoter: 

2559 """ 

2560 A class to abstract away join promotion problems for complex filter 

2561 conditions. 

2562 """ 

2563 

2564 def __init__(self, connector, num_children, negated): 

2565 self.connector = connector 

2566 self.negated = negated 

2567 if self.negated: 

2568 if connector == AND: 

2569 self.effective_connector = OR 

2570 else: 

2571 self.effective_connector = AND 

2572 else: 

2573 self.effective_connector = self.connector 

2574 self.num_children = num_children 

2575 # Maps of table alias to how many times it is seen as required for 

2576 # inner and/or outer joins. 

2577 self.votes = Counter() 

2578 

2579 def __repr__(self): 

2580 return ( 

2581 f"{self.__class__.__qualname__}(connector={self.connector!r}, " 

2582 f"num_children={self.num_children!r}, negated={self.negated!r})" 

2583 ) 

2584 

2585 def add_votes(self, votes): 

2586 """ 

2587 Add single vote per item to self.votes. Parameter can be any 

2588 iterable. 

2589 """ 

2590 self.votes.update(votes) 

2591 

2592 def update_join_types(self, query): 

2593 """ 

2594 Change join types so that the generated query is as efficient as 

2595 possible, but still correct. So, change as many joins as possible 

2596 to INNER, but don't make OUTER joins INNER if that could remove 

2597 results from the query. 

2598 """ 

2599 to_promote = set() 

2600 to_demote = set() 

2601 # The effective_connector is used so that NOT (a AND b) is treated 

2602 # similarly to (a OR b) for join promotion. 

2603 for table, votes in self.votes.items(): 

2604 # We must use outer joins in OR case when the join isn't contained 

2605 # in all of the joins. Otherwise the INNER JOIN itself could remove 

2606 # valid results. Consider the case where a model with rel_a and 

2607 # rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now, 

2608 # if rel_a join doesn't produce any results is null (for example 

2609 # reverse foreign key or null value in direct foreign key), and 

2610 # there is a matching row in rel_b with col=2, then an INNER join 

2611 # to rel_a would remove a valid match from the query. So, we need 

2612 # to promote any existing INNER to LOUTER (it is possible this 

2613 # promotion in turn will be demoted later on). 

2614 if self.effective_connector == OR and votes < self.num_children: 

2615 to_promote.add(table) 

2616 # If connector is AND and there is a filter that can match only 

2617 # when there is a joinable row, then use INNER. For example, in 

2618 # rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL 

2619 # as join output, then the col=1 or col=2 can't match (as 

2620 # NULL=anything is always false). 

2621 # For the OR case, if all children voted for a join to be inner, 

2622 # then we can use INNER for the join. For example: 

2623 # (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell) 

2624 # then if rel_a doesn't produce any rows, the whole condition 

2625 # can't match. Hence we can safely use INNER join. 

2626 if self.effective_connector == AND or ( 

2627 self.effective_connector == OR and votes == self.num_children 

2628 ): 

2629 to_demote.add(table) 

2630 # Finally, what happens in cases where we have: 

2631 # (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0 

2632 # Now, we first generate the OR clause, and promote joins for it 

2633 # in the first if branch above. Both rel_a and rel_b are promoted 

2634 # to LOUTER joins. After that we do the AND case. The OR case 

2635 # voted no inner joins but the rel_a__col__gte=0 votes inner join 

2636 # for rel_a. We demote it back to INNER join (in AND case a single 

2637 # vote is enough). The demotion is OK, if rel_a doesn't produce 

2638 # rows, then the rel_a__col__gte=0 clause can't be true, and thus 

2639 # the whole clause must be false. So, it is safe to use INNER 

2640 # join. 

2641 # Note that in this example we could just as well have the __gte 

2642 # clause and the OR clause swapped. Or we could replace the __gte 

2643 # clause with an OR clause containing rel_a__col=1|rel_a__col=2, 

2644 # and again we could safely demote to INNER. 

2645 query.promote_joins(to_promote) 

2646 query.demote_joins(to_demote) 

2647 return to_demote