Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# orm/unitofwork.py 

2# Copyright (C) 2005-2020 the SQLAlchemy authors and contributors 

3# <see AUTHORS file> 

4# 

5# This module is part of SQLAlchemy and is released under 

6# the MIT License: http://www.opensource.org/licenses/mit-license.php 

7 

8"""The internals for the unit of work system. 

9 

10The session's flush() process passes objects to a contextual object 

11here, which assembles flush tasks based on mappers and their properties, 

12organizes them in order of dependency, and executes. 

13 

14""" 

15 

16from . import attributes 

17from . import exc as orm_exc 

18from . import persistence 

19from . import util as orm_util 

20from .. import event 

21from .. import util 

22from ..util import topological 

23 

24 

25def track_cascade_events(descriptor, prop): 

26 """Establish event listeners on object attributes which handle 

27 cascade-on-set/append. 

28 

29 """ 

30 key = prop.key 

31 

32 def append(state, item, initiator): 

33 # process "save_update" cascade rules for when 

34 # an instance is appended to the list of another instance 

35 

36 if item is None: 

37 return 

38 

39 sess = state.session 

40 if sess: 

41 if sess._warn_on_events: 

42 sess._flush_warning("collection append") 

43 

44 prop = state.manager.mapper._props[key] 

45 item_state = attributes.instance_state(item) 

46 if ( 

47 prop._cascade.save_update 

48 and (prop.cascade_backrefs or key == initiator.key) 

49 and not sess._contains_state(item_state) 

50 ): 

51 sess._save_or_update_state(item_state) 

52 return item 

53 

54 def remove(state, item, initiator): 

55 if item is None: 

56 return 

57 

58 sess = state.session 

59 

60 prop = state.manager.mapper._props[key] 

61 

62 if sess and sess._warn_on_events: 

63 sess._flush_warning( 

64 "collection remove" 

65 if prop.uselist 

66 else "related attribute delete" 

67 ) 

68 

69 if ( 

70 item is not None 

71 and item is not attributes.NEVER_SET 

72 and item is not attributes.PASSIVE_NO_RESULT 

73 and prop._cascade.delete_orphan 

74 ): 

75 # expunge pending orphans 

76 item_state = attributes.instance_state(item) 

77 

78 if prop.mapper._is_orphan(item_state): 

79 if sess and item_state in sess._new: 

80 sess.expunge(item) 

81 else: 

82 # the related item may or may not itself be in a 

83 # Session, however the parent for which we are catching 

84 # the event is not in a session, so memoize this on the 

85 # item 

86 item_state._orphaned_outside_of_session = True 

87 

88 def set_(state, newvalue, oldvalue, initiator): 

89 # process "save_update" cascade rules for when an instance 

90 # is attached to another instance 

91 if oldvalue is newvalue: 

92 return newvalue 

93 

94 sess = state.session 

95 if sess: 

96 

97 if sess._warn_on_events: 

98 sess._flush_warning("related attribute set") 

99 

100 prop = state.manager.mapper._props[key] 

101 if newvalue is not None: 

102 newvalue_state = attributes.instance_state(newvalue) 

103 if ( 

104 prop._cascade.save_update 

105 and (prop.cascade_backrefs or key == initiator.key) 

106 and not sess._contains_state(newvalue_state) 

107 ): 

108 sess._save_or_update_state(newvalue_state) 

109 

110 if ( 

111 oldvalue is not None 

112 and oldvalue is not attributes.NEVER_SET 

113 and oldvalue is not attributes.PASSIVE_NO_RESULT 

114 and prop._cascade.delete_orphan 

115 ): 

116 # possible to reach here with attributes.NEVER_SET ? 

117 oldvalue_state = attributes.instance_state(oldvalue) 

118 

119 if oldvalue_state in sess._new and prop.mapper._is_orphan( 

120 oldvalue_state 

121 ): 

122 sess.expunge(oldvalue) 

123 return newvalue 

124 

125 event.listen(descriptor, "append", append, raw=True, retval=True) 

126 event.listen(descriptor, "remove", remove, raw=True, retval=True) 

127 event.listen(descriptor, "set", set_, raw=True, retval=True) 

128 

129 

130class UOWTransaction(object): 

131 def __init__(self, session): 

132 self.session = session 

133 

134 # dictionary used by external actors to 

135 # store arbitrary state information. 

136 self.attributes = {} 

137 

138 # dictionary of mappers to sets of 

139 # DependencyProcessors, which are also 

140 # set to be part of the sorted flush actions, 

141 # which have that mapper as a parent. 

142 self.deps = util.defaultdict(set) 

143 

144 # dictionary of mappers to sets of InstanceState 

145 # items pending for flush which have that mapper 

146 # as a parent. 

147 self.mappers = util.defaultdict(set) 

148 

149 # a dictionary of Preprocess objects, which gather 

150 # additional states impacted by the flush 

151 # and determine if a flush action is needed 

152 self.presort_actions = {} 

153 

154 # dictionary of PostSortRec objects, each 

155 # one issues work during the flush within 

156 # a certain ordering. 

157 self.postsort_actions = {} 

158 

159 # a set of 2-tuples, each containing two 

160 # PostSortRec objects where the second 

161 # is dependent on the first being executed 

162 # first 

163 self.dependencies = set() 

164 

165 # dictionary of InstanceState-> (isdelete, listonly) 

166 # tuples, indicating if this state is to be deleted 

167 # or insert/updated, or just refreshed 

168 self.states = {} 

169 

170 # tracks InstanceStates which will be receiving 

171 # a "post update" call. Keys are mappers, 

172 # values are a set of states and a set of the 

173 # columns which should be included in the update. 

174 self.post_update_states = util.defaultdict(lambda: (set(), set())) 

175 

176 @property 

177 def has_work(self): 

178 return bool(self.states) 

179 

180 def was_already_deleted(self, state): 

181 """return true if the given state is expired and was deleted 

182 previously. 

183 """ 

184 if state.expired: 

185 try: 

186 state._load_expired(state, attributes.PASSIVE_OFF) 

187 except orm_exc.ObjectDeletedError: 

188 self.session._remove_newly_deleted([state]) 

189 return True 

190 return False 

191 

192 def is_deleted(self, state): 

193 """return true if the given state is marked as deleted 

194 within this uowtransaction.""" 

195 

196 return state in self.states and self.states[state][0] 

197 

198 def memo(self, key, callable_): 

199 if key in self.attributes: 

200 return self.attributes[key] 

201 else: 

202 self.attributes[key] = ret = callable_() 

203 return ret 

204 

205 def remove_state_actions(self, state): 

206 """remove pending actions for a state from the uowtransaction.""" 

207 

208 isdelete = self.states[state][0] 

209 

210 self.states[state] = (isdelete, True) 

211 

212 def get_attribute_history( 

213 self, state, key, passive=attributes.PASSIVE_NO_INITIALIZE 

214 ): 

215 """facade to attributes.get_state_history(), including 

216 caching of results.""" 

217 

218 hashkey = ("history", state, key) 

219 

220 # cache the objects, not the states; the strong reference here 

221 # prevents newly loaded objects from being dereferenced during the 

222 # flush process 

223 

224 if hashkey in self.attributes: 

225 history, state_history, cached_passive = self.attributes[hashkey] 

226 # if the cached lookup was "passive" and now 

227 # we want non-passive, do a non-passive lookup and re-cache 

228 

229 if ( 

230 not cached_passive & attributes.SQL_OK 

231 and passive & attributes.SQL_OK 

232 ): 

233 impl = state.manager[key].impl 

234 history = impl.get_history( 

235 state, 

236 state.dict, 

237 attributes.PASSIVE_OFF | attributes.LOAD_AGAINST_COMMITTED, 

238 ) 

239 if history and impl.uses_objects: 

240 state_history = history.as_state() 

241 else: 

242 state_history = history 

243 self.attributes[hashkey] = (history, state_history, passive) 

244 else: 

245 impl = state.manager[key].impl 

246 # TODO: store the history as (state, object) tuples 

247 # so we don't have to keep converting here 

248 history = impl.get_history( 

249 state, state.dict, passive | attributes.LOAD_AGAINST_COMMITTED 

250 ) 

251 if history and impl.uses_objects: 

252 state_history = history.as_state() 

253 else: 

254 state_history = history 

255 self.attributes[hashkey] = (history, state_history, passive) 

256 

257 return state_history 

258 

259 def has_dep(self, processor): 

260 return (processor, True) in self.presort_actions 

261 

262 def register_preprocessor(self, processor, fromparent): 

263 key = (processor, fromparent) 

264 if key not in self.presort_actions: 

265 self.presort_actions[key] = Preprocess(processor, fromparent) 

266 

267 def register_object( 

268 self, 

269 state, 

270 isdelete=False, 

271 listonly=False, 

272 cancel_delete=False, 

273 operation=None, 

274 prop=None, 

275 ): 

276 if not self.session._contains_state(state): 

277 # this condition is normal when objects are registered 

278 # as part of a relationship cascade operation. it should 

279 # not occur for the top-level register from Session.flush(). 

280 if not state.deleted and operation is not None: 

281 util.warn( 

282 "Object of type %s not in session, %s operation " 

283 "along '%s' will not proceed" 

284 % (orm_util.state_class_str(state), operation, prop) 

285 ) 

286 return False 

287 

288 if state not in self.states: 

289 mapper = state.manager.mapper 

290 

291 if mapper not in self.mappers: 

292 self._per_mapper_flush_actions(mapper) 

293 

294 self.mappers[mapper].add(state) 

295 self.states[state] = (isdelete, listonly) 

296 else: 

297 if not listonly and (isdelete or cancel_delete): 

298 self.states[state] = (isdelete, False) 

299 return True 

300 

301 def register_post_update(self, state, post_update_cols): 

302 mapper = state.manager.mapper.base_mapper 

303 states, cols = self.post_update_states[mapper] 

304 states.add(state) 

305 cols.update(post_update_cols) 

306 

307 def _per_mapper_flush_actions(self, mapper): 

308 saves = SaveUpdateAll(self, mapper.base_mapper) 

309 deletes = DeleteAll(self, mapper.base_mapper) 

310 self.dependencies.add((saves, deletes)) 

311 

312 for dep in mapper._dependency_processors: 

313 dep.per_property_preprocessors(self) 

314 

315 for prop in mapper.relationships: 

316 if prop.viewonly: 

317 continue 

318 dep = prop._dependency_processor 

319 dep.per_property_preprocessors(self) 

320 

321 @util.memoized_property 

322 def _mapper_for_dep(self): 

323 """return a dynamic mapping of (Mapper, DependencyProcessor) to 

324 True or False, indicating if the DependencyProcessor operates 

325 on objects of that Mapper. 

326 

327 The result is stored in the dictionary persistently once 

328 calculated. 

329 

330 """ 

331 return util.PopulateDict( 

332 lambda tup: tup[0]._props.get(tup[1].key) is tup[1].prop 

333 ) 

334 

335 def filter_states_for_dep(self, dep, states): 

336 """Filter the given list of InstanceStates to those relevant to the 

337 given DependencyProcessor. 

338 

339 """ 

340 mapper_for_dep = self._mapper_for_dep 

341 return [s for s in states if mapper_for_dep[(s.manager.mapper, dep)]] 

342 

343 def states_for_mapper_hierarchy(self, mapper, isdelete, listonly): 

344 checktup = (isdelete, listonly) 

345 for mapper in mapper.base_mapper.self_and_descendants: 

346 for state in self.mappers[mapper]: 

347 if self.states[state] == checktup: 

348 yield state 

349 

350 def _generate_actions(self): 

351 """Generate the full, unsorted collection of PostSortRecs as 

352 well as dependency pairs for this UOWTransaction. 

353 

354 """ 

355 # execute presort_actions, until all states 

356 # have been processed. a presort_action might 

357 # add new states to the uow. 

358 while True: 

359 ret = False 

360 for action in list(self.presort_actions.values()): 

361 if action.execute(self): 

362 ret = True 

363 if not ret: 

364 break 

365 

366 # see if the graph of mapper dependencies has cycles. 

367 self.cycles = cycles = topological.find_cycles( 

368 self.dependencies, list(self.postsort_actions.values()) 

369 ) 

370 

371 if cycles: 

372 # if yes, break the per-mapper actions into 

373 # per-state actions 

374 convert = dict( 

375 (rec, set(rec.per_state_flush_actions(self))) for rec in cycles 

376 ) 

377 

378 # rewrite the existing dependencies to point to 

379 # the per-state actions for those per-mapper actions 

380 # that were broken up. 

381 for edge in list(self.dependencies): 

382 if ( 

383 None in edge 

384 or edge[0].disabled 

385 or edge[1].disabled 

386 or cycles.issuperset(edge) 

387 ): 

388 self.dependencies.remove(edge) 

389 elif edge[0] in cycles: 

390 self.dependencies.remove(edge) 

391 for dep in convert[edge[0]]: 

392 self.dependencies.add((dep, edge[1])) 

393 elif edge[1] in cycles: 

394 self.dependencies.remove(edge) 

395 for dep in convert[edge[1]]: 

396 self.dependencies.add((edge[0], dep)) 

397 

398 return set( 

399 [a for a in self.postsort_actions.values() if not a.disabled] 

400 ).difference(cycles) 

401 

402 def execute(self): 

403 postsort_actions = self._generate_actions() 

404 

405 # sort = topological.sort(self.dependencies, postsort_actions) 

406 # print "--------------" 

407 # print "\ndependencies:", self.dependencies 

408 # print "\ncycles:", self.cycles 

409 # print "\nsort:", list(sort) 

410 # print "\nCOUNT OF POSTSORT ACTIONS", len(postsort_actions) 

411 

412 # execute 

413 if self.cycles: 

414 for set_ in topological.sort_as_subsets( 

415 self.dependencies, postsort_actions 

416 ): 

417 while set_: 

418 n = set_.pop() 

419 n.execute_aggregate(self, set_) 

420 else: 

421 for rec in topological.sort(self.dependencies, postsort_actions): 

422 rec.execute(self) 

423 

424 def finalize_flush_changes(self): 

425 """mark processed objects as clean / deleted after a successful 

426 flush(). 

427 

428 this method is called within the flush() method after the 

429 execute() method has succeeded and the transaction has been committed. 

430 

431 """ 

432 if not self.states: 

433 return 

434 

435 states = set(self.states) 

436 isdel = set( 

437 s for (s, (isdelete, listonly)) in self.states.items() if isdelete 

438 ) 

439 other = states.difference(isdel) 

440 if isdel: 

441 self.session._remove_newly_deleted(isdel) 

442 if other: 

443 self.session._register_persistent(other) 

444 

445 

446class IterateMappersMixin(object): 

447 def _mappers(self, uow): 

448 if self.fromparent: 

449 return iter( 

450 m 

451 for m in self.dependency_processor.parent.self_and_descendants 

452 if uow._mapper_for_dep[(m, self.dependency_processor)] 

453 ) 

454 else: 

455 return self.dependency_processor.mapper.self_and_descendants 

456 

457 

458class Preprocess(IterateMappersMixin): 

459 __slots__ = ( 

460 "dependency_processor", 

461 "fromparent", 

462 "processed", 

463 "setup_flush_actions", 

464 ) 

465 

466 def __init__(self, dependency_processor, fromparent): 

467 self.dependency_processor = dependency_processor 

468 self.fromparent = fromparent 

469 self.processed = set() 

470 self.setup_flush_actions = False 

471 

472 def execute(self, uow): 

473 delete_states = set() 

474 save_states = set() 

475 

476 for mapper in self._mappers(uow): 

477 for state in uow.mappers[mapper].difference(self.processed): 

478 (isdelete, listonly) = uow.states[state] 

479 if not listonly: 

480 if isdelete: 

481 delete_states.add(state) 

482 else: 

483 save_states.add(state) 

484 

485 if delete_states: 

486 self.dependency_processor.presort_deletes(uow, delete_states) 

487 self.processed.update(delete_states) 

488 if save_states: 

489 self.dependency_processor.presort_saves(uow, save_states) 

490 self.processed.update(save_states) 

491 

492 if delete_states or save_states: 

493 if not self.setup_flush_actions and ( 

494 self.dependency_processor.prop_has_changes( 

495 uow, delete_states, True 

496 ) 

497 or self.dependency_processor.prop_has_changes( 

498 uow, save_states, False 

499 ) 

500 ): 

501 self.dependency_processor.per_property_flush_actions(uow) 

502 self.setup_flush_actions = True 

503 return True 

504 else: 

505 return False 

506 

507 

508class PostSortRec(object): 

509 __slots__ = ("disabled",) 

510 

511 def __new__(cls, uow, *args): 

512 key = (cls,) + args 

513 if key in uow.postsort_actions: 

514 return uow.postsort_actions[key] 

515 else: 

516 uow.postsort_actions[key] = ret = object.__new__(cls) 

517 ret.disabled = False 

518 return ret 

519 

520 def execute_aggregate(self, uow, recs): 

521 self.execute(uow) 

522 

523 

524class ProcessAll(IterateMappersMixin, PostSortRec): 

525 __slots__ = "dependency_processor", "isdelete", "fromparent" 

526 

527 def __init__(self, uow, dependency_processor, isdelete, fromparent): 

528 self.dependency_processor = dependency_processor 

529 self.isdelete = isdelete 

530 self.fromparent = fromparent 

531 uow.deps[dependency_processor.parent.base_mapper].add( 

532 dependency_processor 

533 ) 

534 

535 def execute(self, uow): 

536 states = self._elements(uow) 

537 if self.isdelete: 

538 self.dependency_processor.process_deletes(uow, states) 

539 else: 

540 self.dependency_processor.process_saves(uow, states) 

541 

542 def per_state_flush_actions(self, uow): 

543 # this is handled by SaveUpdateAll and DeleteAll, 

544 # since a ProcessAll should unconditionally be pulled 

545 # into per-state if either the parent/child mappers 

546 # are part of a cycle 

547 return iter([]) 

548 

549 def __repr__(self): 

550 return "%s(%s, isdelete=%s)" % ( 

551 self.__class__.__name__, 

552 self.dependency_processor, 

553 self.isdelete, 

554 ) 

555 

556 def _elements(self, uow): 

557 for mapper in self._mappers(uow): 

558 for state in uow.mappers[mapper]: 

559 (isdelete, listonly) = uow.states[state] 

560 if isdelete == self.isdelete and not listonly: 

561 yield state 

562 

563 

564class PostUpdateAll(PostSortRec): 

565 __slots__ = "mapper", "isdelete" 

566 

567 def __init__(self, uow, mapper, isdelete): 

568 self.mapper = mapper 

569 self.isdelete = isdelete 

570 

571 def execute(self, uow): 

572 states, cols = uow.post_update_states[self.mapper] 

573 states = [s for s in states if uow.states[s][0] == self.isdelete] 

574 

575 persistence.post_update(self.mapper, states, uow, cols) 

576 

577 

578class SaveUpdateAll(PostSortRec): 

579 __slots__ = ("mapper",) 

580 

581 def __init__(self, uow, mapper): 

582 self.mapper = mapper 

583 assert mapper is mapper.base_mapper 

584 

585 def execute(self, uow): 

586 persistence.save_obj( 

587 self.mapper, 

588 uow.states_for_mapper_hierarchy(self.mapper, False, False), 

589 uow, 

590 ) 

591 

592 def per_state_flush_actions(self, uow): 

593 states = list( 

594 uow.states_for_mapper_hierarchy(self.mapper, False, False) 

595 ) 

596 base_mapper = self.mapper.base_mapper 

597 delete_all = DeleteAll(uow, base_mapper) 

598 for state in states: 

599 # keep saves before deletes - 

600 # this ensures 'row switch' operations work 

601 action = SaveUpdateState(uow, state) 

602 uow.dependencies.add((action, delete_all)) 

603 yield action 

604 

605 for dep in uow.deps[self.mapper]: 

606 states_for_prop = uow.filter_states_for_dep(dep, states) 

607 dep.per_state_flush_actions(uow, states_for_prop, False) 

608 

609 def __repr__(self): 

610 return "%s(%s)" % (self.__class__.__name__, self.mapper) 

611 

612 

613class DeleteAll(PostSortRec): 

614 __slots__ = ("mapper",) 

615 

616 def __init__(self, uow, mapper): 

617 self.mapper = mapper 

618 assert mapper is mapper.base_mapper 

619 

620 def execute(self, uow): 

621 persistence.delete_obj( 

622 self.mapper, 

623 uow.states_for_mapper_hierarchy(self.mapper, True, False), 

624 uow, 

625 ) 

626 

627 def per_state_flush_actions(self, uow): 

628 states = list( 

629 uow.states_for_mapper_hierarchy(self.mapper, True, False) 

630 ) 

631 base_mapper = self.mapper.base_mapper 

632 save_all = SaveUpdateAll(uow, base_mapper) 

633 for state in states: 

634 # keep saves before deletes - 

635 # this ensures 'row switch' operations work 

636 action = DeleteState(uow, state) 

637 uow.dependencies.add((save_all, action)) 

638 yield action 

639 

640 for dep in uow.deps[self.mapper]: 

641 states_for_prop = uow.filter_states_for_dep(dep, states) 

642 dep.per_state_flush_actions(uow, states_for_prop, True) 

643 

644 def __repr__(self): 

645 return "%s(%s)" % (self.__class__.__name__, self.mapper) 

646 

647 

648class ProcessState(PostSortRec): 

649 __slots__ = "dependency_processor", "isdelete", "state" 

650 

651 def __init__(self, uow, dependency_processor, isdelete, state): 

652 self.dependency_processor = dependency_processor 

653 self.isdelete = isdelete 

654 self.state = state 

655 

656 def execute_aggregate(self, uow, recs): 

657 cls_ = self.__class__ 

658 dependency_processor = self.dependency_processor 

659 isdelete = self.isdelete 

660 our_recs = [ 

661 r 

662 for r in recs 

663 if r.__class__ is cls_ 

664 and r.dependency_processor is dependency_processor 

665 and r.isdelete is isdelete 

666 ] 

667 recs.difference_update(our_recs) 

668 states = [self.state] + [r.state for r in our_recs] 

669 if isdelete: 

670 dependency_processor.process_deletes(uow, states) 

671 else: 

672 dependency_processor.process_saves(uow, states) 

673 

674 def __repr__(self): 

675 return "%s(%s, %s, delete=%s)" % ( 

676 self.__class__.__name__, 

677 self.dependency_processor, 

678 orm_util.state_str(self.state), 

679 self.isdelete, 

680 ) 

681 

682 

683class SaveUpdateState(PostSortRec): 

684 __slots__ = "state", "mapper" 

685 

686 def __init__(self, uow, state): 

687 self.state = state 

688 self.mapper = state.mapper.base_mapper 

689 

690 def execute_aggregate(self, uow, recs): 

691 cls_ = self.__class__ 

692 mapper = self.mapper 

693 our_recs = [ 

694 r for r in recs if r.__class__ is cls_ and r.mapper is mapper 

695 ] 

696 recs.difference_update(our_recs) 

697 persistence.save_obj( 

698 mapper, [self.state] + [r.state for r in our_recs], uow 

699 ) 

700 

701 def __repr__(self): 

702 return "%s(%s)" % ( 

703 self.__class__.__name__, 

704 orm_util.state_str(self.state), 

705 ) 

706 

707 

708class DeleteState(PostSortRec): 

709 __slots__ = "state", "mapper" 

710 

711 def __init__(self, uow, state): 

712 self.state = state 

713 self.mapper = state.mapper.base_mapper 

714 

715 def execute_aggregate(self, uow, recs): 

716 cls_ = self.__class__ 

717 mapper = self.mapper 

718 our_recs = [ 

719 r for r in recs if r.__class__ is cls_ and r.mapper is mapper 

720 ] 

721 recs.difference_update(our_recs) 

722 states = [self.state] + [r.state for r in our_recs] 

723 persistence.delete_obj( 

724 mapper, [s for s in states if uow.states[s][0]], uow 

725 ) 

726 

727 def __repr__(self): 

728 return "%s(%s)" % ( 

729 self.__class__.__name__, 

730 orm_util.state_str(self.state), 

731 )