Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py : 60%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# orm/unitofwork.py
2# Copyright (C) 2005-2020 the SQLAlchemy authors and contributors
3# <see AUTHORS file>
4#
5# This module is part of SQLAlchemy and is released under
6# the MIT License: http://www.opensource.org/licenses/mit-license.php
8"""The internals for the unit of work system.
10The session's flush() process passes objects to a contextual object
11here, which assembles flush tasks based on mappers and their properties,
12organizes them in order of dependency, and executes.
14"""
16from . import attributes
17from . import exc as orm_exc
18from . import persistence
19from . import util as orm_util
20from .. import event
21from .. import util
22from ..util import topological
25def track_cascade_events(descriptor, prop):
26 """Establish event listeners on object attributes which handle
27 cascade-on-set/append.
29 """
30 key = prop.key
32 def append(state, item, initiator):
33 # process "save_update" cascade rules for when
34 # an instance is appended to the list of another instance
36 if item is None:
37 return
39 sess = state.session
40 if sess:
41 if sess._warn_on_events:
42 sess._flush_warning("collection append")
44 prop = state.manager.mapper._props[key]
45 item_state = attributes.instance_state(item)
46 if (
47 prop._cascade.save_update
48 and (prop.cascade_backrefs or key == initiator.key)
49 and not sess._contains_state(item_state)
50 ):
51 sess._save_or_update_state(item_state)
52 return item
54 def remove(state, item, initiator):
55 if item is None:
56 return
58 sess = state.session
60 prop = state.manager.mapper._props[key]
62 if sess and sess._warn_on_events:
63 sess._flush_warning(
64 "collection remove"
65 if prop.uselist
66 else "related attribute delete"
67 )
69 if (
70 item is not None
71 and item is not attributes.NEVER_SET
72 and item is not attributes.PASSIVE_NO_RESULT
73 and prop._cascade.delete_orphan
74 ):
75 # expunge pending orphans
76 item_state = attributes.instance_state(item)
78 if prop.mapper._is_orphan(item_state):
79 if sess and item_state in sess._new:
80 sess.expunge(item)
81 else:
82 # the related item may or may not itself be in a
83 # Session, however the parent for which we are catching
84 # the event is not in a session, so memoize this on the
85 # item
86 item_state._orphaned_outside_of_session = True
88 def set_(state, newvalue, oldvalue, initiator):
89 # process "save_update" cascade rules for when an instance
90 # is attached to another instance
91 if oldvalue is newvalue:
92 return newvalue
94 sess = state.session
95 if sess:
97 if sess._warn_on_events:
98 sess._flush_warning("related attribute set")
100 prop = state.manager.mapper._props[key]
101 if newvalue is not None:
102 newvalue_state = attributes.instance_state(newvalue)
103 if (
104 prop._cascade.save_update
105 and (prop.cascade_backrefs or key == initiator.key)
106 and not sess._contains_state(newvalue_state)
107 ):
108 sess._save_or_update_state(newvalue_state)
110 if (
111 oldvalue is not None
112 and oldvalue is not attributes.NEVER_SET
113 and oldvalue is not attributes.PASSIVE_NO_RESULT
114 and prop._cascade.delete_orphan
115 ):
116 # possible to reach here with attributes.NEVER_SET ?
117 oldvalue_state = attributes.instance_state(oldvalue)
119 if oldvalue_state in sess._new and prop.mapper._is_orphan(
120 oldvalue_state
121 ):
122 sess.expunge(oldvalue)
123 return newvalue
125 event.listen(descriptor, "append", append, raw=True, retval=True)
126 event.listen(descriptor, "remove", remove, raw=True, retval=True)
127 event.listen(descriptor, "set", set_, raw=True, retval=True)
130class UOWTransaction(object):
131 def __init__(self, session):
132 self.session = session
134 # dictionary used by external actors to
135 # store arbitrary state information.
136 self.attributes = {}
138 # dictionary of mappers to sets of
139 # DependencyProcessors, which are also
140 # set to be part of the sorted flush actions,
141 # which have that mapper as a parent.
142 self.deps = util.defaultdict(set)
144 # dictionary of mappers to sets of InstanceState
145 # items pending for flush which have that mapper
146 # as a parent.
147 self.mappers = util.defaultdict(set)
149 # a dictionary of Preprocess objects, which gather
150 # additional states impacted by the flush
151 # and determine if a flush action is needed
152 self.presort_actions = {}
154 # dictionary of PostSortRec objects, each
155 # one issues work during the flush within
156 # a certain ordering.
157 self.postsort_actions = {}
159 # a set of 2-tuples, each containing two
160 # PostSortRec objects where the second
161 # is dependent on the first being executed
162 # first
163 self.dependencies = set()
165 # dictionary of InstanceState-> (isdelete, listonly)
166 # tuples, indicating if this state is to be deleted
167 # or insert/updated, or just refreshed
168 self.states = {}
170 # tracks InstanceStates which will be receiving
171 # a "post update" call. Keys are mappers,
172 # values are a set of states and a set of the
173 # columns which should be included in the update.
174 self.post_update_states = util.defaultdict(lambda: (set(), set()))
176 @property
177 def has_work(self):
178 return bool(self.states)
180 def was_already_deleted(self, state):
181 """return true if the given state is expired and was deleted
182 previously.
183 """
184 if state.expired:
185 try:
186 state._load_expired(state, attributes.PASSIVE_OFF)
187 except orm_exc.ObjectDeletedError:
188 self.session._remove_newly_deleted([state])
189 return True
190 return False
192 def is_deleted(self, state):
193 """return true if the given state is marked as deleted
194 within this uowtransaction."""
196 return state in self.states and self.states[state][0]
198 def memo(self, key, callable_):
199 if key in self.attributes:
200 return self.attributes[key]
201 else:
202 self.attributes[key] = ret = callable_()
203 return ret
205 def remove_state_actions(self, state):
206 """remove pending actions for a state from the uowtransaction."""
208 isdelete = self.states[state][0]
210 self.states[state] = (isdelete, True)
212 def get_attribute_history(
213 self, state, key, passive=attributes.PASSIVE_NO_INITIALIZE
214 ):
215 """facade to attributes.get_state_history(), including
216 caching of results."""
218 hashkey = ("history", state, key)
220 # cache the objects, not the states; the strong reference here
221 # prevents newly loaded objects from being dereferenced during the
222 # flush process
224 if hashkey in self.attributes:
225 history, state_history, cached_passive = self.attributes[hashkey]
226 # if the cached lookup was "passive" and now
227 # we want non-passive, do a non-passive lookup and re-cache
229 if (
230 not cached_passive & attributes.SQL_OK
231 and passive & attributes.SQL_OK
232 ):
233 impl = state.manager[key].impl
234 history = impl.get_history(
235 state,
236 state.dict,
237 attributes.PASSIVE_OFF | attributes.LOAD_AGAINST_COMMITTED,
238 )
239 if history and impl.uses_objects:
240 state_history = history.as_state()
241 else:
242 state_history = history
243 self.attributes[hashkey] = (history, state_history, passive)
244 else:
245 impl = state.manager[key].impl
246 # TODO: store the history as (state, object) tuples
247 # so we don't have to keep converting here
248 history = impl.get_history(
249 state, state.dict, passive | attributes.LOAD_AGAINST_COMMITTED
250 )
251 if history and impl.uses_objects:
252 state_history = history.as_state()
253 else:
254 state_history = history
255 self.attributes[hashkey] = (history, state_history, passive)
257 return state_history
259 def has_dep(self, processor):
260 return (processor, True) in self.presort_actions
262 def register_preprocessor(self, processor, fromparent):
263 key = (processor, fromparent)
264 if key not in self.presort_actions:
265 self.presort_actions[key] = Preprocess(processor, fromparent)
267 def register_object(
268 self,
269 state,
270 isdelete=False,
271 listonly=False,
272 cancel_delete=False,
273 operation=None,
274 prop=None,
275 ):
276 if not self.session._contains_state(state):
277 # this condition is normal when objects are registered
278 # as part of a relationship cascade operation. it should
279 # not occur for the top-level register from Session.flush().
280 if not state.deleted and operation is not None:
281 util.warn(
282 "Object of type %s not in session, %s operation "
283 "along '%s' will not proceed"
284 % (orm_util.state_class_str(state), operation, prop)
285 )
286 return False
288 if state not in self.states:
289 mapper = state.manager.mapper
291 if mapper not in self.mappers:
292 self._per_mapper_flush_actions(mapper)
294 self.mappers[mapper].add(state)
295 self.states[state] = (isdelete, listonly)
296 else:
297 if not listonly and (isdelete or cancel_delete):
298 self.states[state] = (isdelete, False)
299 return True
301 def register_post_update(self, state, post_update_cols):
302 mapper = state.manager.mapper.base_mapper
303 states, cols = self.post_update_states[mapper]
304 states.add(state)
305 cols.update(post_update_cols)
307 def _per_mapper_flush_actions(self, mapper):
308 saves = SaveUpdateAll(self, mapper.base_mapper)
309 deletes = DeleteAll(self, mapper.base_mapper)
310 self.dependencies.add((saves, deletes))
312 for dep in mapper._dependency_processors:
313 dep.per_property_preprocessors(self)
315 for prop in mapper.relationships:
316 if prop.viewonly:
317 continue
318 dep = prop._dependency_processor
319 dep.per_property_preprocessors(self)
321 @util.memoized_property
322 def _mapper_for_dep(self):
323 """return a dynamic mapping of (Mapper, DependencyProcessor) to
324 True or False, indicating if the DependencyProcessor operates
325 on objects of that Mapper.
327 The result is stored in the dictionary persistently once
328 calculated.
330 """
331 return util.PopulateDict(
332 lambda tup: tup[0]._props.get(tup[1].key) is tup[1].prop
333 )
335 def filter_states_for_dep(self, dep, states):
336 """Filter the given list of InstanceStates to those relevant to the
337 given DependencyProcessor.
339 """
340 mapper_for_dep = self._mapper_for_dep
341 return [s for s in states if mapper_for_dep[(s.manager.mapper, dep)]]
343 def states_for_mapper_hierarchy(self, mapper, isdelete, listonly):
344 checktup = (isdelete, listonly)
345 for mapper in mapper.base_mapper.self_and_descendants:
346 for state in self.mappers[mapper]:
347 if self.states[state] == checktup:
348 yield state
350 def _generate_actions(self):
351 """Generate the full, unsorted collection of PostSortRecs as
352 well as dependency pairs for this UOWTransaction.
354 """
355 # execute presort_actions, until all states
356 # have been processed. a presort_action might
357 # add new states to the uow.
358 while True:
359 ret = False
360 for action in list(self.presort_actions.values()):
361 if action.execute(self):
362 ret = True
363 if not ret:
364 break
366 # see if the graph of mapper dependencies has cycles.
367 self.cycles = cycles = topological.find_cycles(
368 self.dependencies, list(self.postsort_actions.values())
369 )
371 if cycles:
372 # if yes, break the per-mapper actions into
373 # per-state actions
374 convert = dict(
375 (rec, set(rec.per_state_flush_actions(self))) for rec in cycles
376 )
378 # rewrite the existing dependencies to point to
379 # the per-state actions for those per-mapper actions
380 # that were broken up.
381 for edge in list(self.dependencies):
382 if (
383 None in edge
384 or edge[0].disabled
385 or edge[1].disabled
386 or cycles.issuperset(edge)
387 ):
388 self.dependencies.remove(edge)
389 elif edge[0] in cycles:
390 self.dependencies.remove(edge)
391 for dep in convert[edge[0]]:
392 self.dependencies.add((dep, edge[1]))
393 elif edge[1] in cycles:
394 self.dependencies.remove(edge)
395 for dep in convert[edge[1]]:
396 self.dependencies.add((edge[0], dep))
398 return set(
399 [a for a in self.postsort_actions.values() if not a.disabled]
400 ).difference(cycles)
402 def execute(self):
403 postsort_actions = self._generate_actions()
405 # sort = topological.sort(self.dependencies, postsort_actions)
406 # print "--------------"
407 # print "\ndependencies:", self.dependencies
408 # print "\ncycles:", self.cycles
409 # print "\nsort:", list(sort)
410 # print "\nCOUNT OF POSTSORT ACTIONS", len(postsort_actions)
412 # execute
413 if self.cycles:
414 for set_ in topological.sort_as_subsets(
415 self.dependencies, postsort_actions
416 ):
417 while set_:
418 n = set_.pop()
419 n.execute_aggregate(self, set_)
420 else:
421 for rec in topological.sort(self.dependencies, postsort_actions):
422 rec.execute(self)
424 def finalize_flush_changes(self):
425 """mark processed objects as clean / deleted after a successful
426 flush().
428 this method is called within the flush() method after the
429 execute() method has succeeded and the transaction has been committed.
431 """
432 if not self.states:
433 return
435 states = set(self.states)
436 isdel = set(
437 s for (s, (isdelete, listonly)) in self.states.items() if isdelete
438 )
439 other = states.difference(isdel)
440 if isdel:
441 self.session._remove_newly_deleted(isdel)
442 if other:
443 self.session._register_persistent(other)
446class IterateMappersMixin(object):
447 def _mappers(self, uow):
448 if self.fromparent:
449 return iter(
450 m
451 for m in self.dependency_processor.parent.self_and_descendants
452 if uow._mapper_for_dep[(m, self.dependency_processor)]
453 )
454 else:
455 return self.dependency_processor.mapper.self_and_descendants
458class Preprocess(IterateMappersMixin):
459 __slots__ = (
460 "dependency_processor",
461 "fromparent",
462 "processed",
463 "setup_flush_actions",
464 )
466 def __init__(self, dependency_processor, fromparent):
467 self.dependency_processor = dependency_processor
468 self.fromparent = fromparent
469 self.processed = set()
470 self.setup_flush_actions = False
472 def execute(self, uow):
473 delete_states = set()
474 save_states = set()
476 for mapper in self._mappers(uow):
477 for state in uow.mappers[mapper].difference(self.processed):
478 (isdelete, listonly) = uow.states[state]
479 if not listonly:
480 if isdelete:
481 delete_states.add(state)
482 else:
483 save_states.add(state)
485 if delete_states:
486 self.dependency_processor.presort_deletes(uow, delete_states)
487 self.processed.update(delete_states)
488 if save_states:
489 self.dependency_processor.presort_saves(uow, save_states)
490 self.processed.update(save_states)
492 if delete_states or save_states:
493 if not self.setup_flush_actions and (
494 self.dependency_processor.prop_has_changes(
495 uow, delete_states, True
496 )
497 or self.dependency_processor.prop_has_changes(
498 uow, save_states, False
499 )
500 ):
501 self.dependency_processor.per_property_flush_actions(uow)
502 self.setup_flush_actions = True
503 return True
504 else:
505 return False
508class PostSortRec(object):
509 __slots__ = ("disabled",)
511 def __new__(cls, uow, *args):
512 key = (cls,) + args
513 if key in uow.postsort_actions:
514 return uow.postsort_actions[key]
515 else:
516 uow.postsort_actions[key] = ret = object.__new__(cls)
517 ret.disabled = False
518 return ret
520 def execute_aggregate(self, uow, recs):
521 self.execute(uow)
524class ProcessAll(IterateMappersMixin, PostSortRec):
525 __slots__ = "dependency_processor", "isdelete", "fromparent"
527 def __init__(self, uow, dependency_processor, isdelete, fromparent):
528 self.dependency_processor = dependency_processor
529 self.isdelete = isdelete
530 self.fromparent = fromparent
531 uow.deps[dependency_processor.parent.base_mapper].add(
532 dependency_processor
533 )
535 def execute(self, uow):
536 states = self._elements(uow)
537 if self.isdelete:
538 self.dependency_processor.process_deletes(uow, states)
539 else:
540 self.dependency_processor.process_saves(uow, states)
542 def per_state_flush_actions(self, uow):
543 # this is handled by SaveUpdateAll and DeleteAll,
544 # since a ProcessAll should unconditionally be pulled
545 # into per-state if either the parent/child mappers
546 # are part of a cycle
547 return iter([])
549 def __repr__(self):
550 return "%s(%s, isdelete=%s)" % (
551 self.__class__.__name__,
552 self.dependency_processor,
553 self.isdelete,
554 )
556 def _elements(self, uow):
557 for mapper in self._mappers(uow):
558 for state in uow.mappers[mapper]:
559 (isdelete, listonly) = uow.states[state]
560 if isdelete == self.isdelete and not listonly:
561 yield state
564class PostUpdateAll(PostSortRec):
565 __slots__ = "mapper", "isdelete"
567 def __init__(self, uow, mapper, isdelete):
568 self.mapper = mapper
569 self.isdelete = isdelete
571 def execute(self, uow):
572 states, cols = uow.post_update_states[self.mapper]
573 states = [s for s in states if uow.states[s][0] == self.isdelete]
575 persistence.post_update(self.mapper, states, uow, cols)
578class SaveUpdateAll(PostSortRec):
579 __slots__ = ("mapper",)
581 def __init__(self, uow, mapper):
582 self.mapper = mapper
583 assert mapper is mapper.base_mapper
585 def execute(self, uow):
586 persistence.save_obj(
587 self.mapper,
588 uow.states_for_mapper_hierarchy(self.mapper, False, False),
589 uow,
590 )
592 def per_state_flush_actions(self, uow):
593 states = list(
594 uow.states_for_mapper_hierarchy(self.mapper, False, False)
595 )
596 base_mapper = self.mapper.base_mapper
597 delete_all = DeleteAll(uow, base_mapper)
598 for state in states:
599 # keep saves before deletes -
600 # this ensures 'row switch' operations work
601 action = SaveUpdateState(uow, state)
602 uow.dependencies.add((action, delete_all))
603 yield action
605 for dep in uow.deps[self.mapper]:
606 states_for_prop = uow.filter_states_for_dep(dep, states)
607 dep.per_state_flush_actions(uow, states_for_prop, False)
609 def __repr__(self):
610 return "%s(%s)" % (self.__class__.__name__, self.mapper)
613class DeleteAll(PostSortRec):
614 __slots__ = ("mapper",)
616 def __init__(self, uow, mapper):
617 self.mapper = mapper
618 assert mapper is mapper.base_mapper
620 def execute(self, uow):
621 persistence.delete_obj(
622 self.mapper,
623 uow.states_for_mapper_hierarchy(self.mapper, True, False),
624 uow,
625 )
627 def per_state_flush_actions(self, uow):
628 states = list(
629 uow.states_for_mapper_hierarchy(self.mapper, True, False)
630 )
631 base_mapper = self.mapper.base_mapper
632 save_all = SaveUpdateAll(uow, base_mapper)
633 for state in states:
634 # keep saves before deletes -
635 # this ensures 'row switch' operations work
636 action = DeleteState(uow, state)
637 uow.dependencies.add((save_all, action))
638 yield action
640 for dep in uow.deps[self.mapper]:
641 states_for_prop = uow.filter_states_for_dep(dep, states)
642 dep.per_state_flush_actions(uow, states_for_prop, True)
644 def __repr__(self):
645 return "%s(%s)" % (self.__class__.__name__, self.mapper)
648class ProcessState(PostSortRec):
649 __slots__ = "dependency_processor", "isdelete", "state"
651 def __init__(self, uow, dependency_processor, isdelete, state):
652 self.dependency_processor = dependency_processor
653 self.isdelete = isdelete
654 self.state = state
656 def execute_aggregate(self, uow, recs):
657 cls_ = self.__class__
658 dependency_processor = self.dependency_processor
659 isdelete = self.isdelete
660 our_recs = [
661 r
662 for r in recs
663 if r.__class__ is cls_
664 and r.dependency_processor is dependency_processor
665 and r.isdelete is isdelete
666 ]
667 recs.difference_update(our_recs)
668 states = [self.state] + [r.state for r in our_recs]
669 if isdelete:
670 dependency_processor.process_deletes(uow, states)
671 else:
672 dependency_processor.process_saves(uow, states)
674 def __repr__(self):
675 return "%s(%s, %s, delete=%s)" % (
676 self.__class__.__name__,
677 self.dependency_processor,
678 orm_util.state_str(self.state),
679 self.isdelete,
680 )
683class SaveUpdateState(PostSortRec):
684 __slots__ = "state", "mapper"
686 def __init__(self, uow, state):
687 self.state = state
688 self.mapper = state.mapper.base_mapper
690 def execute_aggregate(self, uow, recs):
691 cls_ = self.__class__
692 mapper = self.mapper
693 our_recs = [
694 r for r in recs if r.__class__ is cls_ and r.mapper is mapper
695 ]
696 recs.difference_update(our_recs)
697 persistence.save_obj(
698 mapper, [self.state] + [r.state for r in our_recs], uow
699 )
701 def __repr__(self):
702 return "%s(%s)" % (
703 self.__class__.__name__,
704 orm_util.state_str(self.state),
705 )
708class DeleteState(PostSortRec):
709 __slots__ = "state", "mapper"
711 def __init__(self, uow, state):
712 self.state = state
713 self.mapper = state.mapper.base_mapper
715 def execute_aggregate(self, uow, recs):
716 cls_ = self.__class__
717 mapper = self.mapper
718 our_recs = [
719 r for r in recs if r.__class__ is cls_ and r.mapper is mapper
720 ]
721 recs.difference_update(our_recs)
722 states = [self.state] + [r.state for r in our_recs]
723 persistence.delete_obj(
724 mapper, [s for s in states if uow.states[s][0]], uow
725 )
727 def __repr__(self):
728 return "%s(%s)" % (
729 self.__class__.__name__,
730 orm_util.state_str(self.state),
731 )