Package multivac
[hide private]
[frames] | no frames]

Source Code for Package multivac

  1  #! /usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  """ 
  4  MultiVAC generic framework for building tree-based status applications. 
  5   
  6  This modules provides an implementation of a generic framework, called 
  7  MultiVAC, for use in building applications working with trees of elements, in 
  8  which the status of a node derives from the status of the leaves. 
  9   
 10  Two classes are defined, one for each table in the relational model 
 11  L{Element},  L{Project}. Also, two additional classes are defined for the 
 12  association relationships between the former: L{TagElement} and L{TagProject}. 
 13  """ 
 14   
 15  from version import * 
 16   
 17  import os 
 18  import sys 
 19  import warnings 
 20   
 21  import sqlalchemy 
 22   
 23  from sqlalchemy import (Column, MetaData, Table, Index, 
 24                          ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint, 
 25                          Boolean, Integer, String, Text, 
 26                          create_engine, func, event) 
 27  from sqlalchemy.ext.associationproxy import association_proxy 
 28  from sqlalchemy.ext.hybrid import hybrid_property 
 29  from sqlalchemy.orm import mapper, relationship, scoped_session, sessionmaker 
 30  from sqlalchemy.orm.collections import attribute_mapped_collection 
 31  from sqlalchemy.orm.exc import NoResultFound 
 32  from sqlalchemy.orm.session import object_session 
 33  from sqlalchemy.orm.util import identity_key 
 34  from sqlalchemy.sql.expression import literal, literal_column 
 35   
 36  #: Metadata object for this model 
 37  metadata = MetaData() 
 38  #: Name-indexed dict containing the definition of every table 
 39  tables = {} 
 40  #: Name-indexed dict containing the mapper of every table 
 41  mappers = {} 
42 43 -def init(session_maker, **kwargs):
44 """ 45 Initialize the db connection, set up the tables and map the classes. 46 47 @param session_maker: Session generator to bind to the model 48 @type session_maker: sqlalchemy.orm.session.Session factory 49 @param kwargs: Additional settings for the mapper 50 @type kwargs: dict 51 """ 52 53 # Bind engine 54 metadata.bind = session_maker.bind 55 # Setup model 56 setup_tables() 57 setup_mappers(**kwargs) 58 setup_events(session_maker, **kwargs)
59
60 61 -def setup_tables():
62 """ 63 Define the tables, columns, keys and constraints of the DB. 64 """ 65 66 global tables 67 68 tables['element'] = Table('element', metadata, 69 Column('id', Integer, nullable=False), 70 Column('forced_status', Boolean, nullable=False, default=False), 71 Column('name', String(120), nullable=False), 72 Column('parent_id', Integer, nullable=True), 73 Column('project_id', Integer, nullable=False), 74 Column('status', String(20), nullable=False), 75 PrimaryKeyConstraint('id'), 76 ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), 77 ForeignKeyConstraint(['project_id', 'parent_id'], ['element.project_id', 'element.id'], ondelete='CASCADE'), 78 UniqueConstraint('project_id', 'parent_id', 'name'), 79 UniqueConstraint('project_id', 'id') 80 ) 81 Index('element_uk_root', tables['element'].c.project_id, tables['element'].c.name, 82 postgresql_where=tables['element'].c.parent_id == None, unique=True 83 ) 84 85 tables['project'] = Table('project', metadata, 86 Column('id', Integer, nullable=False), 87 Column('name', String(20), nullable=False), 88 PrimaryKeyConstraint('id'), 89 UniqueConstraint('name'), 90 ) 91 92 tables['tag_element'] = Table('tag_element', metadata, 93 Column('tag', String(20), nullable=False), 94 Column('element_id', Integer, nullable=False), 95 Column('value', Text(), nullable=True), 96 PrimaryKeyConstraint('element_id', 'tag'), 97 ForeignKeyConstraint(['element_id'], ['element.id'], ondelete='CASCADE'), 98 ) 99 100 tables['tag_project'] = Table('tag_project', metadata, 101 Column('tag', String(20), nullable=False), 102 Column('project_id', Integer, nullable=False), 103 Column('value', Text(), nullable=True), 104 PrimaryKeyConstraint('project_id', 'tag'), 105 ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), 106 )
107
108 -def setup_mappers(**kwargs):
109 """ 110 Define the mapping between tables and classes, and the relationships that link them. 111 112 @kwarg extra_mapping: Mapping between database tables and classes 113 @type extra_mapping: dict 114 @kwarg extra_properties: Dictionary of additional properties for a table 115 @type extra_properties: dict 116 @kwarg extra_extensions: Dictionary of additional extensions for a table 117 @type extra_extensions: dict 118 @kwarg extra_kwargs: Dictionary of additional arguments for a mapper 119 @type extra_kwargs: dict 120 """ 121 122 global mappers 123 124 mapping = { 125 'element' : Element, 126 'project' : Project, 127 'tag_element' : TagElement, 128 'tag_project' : TagProject, 129 } 130 mapping.update(kwargs.get('extra_mapping', dict())) 131 132 assert issubclass(mapping['element'], Element) 133 assert issubclass(mapping['project'], Project) 134 assert issubclass(mapping['tag_element'], TagElement) 135 assert issubclass(mapping['tag_project'], TagProject) 136 137 properties = {} 138 properties['element'] = { 139 '_id' : tables['element'].c.id, 140 '_parent_id' : tables['element'].c.parent_id, 141 '_project_id' : tables['element'].c.project_id, 142 '_status' : tables['element'].c.status, 143 # FIXME: Add project_id to parent and children properties when #1401 is fixed 144 '_children' : relationship(mapping['element'], back_populates='_parent', collection_class=set, 145 primaryjoin = tables['element'].c.parent_id == tables['element'].c.id, 146 cascade='all', passive_deletes=True), 147 '_parent' : relationship(mapping['element'], back_populates='_children', collection_class=set, 148 primaryjoin = tables['element'].c.parent_id == tables['element'].c.id, 149 remote_side = [ tables['element'].c.id ]), 150 '_project' : relationship(mapping['project'], back_populates='elements', collection_class=set), 151 '_tag' : relationship(mapping['tag_element'], collection_class=attribute_mapped_collection('tag'), 152 cascade='all, delete-orphan', passive_deletes=True), 153 } 154 properties['project'] = { 155 '_id' : tables['project'].c.id, 156 'elements' : relationship(mapping['element'], back_populates='_project', collection_class=set, 157 primaryjoin = tables['element'].c.project_id == tables['project'].c.id, 158 cascade='all', passive_deletes=True), 159 '_tag' : relationship(mapping['tag_project'], collection_class=attribute_mapped_collection('tag'), 160 cascade='all, delete-orphan', passive_deletes=True), 161 } 162 properties['tag_element'] = {} 163 properties['tag_project'] = {} 164 165 extra_properties = kwargs.get('extra_properties', dict()) 166 for entity in mapping.iterkeys(): 167 properties[entity].update(extra_properties.get(entity, dict())) 168 169 extensions = {} 170 extensions.update(kwargs.get('extra_extensions', dict())) 171 172 options = {} 173 options.update(kwargs.get('extra_kwargs', dict())) 174 175 for name, cls in mapping.iteritems(): 176 mappers[name] = mapper(cls, tables[name], 177 properties=properties.get(name, None), 178 extension=extensions.get(name, None), 179 **options.get(name, {})) 180 181 """ 182 Association proxy to access its tags and retrieve their corresponding value. 183 Example: instance.tag['name'] = 'value' 184 """ 185 Element.tag = association_proxy('_tag', 'value', creator=lambda tag, value: mapping['tag_element'](tag=tag, value=value)) 186 Project.tag = association_proxy('_tag', 'value', creator=lambda tag, value: mapping['tag_project'](tag=tag, value=value))
187
188 189 -def setup_events(session_maker, **kwargs):
190 """ 191 Define the events of the model. 192 """ 193 mapping = { 194 'element' : Element, 195 'project' : Project, 196 'tag_element' : TagElement, 197 'tag_project' : TagProject, 198 } 199 mapping.update(kwargs.get('extra_mapping', dict())) 200 201 event.listen(mapping['element']._children, 'append', mapping['element']._children_added) 202 event.listen(mapping['element']._children, 'remove', mapping['element']._children_removed) 203 event.listen(session_maker, 'before_flush', _session_before_flush)
204
205 -def _session_before_flush(session, flush_context, instances):
206 """ 207 Ensure that when an Element instance is deleted, the children collection of 208 its parent is notified to update and cascade the status change. 209 """ 210 for instance in session.deleted: 211 if isinstance(instance, Element): 212 if instance.parent: 213 instance.parent.children.remove(instance)
214
215 -class CTENotSupported(UserWarning):
216 """ 217 User-defined exception to warn the user when calling a method 218 that uses Common Table Expressions (CTE) to perform its work. 219 In most cases, there is an alternative slow path of code. 220 """ 221 pass
222
223 -class ORM_Base(object):
224 """ 225 Base class for mapping the tables. 226 """ 227
228 - def __init__(self, *args, **kwargs):
229 """ 230 Base contructor for all mapped entities. 231 Set the value of attributes based on keyword arguments. 232 233 @param args: Optional arguments to the constructor 234 @type args: tuple 235 @param kwargs: Optional keyword arguments to the constructor 236 @type kwargs: dict 237 """ 238 for name, value in kwargs.iteritems(): 239 setattr(self, name, value)
240
241 242 -class Element(ORM_Base):
243 """ 244 Mapping class for the table «element». 245 """ 246 247 @hybrid_property
248 - def id(self):
249 """ 250 Read only accessor to prevent setting this field. 251 252 @return: Surrogate primary key 253 @rtype: int 254 """ 255 return self._id
256 257 @hybrid_property
258 - def children(self):
259 """ 260 The collection of child Elements of this instance. 261 262 @return: This instance collection of children Elements 263 @rtype: set<L{Element}> 264 """ 265 return self._children
266 267 @hybrid_property
268 - def parent_id(self):
269 """ 270 Read only accessor to prevent setting this field. 271 272 @return: Foreign key for the parent L{Element} relationship 273 @rtype: int 274 """ 275 return self._parent_id
276 277 @hybrid_property
278 - def parent(self):
279 """ 280 The parent Element of this instance. 281 282 @return: The parent Element 283 @rtype: L{Element} 284 """ 285 return self._parent
286 287 @parent.setter
288 - def parent(self, parent):
289 """ 290 Setter for the related parent Element of this instance. 291 Ensures project coherence between itself and the parent, and proper 292 children collection initialization. Also cascades status changes. 293 294 @param parent: The parent Element to be assigned 295 @type parent: L{Element} 296 """ 297 # Avoid infinite recursion 298 if self.parent == parent: 299 return 300 assert parent == None or isinstance(parent, Element) 301 # Check project coherence (parent - self) 302 if self.parent != None and parent != None: 303 assert parent.project == self.project 304 # Ensure children initialization and check for existence. DO NOT REMOVE 305 if parent != None: 306 assert parent.children != None 307 # Assign new parent 308 self._parent = parent 309 if self.parent != None: 310 self.project = parent.project 311 # Cascade status changes only if it has a parent 312 if self.status and not self.parent.forced_status: 313 self._cascade_status()
314 315 @hybrid_property
316 - def project_id(self):
317 """ 318 Read only accessor to prevent setting this field. 319 320 @return: Foreign key for the parent L{Project} relationship 321 @rtype: int 322 """ 323 return self._project_id
324 325 @hybrid_property
326 - def project(self):
327 """ 328 The related Project of this instance. 329 330 @return: The related Project 331 @rtype: L{Project} 332 """ 333 return self._project
334 335 @project.setter
336 - def project(self, project):
337 """ 338 Setter for the related Project of this instance. 339 Prevents a second assignation. 340 341 @param project: The Project to be assigned 342 @type project: L{Project} 343 """ 344 # Avoid infinite recursion 345 if self.project == project: 346 return 347 assert isinstance(project, Project) 348 # Avoid second assignation 349 if self.project == None: 350 self._project = project 351 else: 352 raise AttributeError('This attribute cannot be modified once it has been assigned.')
353 354 @hybrid_property
355 - def status(self):
356 """ 357 The status of this instance 358 359 @return: status 360 @rtype: str 361 """ 362 return self._status
363 364 @status.setter
365 - def status(self, status):
366 """ 367 Setter for the status of this instance. 368 Ensures the cascade of a status change. 369 370 @param status: The status to be assigned 371 @type status: str 372 """ 373 # Avoid infinite recursion 374 if self.status == status: 375 return 376 else: 377 self._status = status 378 # Cascade status changes 379 if self.parent and not self.parent.forced_status: 380 self._cascade_status()
381
382 - def __init__(self, *args, **kwargs):
383 """ 384 Constructor for Element instances. 385 Ensures that the «forced_status» field is assigned first to cascade 386 status properly. 387 388 @param args: Optional arguments to the constructor 389 @type args: tuple 390 @param kwargs: Optional keyword arguments to the constructor 391 @type kwargs: dict 392 """ 393 # Assign first force_status field 394 if 'forced_status' in kwargs: 395 setattr(self, 'forced_status', kwargs['forced_status']) 396 del kwargs['forced_status'] 397 # Assign the rest of the fields 398 super(Element, self).__init__(*args, **kwargs)
399 400 @classmethod
401 - def _children_added(cls, parent, child, initiator):
402 """ 403 Listener to be executed when an element has to be added to a 404 children collection. 405 Check the added child status and update the parent's one. 406 407 @param parent: The Element that has a new child added 408 @type parent: L{Element} 409 @param child: The Element being added as a child 410 @type child: L{Element} 411 """ 412 if not child.status or parent.forced_status: 413 return 414 if parent.status > child.status: 415 parent.status = child.status 416 elif (parent.status < child.status) and (len(parent.children) == 1): 417 parent.status = child.status
418 419 @classmethod
420 - def _children_removed(cls, parent, child, initiator):
421 """ 422 Listener to be executed when an element has to be removed from a 423 children collection. 424 Check the removed child status and update the parent's one. 425 426 @param parent: The Element that has a child removed 427 @type parent: L{Element} 428 @param child: The Element being removed as a child 429 @type child: L{Element} 430 """ 431 432 if not child.status or parent.forced_status: 433 return 434 new_children = parent.children.difference([child]) 435 if parent.status == child.status and new_children: 436 new_status = min([c.status for c in new_children]) 437 if parent.status != new_status: 438 parent.status = new_status
439
440 - def _cascade_status(self):
441 """ 442 Propagate its status to its parent, in a recursive manner. 443 """ 444 if self.parent.status > self.status: 445 self.parent.status = self.status 446 elif self.parent.status < self.status: 447 new_status = min([c.status for c in self.parent.children]) 448 if self.parent.status != new_status: 449 self.parent.status = new_status
450 451 @classmethod
452 - def query_tags(cls, session, tags={}):
453 """ 454 Overriden query method to apply tag-based custom filtering, on top of 455 common equality filter. 456 457 @param session: The database session in which to execute the query 458 @type session: Session 459 @param tags: Tag names and values to apply as a filter 460 @type tags: dict 461 @return: A query selecting Element instances, filtered by tags 462 @rtype: Query<L{Element}> 463 """ 464 465 q = session.query(cls) 466 if tags: 467 crit = literal(False) 468 for tag, value in tags.iteritems(): 469 if value == True: 470 crit |= (tables['tag_element'].c.tag == tag) 471 elif value == None or value != False : 472 crit |= (tables['tag_element'].c.tag == tag) & (tables['tag_element'].c.value == value) 473 q = q.join(tables['tag_element']) \ 474 .group_by(*[c for c in tables['element'].columns]) \ 475 .having(func.count(tables['element'].c.id) == len(tags)) \ 476 .filter(crit) 477 return q.from_self()
478 479 @property
480 - def ancestors(self):
481 """ 482 Retrieve all the ancestors of this node. 483 If the node has no parents, return an empty list. 484 Else, start retrieving them from the identity map and, when not there, 485 fetch the rest from the database using a CTE. 486 487 @return: A list of all the ancestors of this node, ordered by proximity. 488 @rtype: list<L{Element}> 489 """ 490 cls = self.__class__ 491 session = object_session(self) 492 # Must flush to get the parent_id from the database 493 session.flush() 494 if self._parent_id is None: 495 # End of the recursion. This Element has no parent. 496 return [] 497 else: 498 # Best case available: retrieve the parent from the identity map 499 key = identity_key(cls, self._parent_id) 500 parent = session.identity_map.get(key, None) 501 if parent: 502 # Parent found in identity map, recurse and return. 503 parents = [parent] 504 parents.extend(parent.ancestors) 505 return parents 506 else: 507 # Parent NOT found in identity map. Must use CTE 508 if session.bind.name != 'postgresql': 509 # If not using PostgreSQL, warn the user and use the non-optimized method. 510 warnings.warn('CTE are only supported on PostgreSQL. Using slower technique for "ancestor" method.', CTENotSupported) 511 parents = [self.parent] 512 parents.extend(self.parent.ancestors) 513 return parents 514 # Use a CTE to retrieve ALL ancestors 515 l0 = literal_column('0').label('level') 516 q_base = session.query(cls, l0).filter_by( 517 id = self._parent_id 518 ).cte(recursive = True) 519 l1 = literal_column('level + 1').label('level') 520 q_rec = session.query(cls, l1).filter( 521 q_base.c.parent_id == cls.id 522 ) 523 q_cte = q_base.union_all(q_rec) 524 return session.query(cls).select_from(q_cte).order_by(q_cte.c.level)
525
526 - def __repr__(self):
527 """ 528 Returns a printable representation of this instance. 529 530 @return: A descriptive string containing most of this instance fields 531 @rtype: str 532 """ 533 return u"%s(id=%s, name=%s, parent_id=%s, project_id=%s, status=%s, forced_status=%s)" % ( 534 self.__class__.__name__, 535 repr(self.id), 536 repr(self.name), 537 repr(self.parent_id), 538 repr(self.project_id), 539 repr(self.status), 540 repr(self.forced_status), 541 )
542
543 - def __str__(self):
544 """ 545 Coerces this instance to a string. 546 547 @return: The name field 548 @rtype: str 549 """ 550 return str(self.name)
551
552 553 -class Project(ORM_Base):
554 """ 555 Mapping class for the table «project». 556 """ 557 558 @hybrid_property
559 - def id(self):
560 """ 561 Read only accessor to prevent setting this field. 562 563 @return: Surrogate primary key 564 @rtype: int 565 """ 566 return self._id
567 568 @classmethod
569 - def query_tags(cls, session, tags={}):
570 """ 571 Overriden query method to apply tag-based custom filtering, on top of 572 common equality filter. 573 574 @param session: The database session in which to execute the query 575 @type session: Session 576 @param tags: Tag names and values to apply as a filter 577 @type tags: dict 578 @return: A query selecting Project instances, filtered by tags 579 @rtype: Query<L{Project}> 580 """ 581 q = session.query(cls) 582 if tags: 583 crit = literal(False) 584 for tag, value in tags.iteritems(): 585 if value == True: 586 crit |= (tables['tag_project'].c.tag == tag) 587 elif value == None or value != False : 588 crit |= (tables['tag_project'].c.tag == tag) & (tables['tag_project'].c.value == value) 589 q = q.join(tables['tag_project']) \ 590 .group_by(*[c for c in tables['project'].columns]) \ 591 .having(func.count(tables['project'].c.id) == len(tags)) \ 592 .filter(crit) 593 return q.from_self()
594
595 - def __repr__(self):
596 """ 597 Returns a printable representation of this instance. 598 599 @return: A descriptive string containing most of this instance fields 600 @rtype: str 601 """ 602 return u"%s(id=%s, name=%s)" % ( 603 self.__class__.__name__, 604 repr(self.id), 605 repr(self.name), 606 )
607
608 - def __str__(self):
609 """ 610 Coerces this instance to a string. 611 612 @return: The name field 613 @rtype: str 614 """ 615 return str(self.name)
616
617 618 -class TagElement(ORM_Base):
619 """ 620 Mapping class for the table «tagelement». 621 """ 622
623 - def __init__(self, *args, **kwargs):
624 """ 625 Constructor for TagElement instances. 626 Ensures that the «tag» field is specified. 627 628 @param args: Optional arguments to the constructor 629 @type args: tuple 630 @param kwargs: Optional keyword arguments to the constructor 631 @type kwargs: dict 632 """ 633 assert 'tag' in kwargs 634 assert kwargs['tag'] is not None 635 super(TagElement, self).__init__(*args, **kwargs)
636
637 638 -class TagProject(ORM_Base):
639 """ 640 Mapping class for the table «tagproject». 641 """ 642
643 - def __init__(self, *args, **kwargs):
644 """ 645 Constructor for TagProject instances. 646 Ensures that the «tag» field is specified. 647 648 @param args: Optional arguments to the constructor 649 @type args: tuple 650 @param kwargs: Optional keyword arguments to the constructor 651 @type kwargs: dict 652 """ 653 assert 'tag' in kwargs 654 assert kwargs['tag'] is not None 655 super(TagProject, self).__init__(*args, **kwargs)
656 657 658 # TODO: Reevaluate status cascading method. CTE? Triggers? 659