Sample Implementation: sync-notes¶
Below is a sample client-side implementation of a pysyncml client. Please note that it uses some of the more advanced features of pysyncml, and may therefore appear overwhelming. For a simpler general guide to implementing client-side SyncML adapters with pysyncml, please see the Implementing a SyncML Client guide.
Approach¶
The sync-notes program maintains the synchronization of a set of files in a given directory with a remote “Note” storage SyncML server. When launched, it scans the directory for any changes, such as new files, deleted files, or modified files and reports those changes to the local pysyncml.Context.Adapter. Then (and at user option), it synchronizes with a potentially pre-configured remote SyncML peer.
Code¶
1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 #------------------------------------------------------------------------------ 4 # file: $Id: notes.py 40 2012-07-22 18:53:36Z griff1n $ 5 # lib: pysyncml.cli.notes 6 # auth: griffin <griffin@uberdev.org> 7 # date: 2012/05/19 8 # copy: (C) CopyLoose 2012 UberDev <hardcore@uberdev.org>, No Rights Reserved. 9 #------------------------------------------------------------------------------ 10 11 ''' 12 A "note" synchronization adapter that stores notes in a 13 directory. Each note is stored in a separate file - although the 14 filename is tracked, it may be lost depending on the SyncML server 15 that is being contacted (if it supports content-type 16 "text/x-s4j-sifn", then the filename will be preserved). 17 18 This program is capable of running as either a client or as a server - 19 for now however, for any given note directory it is recommended to 20 only be used as one or the other, not both. When run in server mode, 21 it currently only supports a single optional authenticated username. 22 23 Brief first-time usage (see "--help" for details) as a client:: 24 25 sync-notes --remote https://example.com/funambol/ds \ 26 --username guest --password guest \ 27 NOTE_DIRECTORY 28 29 Follow-up synchronizations:: 30 31 sync-notes NOTE_DIRECTORY 32 33 Brief first-time usage as a server (listen port defaults to 80):: 34 35 sync-notes --server --listen 8080 NOTE_DIRECTORY 36 37 Follow-up synchronizations:: 38 39 sync-notes --server NOTE_DIRECTORY 40 41 ''' 42 43 #------------------------------------------------------------------------------ 44 # IMPORTS 45 #------------------------------------------------------------------------------ 46 47 import sys, os, re, time, uuid, hashlib, logging, getpass, pysyncml, traceback 48 import BaseHTTPServer, Cookie, urlparse, urllib 49 from optparse import OptionParser 50 from elementtree import ElementTree as ET 51 import sqlalchemy 52 from sqlalchemy import orm 53 from sqlalchemy.ext.declarative import declarative_base, declared_attr 54 from sqlalchemy.orm import sessionmaker 55 from sqlalchemy.orm.exc import NoResultFound 56 57 #------------------------------------------------------------------------------ 58 # GLOBALS 59 #------------------------------------------------------------------------------ 60 61 # create a default device ID that is fairly certain to be globally unique. for 62 # example, the IMEI number for a mobile phone. in this case, we are using 63 # uuid.getnode() which generates a hash based on the local MAC address. 64 # note that this is only used the first time `sync-notes` is used with a 65 # directory - after that, the device ID is retrieved from the sync database. 66 defaultDevID = 'pysyncml.cli.notes:%x:%x' % (uuid.getnode(), time.time()) 67 68 # todo: is having a global dbengine and dbsession really the best way?... 69 dbengine = None 70 dbsession = None 71 72 # setup a logger 73 log = logging.getLogger(__name__) 74 75 #------------------------------------------------------------------------------ 76 class LogFormatter(logging.Formatter): 77 levelString = { 78 logging.DEBUG: '[ ] DEBUG ', 79 logging.INFO: '[--] INFO ', 80 logging.WARNING: '[++] WARNING ', 81 logging.ERROR: '[**] ERROR ', 82 logging.CRITICAL: '[**] CRITICAL', 83 } 84 def __init__(self, logsource, *args, **kw): 85 logging.Formatter.__init__(self, *args, **kw) 86 self.logsource = logsource 87 def format(self, record): 88 msg = record.getMessage() 89 pfx = '%s|%s: ' % (LogFormatter.levelString[record.levelno], record.name) \ 90 if self.logsource else \ 91 '%s ' % (LogFormatter.levelString[record.levelno],) 92 if msg.find('\n') < 0: 93 return '%s%s' % (pfx, record.getMessage()) 94 return pfx + ('\n' + pfx).join(msg.split('\n')) 95 96 #------------------------------------------------------------------------------ 97 # STORAGE MODEL 98 #------------------------------------------------------------------------------ 99 100 # `sync-notes` uses sqlalchemy to store state information. there are two main 101 # ORM objects: 102 # Server: when `sync-notes` is used in server-mode, this stores details 103 # about what port to listen on, authentication info, default 104 # conflict policy, etc. 105 # Note: tracks note state so that changes can be detected. 106 107 #------------------------------------------------------------------------------ 108 class DatabaseObject(object): 109 @declared_attr 110 def __tablename__(cls): 111 return cls.__name__.lower() 112 id = sqlalchemy.Column(sqlalchemy.Integer, autoincrement=True, primary_key=True) 113 @classmethod 114 def q(cls, **kw): 115 return dbsession.query(cls).filter_by(**kw) 116 117 DatabaseObject = declarative_base(cls=DatabaseObject) 118 119 #------------------------------------------------------------------------------ 120 class Server(DatabaseObject): 121 port = sqlalchemy.Column(sqlalchemy.Integer) 122 username = sqlalchemy.Column(sqlalchemy.String) 123 password = sqlalchemy.Column(sqlalchemy.String) 124 policy = sqlalchemy.Column(sqlalchemy.Integer) 125 126 #------------------------------------------------------------------------------ 127 class Note(DatabaseObject, pysyncml.NoteItem): 128 # note: attributes inherited from NoteItem: id, extensions, name, body 129 # attributes then overriden by DatabaseObject (i hope): id 130 # and then attributes overriden here: name 131 # note: the `deleted` attribute exists only to ensure ID's are not recycled 132 # ugh. i need a better solution to that... 133 inode = sqlalchemy.Column(sqlalchemy.Integer, index=True) 134 name = sqlalchemy.Column(sqlalchemy.String) 135 sha256 = sqlalchemy.Column(sqlalchemy.String(64)) 136 deleted = sqlalchemy.Column(sqlalchemy.Boolean) 137 @classmethod 138 def q(cls, deleted=False, **kw): 139 if deleted is not None: 140 kw['deleted'] = deleted 141 return dbsession.query(cls).filter_by(**kw) 142 def __init__(self, *args, **kw): 143 self.deleted = False 144 DatabaseObject.__init__(self, *args, **kw) 145 # TODO: check this... 146 # NOTE: not calling NoteItem.__init__ as it can conflict with the 147 # sqlalchemy stuff done here... 148 # todo: is this really necessary?... 149 skw = dict() 150 skw.update(kw) 151 for key in self.__table__.c.keys(): 152 if key in skw: 153 del skw[key] 154 pysyncml.Ext.__init__(self, *args, **skw) 155 @orm.reconstructor 156 def __dbinit__(self): 157 # note: not calling ``NoteItem.__init__`` - see ``__init__`` notes. 158 pysyncml.Ext.__init__(self) 159 def __str__(self): 160 return 'Note "%s"' % (self.name,) 161 def __repr__(self): 162 return '<Note "%s": inode=%s; sha256=%s>' \ 163 % (self.name, '-' if self.inode is None else str(self.inode), 164 self.sha256) 165 def dump(self, stream, contentType, version, rootdir): 166 # TODO: convert this to a .body @property... 167 with open(os.path.join(rootdir, self.name), 'rb') as fp: 168 self.body = fp.read() 169 pysyncml.NoteItem.dump(self, stream, contentType, version) 170 self.body = None 171 return self 172 @classmethod 173 def load(cls, stream, contentType=None, version=None): 174 base = pysyncml.NoteItem.load(stream, contentType, version) 175 if contentType == pysyncml.TYPE_TEXT_PLAIN: 176 # remove special characters, windows illegal set: \/:*?"<>| 177 base.name = re.sub(r'[^a-zA-Z0-9,_+=!@#$%^&() -]+', '', base.name) 178 # collapse white space and replace with '_' 179 base.name = re.sub(r'\s+', '_', base.name) + '.txt' 180 ret = Note(name=base.name, sha256=hashlib.sha256(base.body).hexdigest()) 181 # temporarily storing the content in "body" attribute (until addItem() 182 # is called) 183 ret.body = base.body 184 return ret 185 186 #------------------------------------------------------------------------------ 187 # STORAGE CONTROLLER 188 #------------------------------------------------------------------------------ 189 190 #------------------------------------------------------------------------------ 191 def hashstream(hash, stream): 192 while True: 193 buf = stream.read(8192) 194 if len(buf) <= 0: 195 break 196 hash.update(buf) 197 return hash 198 199 #------------------------------------------------------------------------------ 200 class FilesystemNoteAgent(pysyncml.BaseNoteAgent): 201 ''' 202 The `FilesystemNoteAgent` is the implementation of the `pysyncml.Agent` 203 interface that provides the glue between the SyncML synchronization engine 204 (i.e. the `pysyncml.Adapter`) and the local datastore. This allows the 205 SyncML Adapter to be agnostic about how the items are actually stored. 206 207 It subclasses `pysyncml.BaseNoteAgent` (instead of `pysyncml.Agent`) in 208 order to take advantage of the "note"-aware functionality already provided 209 by pysyncml. 210 ''' 211 212 #---------------------------------------------------------------------------- 213 def __init__(self, root, ignoreRoot=None, ignoreAll=None, 214 syncstore=None, *args, **kw): 215 ''' 216 The `FilesystemNoteAgent` constructor accepts the following parameters: 217 218 :param root: 219 220 (required) the root directory that the notes are stored in. 221 222 :param ignoreRoot: 223 224 (optional) a regular expression string that specifies files that should 225 be ignored in the root directory (but not in subdirectories). This is 226 primarily useful so that synchronization state can be stored in a SQLite 227 database within the note directory itself without being itself sync\'d. 228 229 :param ignoreAll: 230 231 (optional) similar to `ignoreRoot` except this expression is matched 232 against all files, even files in sub-directories. This is primarily 233 useful if you want to ignore temporary files, such as "~" emacs files 234 and dot-files. 235 236 :param syncstore: 237 238 (optional) specifies the `pysyncml.Store` object that represents the 239 SyncML datastore within the pysyncml framework. If specified, will 240 automatically scan for local changes and report them to the store. 241 If not specified, the caller must eventually manually call the 242 :meth:`scan` methed. 243 244 ''' 245 super(FilesystemNoteAgent, self).__init__(*args, **kw) 246 self.rootdir = root 247 self.ignoreRoot = re.compile(ignoreRoot) if ignoreRoot is not None else None 248 self.ignoreAll = re.compile(ignoreAll) if ignoreAll is not None else None 249 if syncstore is not None: 250 self.scan(syncstore) 251 # the pysyncml.BaseNoteAgent specifies a set of content-types that it 252 # knows how to serialize/deserialize, which includes version "1.0" and 253 # "1.1" of text/plain. it is fairly unclear what the difference is, 254 # but it causes problems when sync'ing with funambol, because it does 255 # not like the multiple "VerCT" nodes that results from it. thus overriding 256 # BaseNoteAgent here for funambol compatibility. 257 # TODO: perhaps the pysyncml framework can be reworked to issue multiple 258 # <Tx> or <Rx> nodes instead of multiple <VerCT> nodes?... 259 self.contentTypes = [ 260 pysyncml.ContentTypeInfo(pysyncml.TYPE_SIF_NOTE, '1.1', True), 261 pysyncml.ContentTypeInfo(pysyncml.TYPE_SIF_NOTE, '1.0'), 262 # pysyncml.ContentTypeInfo(pysyncml.TYPE_TEXT_PLAIN, ['1.1', '1.0']), 263 pysyncml.ContentTypeInfo(pysyncml.TYPE_TEXT_PLAIN, '1.0'), 264 ] 265 266 #---------------------------------------------------------------------------- 267 def scan(self, store): 268 ''' 269 Scans the local notes for changes (either additions, modifications or 270 deletions) and reports them to the `store` object, which is expected to 271 implement the :class:`pysyncml.Store` interface. 272 ''' 273 # todo: this scan assumes that the note index (not the bodies) will 274 # comfortably fit in memory... this is probably a good assumption, 275 # but ideally it would not need to depend on that. 276 # `reg` is a registry of notes, along with the note's state (which can 277 # be one of OK, ADDED, MODIFIED, or DELETED). the _scandir() populates 278 # it with all the files currently in the root directory, and _scanindex() 279 # searches for entries that are missing (i.e. the files have been removed). 280 reg = dict() 281 if store.peer is not None: 282 # if the store is already bound to a remote peer, then we may have 283 # already reported some changes (for example, if option "--local" was 284 # used). therefore pre-populate the registry with any changes that we 285 # have already registered. 286 reg = dict((c.itemID, c.state) for c in store.peer.getRegisteredChanges()) 287 self._scandir('.', store, reg) 288 self._scanindex(store, reg) 289 290 #---------------------------------------------------------------------------- 291 def _scanindex(self, store, reg): 292 # IMPORTANT: this assumes that _scandir has completed and that all 293 # moved files have been recorded, etc. this function 294 # then searches for deleted files... 295 # TODO: this is somewhat of a simplistic algorithm... this 296 # comparison should be done at the same time as the dirwalk 297 # to detect more complex changes such as: files "a" and "b" 298 # are synced. then "a" is deleted and "b" is moved to "a"... 299 # the current algorithm would incorrectly record that as a 300 # non-syncing change to "b", and "a" would not be 301 # deleted. or something like that. 302 for note in Note.q(): 303 if str(note.id) in reg: 304 continue 305 log.debug('locally deleted note: %s', note.name) 306 note.deleted = True 307 store.registerChange(note.id, pysyncml.ITEM_DELETED) 308 reg[str(note.id)] = pysyncml.ITEM_DELETED 309 310 #---------------------------------------------------------------------------- 311 def _scandir(self, dirname, store, reg): 312 curdir = os.path.normcase(os.path.normpath(os.path.join(self.rootdir, dirname))) 313 log.debug('scanning directory "%s"...', curdir) 314 for name in os.listdir(curdir): 315 # apply the "ignoreRoot" and "ignoreAll" regex's - this is primarily to 316 # ignore the pysyncml storage file in the root directory 317 if dirname == '.': 318 if self.ignoreRoot is not None and self.ignoreRoot.match(name): 319 continue 320 if self.ignoreAll is not None and self.ignoreAll.match(name): 321 continue 322 path = os.path.join(curdir, name) 323 if os.path.islink(path): 324 # todo: should i follow?... 325 continue 326 if os.path.isfile(path): 327 self._scanfile(path, os.path.join(dirname, name), store, reg) 328 continue 329 if os.path.isdir(path): 330 # and recurse!... 331 self._scandir(os.path.join(dirname, name), store, reg) 332 333 #---------------------------------------------------------------------------- 334 def _scanfile(self, path, name, store, reg): 335 log.debug('analyzing file "%s"...', path) 336 inode = os.stat(path).st_ino 337 name = os.path.normpath(name) 338 note = None 339 chksum = None 340 try: 341 note = Note.q(name=name).one() 342 log.debug(' matched item %d by name ("%s")', note.id, note.name) 343 except NoResultFound: 344 try: 345 with open(path,'rb') as fp: 346 chksum = hashstream(hashlib.sha256(), fp).hexdigest() 347 note = Note.q(sha256=chksum).one() 348 log.debug(' matched item %d by checksum ("%s")', note.id, note.sha256) 349 except NoResultFound: 350 try: 351 note = Note.q(inode=inode).one() 352 log.debug(' matched item %d by inode (%d)', note.id, note.inode) 353 if note.name != name and note.sha256 != chksum: 354 log.debug(' looks like the inode was recycled... dropping match') 355 raise NoResultFound() 356 except NoResultFound: 357 log.debug('locally added note: %s', path) 358 note = Note(inode=inode, name=name, sha256=chksum) 359 dbsession.add(note) 360 dbsession.flush() 361 store.registerChange(note.id, pysyncml.ITEM_ADDED) 362 reg[str(note.id)] = pysyncml.ITEM_ADDED 363 return 364 if inode != note.inode: 365 log.debug('locally recreated note with new inode: %d => %d (not synchronized)', note.inode, inode) 366 note.inode = inode 367 if name != note.name: 368 # todo: a rename should prolly trigger an update... the lowest 369 # common denominator (text/plain) does not synchronize 370 # names though... 371 log.debug('locally renamed note: %s => %s (not synchronized)', note.name, name) 372 note.name = name 373 # TODO: i *should* store the last-modified and check that instead of 374 # opening and sha256-digesting every single file... if that gets 375 # implemented, however, there should be a "deep check" option 376 # that still checks the checksum since the last-modified can be 377 # unreliable. 378 if chksum is None: 379 with open(path,'rb') as fp: 380 chksum = hashstream(hashlib.sha256(), fp).hexdigest() 381 modified = None 382 if chksum != note.sha256: 383 modified = 'content' 384 note.sha256 = chksum 385 if modified is not None: 386 log.debug('locally modified note: %s (%s)', path, modified) 387 if reg.get(str(note.id)) == pysyncml.ITEM_ADDED: 388 return 389 store.registerChange(note.id, pysyncml.ITEM_MODIFIED) 390 reg[str(note.id)] = pysyncml.ITEM_MODIFIED 391 else: 392 reg[str(note.id)] = pysyncml.ITEM_OK 393 394 #---------------------------------------------------------------------------- 395 def getAllItems(self): 396 for note in Note.q(): 397 yield note 398 399 #---------------------------------------------------------------------------- 400 def dumpItem(self, item, stream, contentType=None, version=None): 401 item.dump(stream, contentType, version, self.rootdir) 402 403 #---------------------------------------------------------------------------- 404 def loadItem(self, stream, contentType=None, version=None): 405 return Note.load(stream, contentType, version) 406 407 #---------------------------------------------------------------------------- 408 def getItem(self, itemID, includeDeleted=False): 409 if includeDeleted: 410 return Note.q(id=int(itemID), deleted=None).one() 411 return Note.q(id=int(itemID)).one() 412 413 #---------------------------------------------------------------------------- 414 def addItem(self, item): 415 path = os.path.join(self.rootdir, item.name) 416 if '.' not in item.name: 417 pbase = item.name 418 psufx = '' 419 else: 420 pbase = item.name[:item.name.rindex('.')] 421 psufx = item.name[item.name.rindex('.'):] 422 count = 0 423 while os.path.exists(path): 424 count += 1 425 item.name = '%s(%d)%s' % (pbase, count, psufx) 426 path = os.path.join(self.rootdir, item.name) 427 with open(path, 'wb') as fp: 428 fp.write(item.body) 429 item.inode = os.stat(path).st_ino 430 delattr(item, 'body') 431 dbsession.add(item) 432 dbsession.flush() 433 log.debug('added: %s', item) 434 return item 435 436 #---------------------------------------------------------------------------- 437 def replaceItem(self, item): 438 curitem = self.getItem(item.id) 439 path = os.path.join(self.rootdir, curitem.name) 440 with open(path, 'wb') as fp: 441 fp.write(item.body) 442 curitem.inode = os.stat(path).st_ino 443 curitem.sha256 = hashlib.sha256(item.body).hexdigest() 444 delattr(item, 'body') 445 dbsession.flush() 446 log.debug('updated: %s', curitem) 447 448 #---------------------------------------------------------------------------- 449 def deleteItem(self, itemID): 450 item = self.getItem(itemID) 451 path = os.path.join(self.rootdir, item.name) 452 if os.path.exists(path): 453 os.unlink(path) 454 item.deleted = True 455 # note: writing log before actual delete as otherwise object is invalid 456 log.debug('deleted: %s', item) 457 # note: not deleting from DB to ensure ID's are not recycled... ugh. i 458 # need a better solution to that... the reason that ID's must not 459 # be recycled is that soft-deletes will delete the object locally, 460 # but it's ID must not be used with a new object as otherwise this 461 # will result in conflicts on the server-side... 462 # dbsession.delete(item) 463 464 #------------------------------------------------------------------------------ 465 # ADAPTER INTEGRATION 466 #------------------------------------------------------------------------------ 467 468 #------------------------------------------------------------------------------ 469 def makeAdapter(opts): 470 ''' 471 Creates a tuple of ( Context, Adapter, Agent ) based on the options 472 specified by `opts`. The Context is the pysyncml.Context created for 473 the storage location specified in `opts`, the Adapter is a newly 474 created Adapter if a previously created one was not found, and Agent 475 is a pysyncml.Agent implementation that is setup to interface 476 between Adapter and the local note storage. 477 ''' 478 479 # create a new pysyncml.Context. the main function that this provides is 480 # to give the Adapter a storage engine to store state information across 481 # synchronizations. 482 483 context = pysyncml.Context(storage='sqlite:///%(rootdir)s%(storageName)s' % 484 dict(rootdir=opts.rootdir, storageName=opts.storageName), 485 owner=None, autoCommit=True) 486 487 # create an Adapter from the current context. this will either create 488 # a new adapter, or load the current local adapter for the specified 489 # context storage location. if it is new, then lots of required 490 # information (such as device info) will not be set, so we need to 491 # check that and specify it if missing. 492 493 adapter = context.Adapter() 494 495 if opts.name is not None: 496 adapter.name = opts.name + ' (pysyncml.cli.notes SyncML Adapter)' 497 498 # TODO: stop ignoring ``opts.remoteUri``... (the router must first support 499 # manual routes...) 500 # if opts.remoteUri is not None: 501 # adapter.router.addRoute(agent.uri, opts.remoteUri) 502 503 if adapter.devinfo is None: 504 log.info('adapter has no device info - registering new device') 505 else: 506 if opts.devid is not None and opts.devid != adapter.devinfo.devID: 507 log.info('adapter has invalid device ID - overwriting with new device info') 508 adapter.devinfo = None 509 510 if adapter.devinfo is None: 511 # setup some information about the local device, most importantly the 512 # device ID, which the remote peer will use to uniquely identify this peer 513 adapter.devinfo = context.DeviceInfo( 514 devID = opts.devid or defaultDevID, 515 devType = pysyncml.DEVTYPE_SERVER if opts.server else pysyncml.DEVTYPE_WORKSTATION, 516 softwareVersion = '0.1', 517 manufacturerName = 'pysyncml', 518 modelName = 'pysyncml.cli.notes', 519 # TODO: adding this for funambol-compatibility... 520 hierarchicalSync = False, 521 ) 522 523 if not opts.server: 524 525 # servers don't have a fixed peer; i.e. the SyncML message itself 526 # defines which peer is connecting. 527 528 if adapter.peer is None: 529 if opts.remote is None: 530 opts.remote = raw_input('SyncML remote URL: ') 531 if opts.username is None: 532 opts.username = raw_input('SyncML remote username (leave empty if none): ') 533 if len(opts.username) <= 0: 534 opts.username = None 535 log.info('adapter has no remote info - registering new remote adapter') 536 else: 537 if opts.remote is not None: 538 if opts.remote != adapter.peer.url \ 539 or opts.username != adapter.peer.username \ 540 or opts.password != adapter.peer.password: 541 #or opts.password is not None: 542 log.info('adapter has invalid or rejected remote info - overwriting with new remote info') 543 adapter.peer = None 544 545 if adapter.peer is None: 546 auth = None 547 if opts.username is not None: 548 auth = pysyncml.NAMESPACE_AUTH_BASIC 549 if opts.password is None: 550 opts.password = getpass.getpass('SyncML remote password: ') 551 # setup the remote connection parameters, if not already stored in 552 # the adapter sync tables or the URL has changed. 553 adapter.peer = context.RemoteAdapter( 554 url = opts.remote, 555 auth = auth, 556 username = opts.username, 557 password = opts.password, 558 ) 559 560 # add a datastore attached to the URI "note". the actual value of 561 # the URI is irrelevant - it is only an identifier for this item 562 # synchronization channel. it must be unique within this adapter 563 # and must stay consistent across synchronizations. 564 565 # TODO: this check should be made redundant... (ie. once the 566 # implementation of Store.merge() is fixed this will 567 # become a single "addStore()" call without the check first). 568 if 'note' in adapter.stores: 569 store = adapter.stores['note'] 570 else: 571 store = adapter.addStore(context.Store( 572 uri = 'note', 573 displayName = opts.name, 574 # TODO: adding this for funambol-compatibility... 575 maxObjSize = None)) 576 577 # create a new agent, which will scan the files stored in the root directory, 578 # looking for changed files, new files, and deleted files. 579 580 agent = FilesystemNoteAgent(opts.rootdir, 581 ignoreRoot='^(%s)$' % (re.escape(opts.syncdir),), 582 syncstore=store) 583 584 if store.peer is None: 585 if opts.local: 586 print 'no pending local changes (not associated yet)' 587 else: 588 log.info('no pending local changes (not associated yet)') 589 else: 590 changes = list(store.peer.getRegisteredChanges()) 591 if len(changes) <= 0: 592 if opts.local: 593 print 'no pending local changes to synchronize' 594 else: 595 log.info('no pending local changes to synchronize') 596 else: 597 if opts.local: 598 print 'pending local changes:' 599 else: 600 log.info('pending local changes:') 601 for c in changes: 602 item = agent.getItem(c.itemID, includeDeleted=True) 603 msg = ' - %s: %s' % (item, pysyncml.state2string(c.state)) 604 if opts.local: 605 print msg 606 else: 607 log.info(msg) 608 609 store.agent = agent 610 611 return (context, adapter, agent) 612 613 #------------------------------------------------------------------------------ 614 def main_server(opts): 615 try: 616 sconf = Server.q().one() 617 except NoResultFound: 618 log.debug('no prior server - creating new server configuration') 619 sconf = Server() 620 dbsession.add(sconf) 621 622 if opts.listen is not None: 623 sconf.port = opts.listen 624 if sconf.port is None: 625 sconf.port = 80 626 if opts.username is not None: 627 if opts.password is None: 628 opts.password = getpass.getpass('SyncML remote password: ') 629 sconf.username = opts.username 630 sconf.password = opts.password 631 # todo: set server policy when pysyncml supports it... 632 633 dbsession.commit() 634 sessions = dict() 635 636 class Handler(BaseHTTPServer.BaseHTTPRequestHandler): 637 def version_string(self): 638 return 'pysyncml/' + pysyncml.versionString 639 def _parsePathParameters(self): 640 self.path_params = dict() 641 pairs = [e.split('=', 1) for e in self.path.split(';')[1:]] 642 for pair in pairs: 643 key = urllib.unquote_plus(pair[0]) 644 if len(pair) < 2: 645 self.path_params[key] = True 646 else: 647 self.path_params[key] = urllib.unquote_plus(pair[1]) 648 def do_POST(self): 649 self._parsePathParameters() 650 log.debug('handling POST request to "%s" (parameters: %r)', self.path, self.path_params) 651 sid = None 652 self.session = None 653 if 'Cookie' in self.headers: 654 cks = Cookie.SimpleCookie(self.headers["Cookie"]) 655 if 'sessionid' in cks: 656 sid = cks['sessionid'].value 657 if sid in sessions: 658 self.session = sessions[sid] 659 self.session.count += 1 660 else: 661 sid = None 662 if sid is None: 663 log.debug('no valid session ID found in cookies - checking path parameters') 664 sid = self.path_params.get('sessionid') 665 if sid in sessions: 666 self.session = sessions[sid] 667 self.session.count += 1 668 else: 669 sid = None 670 if sid is None: 671 while sid is None or sid in sessions: 672 sid = str(uuid.uuid4()) 673 log.debug('request without session ID: creating new session "%s" and setting cookie', sid) 674 self.session = pysyncml.adict(id=sid, count=1, syncml=pysyncml.Session()) 675 sessions[sid] = self.session 676 log.debug('session: id=%s, count=%d', self.session.id, self.session.count) 677 try: 678 response = self.handleRequest() 679 except Exception, e: 680 self.send_response(500) 681 self.end_headers() 682 self.wfile.write(traceback.format_exc()) 683 return 684 self.send_response(200) 685 if self.session.count <= 1: 686 cks = Cookie.SimpleCookie() 687 cks['sessionid'] = sid 688 self.send_header('Set-Cookie', cks.output(header='')) 689 if response.contentType is not None: 690 self.send_header('Content-Type', response.contentType) 691 self.send_header('Content-Length', str(len(response.body))) 692 self.send_header('X-PySyncML-Session', 'id=%s, count=%d' % (self.session.id, self.session.count)) 693 self.end_headers() 694 self.wfile.write(response.body) 695 def handleRequest(self): 696 global dbsession 697 dbsession = sessionmaker(bind=dbengine)() 698 context, adapter, agent = makeAdapter(opts) 699 # TODO: enforce authentication info... 700 # self.assertEqual(adict(auth=pysyncml.NAMESPACE_AUTH_BASIC, 701 # username='guest', password='guest'), 702 # pysyncml.Context.getAuthInfo(request, None)) 703 clen = 0 704 if 'Content-Length' in self.headers: 705 clen = int(self.headers['Content-Length']) 706 request = pysyncml.adict(headers=dict((('content-type', 'application/vnd.syncml+xml'),)), 707 body=self.rfile.read(clen)) 708 self.session.syncml.effectiveID = pysyncml.Context.getTargetID(request) 709 # todo: this should be a bit more robust... 710 urlparts = list(urlparse.urlsplit(self.session.syncml.effectiveID)) 711 if self.path_params.get('sessionid') != self.session.id: 712 urlparts[2] += ';sessionid=' + self.session.id 713 self.session.syncml.returnUrl = urlparse.SplitResult(*urlparts).geturl() 714 response = pysyncml.Response() 715 self.stats = adapter.handleRequest(self.session.syncml, request, response) 716 dbsession.commit() 717 return response 718 719 server = BaseHTTPServer.HTTPServer(('', sconf.port), Handler) 720 log.info('starting server on port %d', sconf.port) 721 server.serve_forever() 722 723 return 0 724 725 #------------------------------------------------------------------------------ 726 def main_client(opts): 727 728 context, adapter, agent = makeAdapter(opts) 729 730 if opts.local: 731 context.save() 732 dbsession.commit() 733 return 0 734 735 mode = { 736 'sync': pysyncml.SYNCTYPE_TWO_WAY, 737 'full': pysyncml.SYNCTYPE_SLOW_SYNC, 738 'pull': pysyncml.SYNCTYPE_ONE_WAY_FROM_SERVER, 739 'push': pysyncml.SYNCTYPE_ONE_WAY_FROM_CLIENT, 740 'pull-over': pysyncml.SYNCTYPE_REFRESH_FROM_SERVER, 741 'push-over': pysyncml.SYNCTYPE_REFRESH_FROM_CLIENT, 742 }[opts.mode] 743 744 if opts.config: 745 sys.stdout.write('Note SyncML adapter configuration:\n') 746 adapter.describe(pysyncml.IndentStream(sys.stdout, ' ')) 747 else: 748 stats = adapter.sync(mode=mode) 749 if not opts.quiet: 750 pysyncml.describeStats(stats, sys.stdout, title='Synchronization Summary') 751 752 context.save() 753 dbsession.commit() 754 755 return 0 756 757 #------------------------------------------------------------------------------ 758 def main(): 759 760 #---------------------------------------------------------------------------- 761 # setup program parameters 762 763 cli = OptionParser(usage='%prog [options] DIRNAME', 764 version='%prog ' + pysyncml.versionString, 765 ) 766 767 cli.add_option('-v', '--verbose', 768 dest='verbose', default=0, action='count', 769 help='enable verbose output to STDERR, mostly for diagnotic' 770 ' purposes (multiple invocations increase verbosity).') 771 772 cli.add_option('-q', '--quiet', 773 dest='quiet', default=False, action='store_true', 774 help='do not display sync summary') 775 776 cli.add_option('-c', '--config', 777 dest='config', default=False, action='store_true', 778 help='configure the local SyncML adapter, display a summary' 779 ' and exit without actually syncronizing') 780 781 cli.add_option('-l', '--local', 782 dest='local', default=False, action='store_true', 783 help='display the pending local changes') 784 785 cli.add_option('-i', '--id', 786 dest='devid', default=None, action='store', 787 help='overrides the default device ID, either the store' 788 ' value from a previous sync or the generated default' 789 ' (currently "%s" - generated based on local MAC address' 790 ' and current time)' 791 % (defaultDevID,)) 792 793 cli.add_option('-n', '--name', 794 dest='name', default=None, action='store', 795 help='sets the local note adapter/store name (no default)') 796 797 cli.add_option('-m', '--mode', 798 dest='mode', default='sync', action='store', 799 help='set the synchronization mode - can be one of "sync"' 800 ' (for two-way synchronization), "full" (for a complete' 801 ' re-synchronization), "pull" (for fetching remote' 802 ' changes only), "push" (for pushing local changes only),' 803 ' or "pull-over" (to obliterate the local data and' 804 ' download the remote data) or "push-over" (to obliterate' 805 ' the remote data and upload the local data); the default' 806 ' is "%default".') 807 808 cli.add_option('-r', '--remote', 809 dest='remote', default=None, action='store', 810 help='specifies the remote URL of the SyncML synchronization' 811 ' server - only required if the target ``DIRNAME`` has never' 812 ' been synchronized, or the synchronization meta information' 813 ' was lost.') 814 815 cli.add_option('-R', '--remote-uri', 816 dest='remoteUri', default=None, action='store', 817 help='specifies the remote URI of the note datastore. if' 818 ' left unspecified, pysyncml will attempt to identify it' 819 ' automatically.') 820 821 cli.add_option('-s', '--server', 822 dest='server', default=False, action='store_true', 823 help='enables HTTP server mode') 824 825 cli.add_option('-L', '--listen', 826 dest='listen', default=None, action='store', type='int', 827 help='specifies the port to listen on for server mode' 828 ' (implies --server and defaults to port 80)') 829 830 # todo: add a "policy" to configure how the server mode should handle 831 # conflicts... 832 833 cli.add_option('-u', '--username', 834 dest='username', default=None, action='store', 835 help='specifies the remote server username to log in with' 836 ' (in client mode) or to require authorization for (in' 837 ' server mode)') 838 839 cli.add_option('-p', '--password', 840 dest='password', default=None, action='store', 841 help='specifies the remote server password to log in with' 842 ' in client mode (if "--remote" and "--username" is' 843 ' specified, but not "--password", the password will be' 844 ' prompted for to avoid leaking the password into the' 845 ' local hosts environment, which is the recommended' 846 ' approach). in server mode, specifies the password for' 847 ' the required username (a present "--username" and missing' 848 ' "--password" is handled the same way as in client' 849 ' mode)') 850 851 (opts, args) = cli.parse_args() 852 853 if len(args) != 1: 854 cli.error('expected exactly one argument DIRNAME - please see "--help" for details.') 855 856 # setup logging (based on requested verbosity) 857 rootlog = logging.getLogger() 858 handler = logging.StreamHandler(sys.stderr) 859 handler.setFormatter(LogFormatter(opts.verbose >= 2)) 860 rootlog.addHandler(handler) 861 if opts.verbose >= 3: rootlog.setLevel(logging.DEBUG) 862 elif opts.verbose == 2: rootlog.setLevel(logging.INFO) 863 elif opts.verbose == 1: rootlog.setLevel(logging.INFO) 864 else: rootlog.setLevel(logging.FATAL) 865 866 # setup storage locations for note tracking and pysyncml internal data 867 opts.syncdir = '.sync' 868 opts.storageName = os.path.join(opts.syncdir, 'syncml.db') 869 opts.indexStorage = os.path.join(opts.syncdir, 'index.db') 870 opts.rootdir = args[0] 871 if not opts.rootdir.startswith('/') and not opts.rootdir.startswith('.'): 872 opts.rootdir = './' + opts.rootdir 873 if not opts.rootdir.endswith('/'): 874 opts.rootdir += '/' 875 876 if not os.path.isdir(opts.rootdir): 877 cli.error('note root directory "%s" does not exist' % (opts.rootdir,)) 878 879 if not os.path.isdir(os.path.join(opts.rootdir, opts.syncdir)): 880 os.makedirs(os.path.join(opts.rootdir, opts.syncdir)) 881 882 #---------------------------------------------------------------------------- 883 # prepare storage 884 885 global dbengine, dbsession 886 dbengine = sqlalchemy.create_engine('sqlite:///%s%s' % (opts.rootdir, opts.indexStorage)) 887 dbsession = sessionmaker(bind=dbengine)() 888 # TODO: how to detect if my schema has changed?... 889 if not os.path.isfile('%s%s' % (opts.rootdir, opts.indexStorage)): 890 DatabaseObject.metadata.create_all(dbengine) 891 892 if opts.server or opts.listen is not None: 893 opts.server = True 894 return main_server(opts) 895 896 return main_client(opts) 897 898 #------------------------------------------------------------------------------ 899 if __name__ == '__main__': 900 sys.exit(main()) 901 902 #------------------------------------------------------------------------------ 903 # end of $Id: notes.py 40 2012-07-22 18:53:36Z griff1n $ 904 #------------------------------------------------------------------------------