Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from inspect import getmembers, getmro, isclass 

2from pkgutil import iter_modules 

3import sys 

4 

5from venusian.advice import getFrameInfo 

6 

7ATTACH_ATTR = "__venusian_callbacks__" 

8LIFTONLY_ATTR = "__venusian_liftonly_callbacks__" 

9 

10 

11class Scanner(object): 

12 def __init__(self, **kw): 

13 self.__dict__.update(kw) 

14 

15 def scan(self, package, categories=None, onerror=None, ignore=None): 

16 """ Scan a Python package and any of its subpackages. All 

17 top-level objects will be considered; those marked with 

18 venusian callback attributes related to ``category`` will be 

19 processed. 

20 

21 The ``package`` argument should be a reference to a Python 

22 package or module object. 

23 

24 The ``categories`` argument should be sequence of Venusian 

25 callback categories (each category usually a string) or the 

26 special value ``None`` which means all Venusian callback 

27 categories. The default is ``None``. 

28 

29 The ``onerror`` argument should either be ``None`` or a callback 

30 function which behaves the same way as the ``onerror`` callback 

31 function described in 

32 http://docs.python.org/library/pkgutil.html#pkgutil.walk_packages . 

33 By default, during a scan, Venusian will propagate all errors that 

34 happen during its code importing process, including 

35 :exc:`ImportError`. If you use a custom ``onerror`` callback, you 

36 can change this behavior. 

37  

38 Here's an example ``onerror`` callback that ignores 

39 :exc:`ImportError`:: 

40 

41 import sys 

42 def onerror(name): 

43 if not issubclass(sys.exc_info()[0], ImportError): 

44 raise # reraise the last exception 

45 

46 The ``name`` passed to ``onerror`` is the module or package dotted 

47 name that could not be imported due to an exception. 

48 

49 .. versionadded:: 1.0 

50 the ``onerror`` callback 

51 

52 The ``ignore`` argument allows you to ignore certain modules, 

53 packages, or global objects during a scan. It should be a sequence 

54 containing strings and/or callables that will be used to match 

55 against the full dotted name of each object encountered during a 

56 scan. The sequence can contain any of these three types of objects: 

57 

58 - A string representing a full dotted name. To name an object by 

59 dotted name, use a string representing the full dotted name. For 

60 example, if you want to ignore the ``my.package`` package *and any 

61 of its subobjects or subpackages* during the scan, pass 

62 ``ignore=['my.package']``. 

63 

64 - A string representing a relative dotted name. To name an object 

65 relative to the ``package`` passed to this method, use a string 

66 beginning with a dot. For example, if the ``package`` you've 

67 passed is imported as ``my.package``, and you pass 

68 ``ignore=['.mymodule']``, the ``my.package.mymodule`` mymodule *and 

69 any of its subobjects or subpackages* will be omitted during scan 

70 processing. 

71 

72 - A callable that accepts a full dotted name string of an object as 

73 its single positional argument and returns ``True`` or ``False``. 

74 For example, if you want to skip all packages, modules, and global 

75 objects with a full dotted path that ends with the word "tests", you 

76 can use ``ignore=[re.compile('tests$').search]``. If the callable 

77 returns ``True`` (or anything else truthy), the object is ignored, 

78 if it returns ``False`` (or anything else falsy) the object is not 

79 ignored. *Note that unlike string matches, ignores that use a 

80 callable don't cause submodules and subobjects of a module or 

81 package represented by a dotted name to also be ignored, they match 

82 individual objects found during a scan, including packages, 

83 modules, and global objects*. 

84 

85 You can mix and match the three types of strings in the list. For 

86 example, if the package being scanned is ``my``, 

87 ``ignore=['my.package', '.someothermodule', 

88 re.compile('tests$').search]`` would cause ``my.package`` (and all 

89 its submodules and subobjects) to be ignored, ``my.someothermodule`` 

90 to be ignored, and any modules, packages, or global objects found 

91 during the scan that have a full dotted name that ends with the word 

92 ``tests`` to be ignored. 

93 

94 Note that packages and modules matched by any ignore in the list will 

95 not be imported, and their top-level code will not be run as a result. 

96 

97 A string or callable alone can also be passed as ``ignore`` without a 

98 surrounding list. 

99  

100 .. versionadded:: 1.0a3 

101 the ``ignore`` argument 

102 """ 

103 

104 pkg_name = package.__name__ 

105 

106 if ignore is not None and ( 

107 isinstance(ignore, str) or not hasattr(ignore, "__iter__") 

108 ): 

109 ignore = [ignore] 

110 elif ignore is None: 

111 ignore = [] 

112 

113 # non-leading-dotted name absolute object name 

114 str_ignores = [ign for ign in ignore if isinstance(ign, str)] 

115 # leading dotted name relative to scanned package 

116 rel_ignores = [ign for ign in str_ignores if ign.startswith(".")] 

117 # non-leading dotted names 

118 abs_ignores = [ign for ign in str_ignores if not ign.startswith(".")] 

119 # functions, e.g. re.compile('pattern').search 

120 callable_ignores = [ign for ign in ignore if callable(ign)] 

121 

122 def _ignore(fullname): 

123 for ign in rel_ignores: 

124 if fullname.startswith(pkg_name + ign): 

125 return True 

126 for ign in abs_ignores: 

127 # non-leading-dotted name absolute object name 

128 if fullname.startswith(ign): 

129 return True 

130 for ign in callable_ignores: 

131 if ign(fullname): 

132 return True 

133 return False 

134 

135 def invoke(mod_name, name, ob): 

136 

137 fullname = mod_name + "." + name 

138 

139 if _ignore(fullname): 

140 return 

141 

142 category_keys = categories 

143 try: 

144 # Some metaclasses do insane things when asked for an 

145 # ``ATTACH_ATTR``, like not raising an AttributeError but 

146 # some other arbitary exception. Some even shittier 

147 # introspected code lets us access ``ATTACH_ATTR`` far but 

148 # barfs on a second attribute access for ``attached_to`` 

149 # (still not raising an AttributeError, but some other 

150 # arbitrary exception). Finally, the shittiest code of all 

151 # allows the attribute access of the ``ATTACH_ATTR`` *and* 

152 # ``attached_to``, (say, both ``ob.__getattr__`` and 

153 # ``attached_categories.__getattr__`` returning a proxy for 

154 # any attribute access), which either a) isn't callable or b) 

155 # is callable, but, when called, shits its pants in an 

156 # potentially arbitrary way (although for b, only TypeError 

157 # has been seen in the wild, from PyMongo). Thus the 

158 # catchall except: return here, which in any other case would 

159 # be high treason. 

160 attached_categories = getattr(ob, ATTACH_ATTR) 

161 if not attached_categories.attached_to(mod_name, name, ob): 

162 return 

163 except: 

164 return 

165 if category_keys is None: 

166 category_keys = list(attached_categories.keys()) 

167 try: 

168 # When metaclasses return proxies for any attribute access 

169 # the list may contain keys of different types which might 

170 # not be sortable. In that case we can just return, 

171 # because we're not dealing with a proper venusian 

172 # callback. 

173 category_keys.sort() 

174 except TypeError: # pragma: no cover 

175 return 

176 for category in category_keys: 

177 callbacks = attached_categories.get(category, []) 

178 try: 

179 # Metaclasses might trick us by reaching this far and then 

180 # fail with too little values to unpack. 

181 for callback, cb_mod_name, liftid, scope in callbacks: 

182 if cb_mod_name != mod_name: 

183 # avoid processing objects that were imported into 

184 # this module but were not actually defined there 

185 continue 

186 callback(self, name, ob) 

187 except ValueError: # pragma: nocover 

188 continue 

189 

190 for name, ob in getmembers(package): 

191 # whether it's a module or a package, we need to scan its 

192 # members; walk_packages only iterates over submodules and 

193 # subpackages 

194 invoke(pkg_name, name, ob) 

195 

196 if hasattr(package, "__path__"): # package, not module 

197 results = walk_packages( 

198 package.__path__, 

199 package.__name__ + ".", 

200 onerror=onerror, 

201 ignore=_ignore, 

202 ) 

203 

204 for importer, modname, ispkg in results: 

205 loader = importer.find_module(modname) 

206 if loader is not None: # happens on pypy with orphaned pyc 

207 try: 

208 get_filename = getattr(loader, "get_filename", None) 

209 if get_filename is None: # pragma: nocover 

210 get_filename = loader._get_filename 

211 try: 

212 fn = get_filename(modname) 

213 except TypeError: # pragma: nocover 

214 fn = get_filename() 

215 

216 # NB: use __import__(modname) rather than 

217 # loader.load_module(modname) to prevent 

218 # inappropriate double-execution of module code 

219 try: 

220 __import__(modname) 

221 except Exception: 

222 if onerror is not None: 

223 onerror(modname) 

224 else: 

225 raise 

226 module = sys.modules.get(modname) 

227 if module is not None: 

228 for name, ob in getmembers(module, None): 

229 invoke(modname, name, ob) 

230 finally: 

231 if hasattr(loader, "file") and hasattr( 

232 loader.file, "close" 

233 ): # pragma: nocover 

234 loader.file.close() 

235 

236 

237class AttachInfo(object): 

238 """ 

239 An instance of this class is returned by the 

240 :func:`venusian.attach` function. It has the following 

241 attributes: 

242 

243 ``scope`` 

244 

245 One of ``exec``, ``module``, ``class``, ``function call`` or 

246 ``unknown`` (each a string). This is the scope detected while 

247 executing the decorator which runs the attach function. 

248 

249 ``module`` 

250 

251 The module in which the decorated function was defined. 

252 

253 ``locals`` 

254 

255 A dictionary containing decorator frame's f_locals. 

256 

257 ``globals`` 

258 

259 A dictionary containing decorator frame's f_globals. 

260 

261 ``category`` 

262 

263 The ``category`` argument passed to ``attach`` (or ``None``, the 

264 default). 

265 

266 ``codeinfo`` 

267 

268 A tuple in the form ``(filename, lineno, function, sourceline)`` 

269 representing the context of the venusian decorator used. Eg. 

270 ``('/home/chrism/projects/venusian/tests/test_advice.py', 81, 

271 'testCallInfo', 'add_handler(foo, bar)')`` 

272  

273 """ 

274 

275 def __init__(self, **kw): 

276 self.__dict__.update(kw) 

277 

278 

279class Categories(dict): 

280 def __init__(self, attached_to): 

281 super(dict, self).__init__() 

282 if isinstance(attached_to, tuple): 

283 self.attached_id = attached_to 

284 else: 

285 self.attached_id = id(attached_to) 

286 self.lifted = False 

287 

288 def attached_to(self, mod_name, name, obj): 

289 if isinstance(self.attached_id, int): 

290 return self.attached_id == id(obj) 

291 return self.attached_id == (mod_name, name) 

292 

293 

294def attach(wrapped, callback, category=None, depth=1, name=None): 

295 """ Attach a callback to the wrapped object. It will be found 

296 later during a scan. This function returns an instance of the 

297 :class:`venusian.AttachInfo` class. 

298 

299 ``category`` should be ``None`` or a string representing a decorator 

300 category name. 

301 

302 ``name`` should be ``None`` or a string representing a subcategory within 

303 the category. This will be used by the ``lift`` class decorator to 

304 determine if decorations of a method should be inherited or overridden. 

305 """ 

306 

307 frame = sys._getframe(depth + 1) 

308 scope, module, f_locals, f_globals, codeinfo = getFrameInfo(frame) 

309 module_name = getattr(module, "__name__", None) 

310 wrapped_name = getattr(wrapped, "__name__", None) 

311 class_name = codeinfo[2] 

312 

313 liftid = "%s %s" % (wrapped_name, name) 

314 

315 if scope == "class": 

316 # we're in the midst of a class statement 

317 categories = f_locals.get(ATTACH_ATTR, None) 

318 if categories is None or not categories.attached_to( 

319 module_name, class_name, None 

320 ): 

321 categories = Categories((module_name, class_name)) 

322 f_locals[ATTACH_ATTR] = categories 

323 callbacks = categories.setdefault(category, []) 

324 else: 

325 categories = getattr(wrapped, ATTACH_ATTR, None) 

326 if categories is None or not categories.attached_to( 

327 module_name, wrapped_name, wrapped 

328 ): 

329 # if there aren't any attached categories, or we've retrieved 

330 # some by inheritance, we need to create new ones 

331 categories = Categories(wrapped) 

332 setattr(wrapped, ATTACH_ATTR, categories) 

333 callbacks = categories.setdefault(category, []) 

334 

335 callbacks.append((callback, module_name, liftid, scope)) 

336 

337 return AttachInfo( 

338 scope=scope, 

339 module=module, 

340 locals=f_locals, 

341 globals=f_globals, 

342 category=category, 

343 codeinfo=codeinfo, 

344 ) 

345 

346 

347def walk_packages(path=None, prefix="", onerror=None, ignore=None): 

348 """Yields (module_loader, name, ispkg) for all modules recursively 

349 on path, or, if path is None, all accessible modules. 

350 

351 'path' should be either None or a list of paths to look for 

352 modules in. 

353 

354 'prefix' is a string to output on the front of every module name 

355 on output. 

356 

357 Note that this function must import all *packages* (NOT all 

358 modules!) on the given path, in order to access the __path__ 

359 attribute to find submodules. 

360 

361 'onerror' is a function which gets called with one argument (the name of 

362 the package which was being imported) if any exception occurs while 

363 trying to import a package. If no onerror function is supplied, any 

364 exception is exceptions propagated, terminating the search. 

365 

366 'ignore' is a function fed a fullly dotted name; if it returns True, the 

367 object is skipped and not returned in results (and if it's a package it's 

368 not imported). 

369 

370 Examples: 

371 

372 # list all modules python can access 

373 walk_packages() 

374 

375 # list all submodules of ctypes 

376 walk_packages(ctypes.__path__, ctypes.__name__+'.') 

377 

378 # NB: we can't just use pkgutils.walk_packages because we need to ignore 

379 # things 

380 """ 

381 

382 def seen(p, m={}): 

383 if p in m: # pragma: no cover 

384 return True 

385 m[p] = True 

386 

387 # iter_modules is nonrecursive 

388 for importer, name, ispkg in iter_modules(path, prefix): 

389 

390 if ignore is not None and ignore(name): 

391 # if name is a package, ignoring here will cause 

392 # all subpackages and submodules to be ignored too 

393 continue 

394 

395 # do any onerror handling before yielding 

396 

397 if ispkg: 

398 try: 

399 __import__(name) 

400 except Exception: 

401 if onerror is not None: 

402 onerror(name) 

403 else: 

404 raise 

405 else: 

406 yield importer, name, ispkg 

407 path = getattr(sys.modules[name], "__path__", None) or [] 

408 

409 # don't traverse path items we've seen before 

410 path = [p for p in path if not seen(p)] 

411 

412 for item in walk_packages(path, name + ".", onerror, ignore): 

413 yield item 

414 else: 

415 yield importer, name, ispkg 

416 

417 

418class lift(object): 

419 """ 

420 A class decorator which 'lifts' superclass venusian configuration 

421 decorations into subclasses. For example:: 

422 

423 from venusian import lift 

424 from somepackage import venusian_decorator 

425 

426 class Super(object): 

427 @venusian_decorator() 

428 def boo(self): pass 

429 

430 @venusian_decorator() 

431 def hiss(self): pass 

432 

433 @venusian_decorator() 

434 def jump(self): pass 

435  

436 @lift() 

437 class Sub(Super): 

438 def boo(self): pass 

439 

440 def hiss(self): pass 

441  

442 @venusian_decorator() 

443 def smack(self): pass 

444 

445 The above configuration will cause the callbacks of seven venusian 

446 decorators. The ones attached to Super.boo, Super.hiss, and Super.jump 

447 *plus* ones attached to Sub.boo, Sub.hiss, Sub.hump and Sub.smack. 

448 

449 If a subclass overrides a decorator on a method, its superclass decorators 

450 will be ignored for the subclass. That means that in this configuration:: 

451 

452 from venusian import lift 

453 from somepackage import venusian_decorator 

454 

455 class Super(object): 

456 @venusian_decorator() 

457 def boo(self): pass 

458 

459 @venusian_decorator() 

460 def hiss(self): pass 

461 

462 @lift() 

463 class Sub(Super): 

464 

465 def boo(self): pass 

466  

467 @venusian_decorator() 

468 def hiss(self): pass 

469 

470 Only four, not five decorator callbacks will be run: the ones attached to 

471 Super.boo and Super.hiss, the inherited one of Sub.boo and the 

472 non-inherited one of Sub.hiss. The inherited decorator on Super.hiss will 

473 be ignored for the subclass. 

474 

475 The ``lift`` decorator takes a single argument named 'categories'. If 

476 supplied, it should be a tuple of category names. Only decorators 

477 in this category will be lifted if it is suppled. 

478  

479 """ 

480 

481 def __init__(self, categories=None): 

482 self.categories = categories 

483 

484 def __call__(self, wrapped): 

485 if not isclass(wrapped): 

486 raise RuntimeError( 

487 '"lift" only works as a class decorator; you tried to use ' 

488 "it against %r" % wrapped 

489 ) 

490 frame = sys._getframe(1) 

491 scope, module, f_locals, f_globals, codeinfo = getFrameInfo(frame) 

492 module_name = getattr(module, "__name__", None) 

493 newcategories = Categories(wrapped) 

494 newcategories.lifted = True 

495 for cls in getmro(wrapped): 

496 attached_categories = cls.__dict__.get(ATTACH_ATTR, None) 

497 if attached_categories is None: 

498 attached_categories = cls.__dict__.get(LIFTONLY_ATTR, None) 

499 if attached_categories is not None: 

500 for cname, category in attached_categories.items(): 

501 if cls is not wrapped: 

502 if self.categories and not cname in self.categories: 

503 continue 

504 callbacks = newcategories.get(cname, []) 

505 newcallbacks = [] 

506 for cb, _, liftid, cscope in category: 

507 append = True 

508 toappend = (cb, module_name, liftid, cscope) 

509 if cscope == "class": 

510 for ncb, _, nliftid, nscope in callbacks: 

511 if nscope == "class" and liftid == nliftid: 

512 append = False 

513 if append: 

514 newcallbacks.append(toappend) 

515 newcategory = list(callbacks) + newcallbacks 

516 newcategories[cname] = newcategory 

517 if attached_categories.lifted: 

518 break 

519 if newcategories: # if it has any keys 

520 setattr(wrapped, ATTACH_ATTR, newcategories) 

521 return wrapped 

522 

523 

524class onlyliftedfrom(object): 

525 """ 

526 A class decorator which marks a class as 'only lifted from'. Decorations 

527 made on methods of the class won't have their callbacks called directly, 

528 but classes which inherit from only-lifted-from classes which also use the 

529 ``lift`` class decorator will use the superclass decoration callbacks. 

530 

531 For example:: 

532  

533 from venusian import lift, onlyliftedfrom 

534 from somepackage import venusian_decorator 

535 

536 @onlyliftedfrom() 

537 class Super(object): 

538 @venusian_decorator() 

539 def boo(self): pass 

540 

541 @venusian_decorator() 

542 def hiss(self): pass 

543 

544 @lift() 

545 class Sub(Super): 

546 

547 def boo(self): pass 

548  

549 def hiss(self): pass 

550 

551 Only two decorator callbacks will be run: the ones attached to Sub.boo and 

552 Sub.hiss. The inherited decorators on Super.boo and Super.hiss will be 

553 not be registered. 

554 """ 

555 

556 def __call__(self, wrapped): 

557 if not isclass(wrapped): 

558 raise RuntimeError( 

559 '"onlyliftedfrom" only works as a class decorator; you tried ' 

560 "to use it against %r" % wrapped 

561 ) 

562 cats = getattr(wrapped, ATTACH_ATTR, None) 

563 class_name = wrapped.__name__ 

564 module_name = wrapped.__module__ 

565 key = (module_name, class_name, wrapped) 

566 if cats is None or not cats.attached_to(*key): 

567 # we either have no categories or our categories are defined 

568 # in a superclass 

569 return 

570 delattr(wrapped, ATTACH_ATTR) 

571 setattr(wrapped, LIFTONLY_ATTR, cats) 

572 return wrapped