1 """
2 Code borrowed from Michael Hudson's docextractor package with the author's
3 permission.
4
5 The original code is available at http://codespeak.net/svn/user/mwh/docextractor/.
6
7 Changes:
8 * do not print warnings to stdout (in System.warning)
9 * collect all function calls
10 """
11
12
13 from compiler import ast
14 import sys
15 import os
16 import cPickle as pickle
17 import __builtin__
18 import sets
19
20 from compiler.transformer import parse, parseFile
21 from compiler.visitor import walk
22
23 import ast_pp
24
25
27 assert isinstance(node, ast.CallFunc)
28
30 if isinstance(node, ast.CallFunc):
31 return None
32 elif isinstance(node, ast.Name):
33 return node.name
34 elif isinstance(node, str):
35 return node
36 elif isinstance(node, tuple):
37 if len(node) == 1:
38 return node[0]
39 else:
40 return "%s.%s" % (get_name(node[:-1][0]), node[-1])
41 elif isinstance(node, ast.Getattr):
42 return get_name(node.asList())
43 else:
44 raise TypeError("Bad function name type: %s." % node)
45
46 return get_name(node.node)
47
49 if not isinstance(node, ast.Node):
50 return
51
52 for child in node.getChildren():
53 if isinstance(child, ast.CallFunc):
54 func_called = get_call_name(child)
55 fc[func_called] = 1
56
57 get_function_calls(child, fc)
58
59
61 - def __init__(self, system, prefix, name, docstring, parent=None):
62 self.system = system
63 self.prefix = prefix
64 self.name = name
65 self.docstring = docstring
66 self.parent = parent
67 self.setup()
69 self.contents = {}
70 self.orderedcontents = []
71 self._name2fullname = {}
75 docstring = self.docstring
76 if docstring:
77 docstring = docstring.rstrip()
78 if len(docstring) > 20:
79 docstring = docstring[:8] + '...' + docstring[-8:]
80 return docstring
82 return "%s %r"%(self.__class__.__name__, self.fullName())
88
90 parts = dottedname.split('.')
91 obj = self
92 system = self.system
93 while parts[0] not in obj._name2fullname:
94 obj = obj.parent
95 if obj is None:
96 if parts[0] in system.allobjects:
97 obj = system.allobjects[parts[0]]
98 break
99 for othersys in system.moresystems:
100 if parts[0] in othersys.allobjects:
101 obj = othersys.allobjects[parts[0]]
102 break
103 else:
104 if verbose:
105 print "1 didn't find %r from %r"%(dottedname,
106 self.fullName())
107 return None
108 break
109 else:
110 fn = obj._name2fullname[parts[0]]
111 if fn in system.allobjects:
112 obj = system.allobjects[fn]
113 else:
114 if verbose:
115 print "1.5 didn't find %r from %r"%(dottedname,
116 self.fullName())
117 return None
118 for p in parts[1:]:
119 if p not in obj.contents:
120 if verbose:
121 print "2 didn't find %r from %r"%(dottedname,
122 self.fullName())
123 return None
124 obj = obj.contents[p]
125 if verbose:
126 print dottedname, '->', obj.fullName(), 'in', self.fullName()
127 return obj
128
130 if '.' not in dottedname:
131 start, rest = dottedname, ''
132 else:
133 start, rest = dottedname.split('.', 1)
134 rest = '.' + rest
135 obj = self
136 while start not in obj._name2fullname:
137 obj = obj.parent
138 if obj is None:
139 return dottedname
140 return obj._name2fullname[start] + rest
141
143
144
145 r = {}
146 for k, v in self.__dict__.iteritems():
147 if isinstance(v, Documentable):
148 r['$'+k] = v.fullName()
149 elif isinstance(v, list) and v:
150 for vv in v:
151 if vv is not None and not isinstance(vv, Documentable):
152 r[k] = v
153 break
154 else:
155 rr = []
156 for vv in v:
157 if vv is None:
158 rr.append(vv)
159 else:
160 rr.append(vv.fullName())
161 r['@'+k] = rr
162 elif isinstance(v, dict) and v:
163 for vv in v.itervalues():
164 if not isinstance(vv, Documentable):
165 r[k] = v
166 break
167 else:
168 rr = {}
169 for kk, vv in v.iteritems():
170 rr[kk] = vv.fullName()
171 r['!'+k] = rr
172 else:
173 r[k] = v
174 return r
175
177 kind = "Package"
179 raise NameError
180
181
183 kind = "Module"
185 if name in self._name2fullname:
186 return self._name2fullname[name]
187 elif name in __builtin__.__dict__:
188 return name
189 else:
190 self.system.warning("optimistic name resolution", name)
191 return name
192
193
194 -class Class(Documentable):
195 kind = "Class"
197 super(Class, self).setup()
198 self.bases = []
199 self.rawbases = []
200 self.baseobjects = []
201 self.subclasses = []
202
203
206
207
210 self.system = system
211 self.modname = modname
212 self.morenodes = []
213
215 for child in node.getChildNodes():
216 self.visit(child)
217
218 - def postpone(self, docable, node):
219 self.morenodes.append((docable, node))
220
222 if self.system.current and self.modname in self.system.current.contents:
223 m = self.system.current.contents[self.modname]
224 assert m.docstring is None
225 m.docstring = node.doc
226 self.system.push(m, node)
227 self.default(node)
228 self.system.pop(m)
229 else:
230 if not self.system.current:
231 roots = [x for x in self.system.rootobjects if x.name == self.modname]
232 if roots:
233 mod, = roots
234 self.system.push(mod, node)
235 self.default(node)
236 self.system.pop(mod)
237 return
238 self.system.pushModule(self.modname, node.doc)
239 self.default(node)
240 self.system.popModule()
241
243 cls = self.system.pushClass(node.name, node.doc)
244 if node.lineno is not None:
245 cls.linenumber = node.lineno
246 for n in node.bases:
247 str_base = ast_pp.pp(n)
248 cls.rawbases.append(str_base)
249 base = cls.dottedNameToFullName(str_base)
250 cls.bases.append(base)
251 self.default(node)
252 self.system.popClass()
253
255 modname = expandModname(self.system, node.modname)
256 name2fullname = self.system.current._name2fullname
257 for fromname, asname in node.names:
258 if fromname == '*':
259 self.system.warning("import *", modname)
260 if modname not in self.system.allobjects:
261 return
262 mod = self.system.allobjects[modname]
263
264
265
266
267 if isinstance(mod, Package):
268 self.system.warning("import * from a package", modname)
269 return
270 if mod.processed:
271 for n in mod.contents:
272 name2fullname[n] = modname + '.' + n
273 else:
274 self.system.warning("unresolvable import *", modname)
275 return
276 if asname is None:
277 asname = fromname
278 name2fullname[asname] = modname + '.' + fromname
279
281 name2fullname = self.system.current._name2fullname
282 for fromname, asname in node.names:
283 fullname = expandModname(self.system, fromname)
284 if asname is None:
285 asname = fromname.split('.', 1)[0]
286
287 parts = fullname.split('.')
288 for i, part in enumerate(fullname.split('.')[::-1]):
289 if part == asname:
290 fullname = '.'.join(parts[:len(parts)-i])
291 name2fullname[asname] = fullname
292 break
293 else:
294 name2fullname[asname] = '.'.join(parts)
295 else:
296 name2fullname[asname] = fullname
297
299 fc = {}
300 get_function_calls(node, fc)
301 func = self.system.pushFunction(node.name, node.doc, fc)
302 if node.lineno is not None:
303 func.linenumber = node.lineno
304
305
306
307 argnames = node.argnames[:]
308 kwname = starargname = None
309 if node.kwargs:
310 kwname = argnames.pop(-1)
311 if node.varargs:
312 starargname = argnames.pop(-1)
313 defaults = []
314 for default in node.defaults:
315 try:
316 defaults.append(ast_pp.pp(default))
317 except (KeyboardInterrupt, SystemExit):
318 raise
319 except Exception, e:
320 self.system.warning("unparseable default", "%s: %s %r"%(e.__class__.__name__,
321 e, default))
322 defaults.append('???')
323
324
325
326 argnames2 = []
327 for argname in argnames:
328 if isinstance(argname, tuple):
329 argname = list(argname)
330 argnames2.append(argname)
331 func.argspec = (argnames2, starargname, kwname, tuple(defaults))
332 self.postpone(func, node.code)
333 self.system.popFunction()
334
335 states = [
336 'blank',
337 'preparse',
338 'importstarred',
339 'parsed',
340 'finalized',
341 ]
342
343
345 Class = Class
346 Module = Module
347 Package = Package
348 Function = Function
349 ModuleVistor = ModuleVistor
350
352 self.current = None
353 self._stack = []
354 self.allobjects = {}
355 self.orderedallobjects = []
356 self.rootobjects = []
357 self.warnings = {}
358
359
360 self.importstargraph = {}
361 self.func_called = {}
362 self.state = 'blank'
363 self.packages = []
364 self.moresystems = []
365 self.urlprefix = ''
366
367 - def _push(self, cls, name, docstring):
368 if self.current:
369 prefix = self.current.fullName() + '.'
370 parent = self.current
371 else:
372 prefix = ''
373 parent = None
374 obj = cls(self, prefix, name, docstring, parent)
375 if parent:
376 parent.orderedcontents.append(obj)
377 parent.contents[name] = obj
378 parent._name2fullname[name] = obj.fullName()
379 else:
380 self.rootobjects.append(obj)
381 self.current = obj
382 self.orderedallobjects.append(obj)
383 fullName = obj.fullName()
384
385 if fullName in self.allobjects:
386 obj = self.handleDuplicate(obj)
387 else:
388 self.allobjects[obj.fullName()] = obj
389 return obj
390
392 '''This is called when we see two objects with the same
393 .fullName(), for example:
394
395 class C:
396 if something:
397 def meth(self):
398 implementation 1
399 else:
400 def meth(self):
401 implementation 2
402
403 The default is that the second definition "wins".
404 '''
405 i = 0
406 fn = obj.fullName()
407 while (fn + ' ' + str(i)) in self.allobjects:
408 i += 1
409 prev = self.allobjects[obj.fullName()]
410 prev.name = obj.name + ' ' + str(i)
411 self.allobjects[prev.fullName()] = prev
412 self.warning("duplicate", self.allobjects[obj.fullName()])
413 self.allobjects[obj.fullName()] = obj
414 return obj
415
416
417 - def _pop(self, cls):
418 assert isinstance(self.current, cls)
419
420
421
422
423 self.current = self.current.parent
424
425 - def push(self, obj, node=None):
426 self._stack.append(self.current)
427 self.current = obj
428
429 - def pop(self, obj):
430 assert self.current is obj, "%r is not %r"%(self.current, obj)
431 self.current = self._stack.pop()
432
437
442
444 self.func_called.update(func_called)
445 return self._push(self.Function, name, docstring)
448
453
455 for o in self.rootobjects:
456 self._report(o, '')
457
462
464 if '.' not in n:
465 return n
466 mod, clsname = n.split('.')
467 if not mod or mod not in self.allobjects:
468 return n
469 m = self.allobjects[mod]
470 if not isinstance(m, Module):
471 return n
472 if clsname in m._name2fullname:
473 newname = m.name2fullname(clsname)
474 if newname not in self.allobjects:
475 return self.resolveAlias(newname)
476 else:
477 return newname
478
480 for ob in self.orderedallobjects:
481 if not isinstance(ob, Class):
482 continue
483 for i, b in enumerate(ob.bases):
484 if b not in self.allobjects:
485 ob.bases[i] = self.resolveAlias(b)
486
488 if self.current is not None:
489 fn = self.current.fullName()
490 else:
491 fn = '<None>'
492 self.warnings.setdefault(type, []).append((fn, detail))
493
495 for o in self.orderedallobjects:
496 if isinstance(o, cls):
497 yield o
498
501
509
511 state = self.__dict__.copy()
512 del state['moresystems']
513 return state
514
516 self.moresystems = []
517
518
519 self.__dict__.update(state)
520 for obj in self.orderedallobjects:
521 for k, v in obj.__dict__.copy().iteritems():
522 if k.startswith('$'):
523 del obj.__dict__[k]
524 obj.__dict__[k[1:]] = self.allobjects[v]
525 elif k.startswith('@'):
526 n = []
527 for vv in v:
528 if vv is None:
529 n.append(None)
530 else:
531 n.append(self.allobjects[vv])
532 del obj.__dict__[k]
533 obj.__dict__[k[1:]] = n
534 elif k.startswith('!'):
535 n = {}
536 for kk, vv in v.iteritems():
537 n[kk] = self.allobjects[vv]
538 del obj.__dict__[k]
539 obj.__dict__[k[1:]] = n
540
541
543 c = system.current
544 if '.' in modname:
545 prefix, suffix = modname.split('.', 1)
546 suffix = '.' + suffix
547 else:
548 prefix, suffix = modname, ''
549 while c is not None and not isinstance(c, Package):
550 c = c.parent
551 while c is not None:
552 if prefix in c.contents:
553 break
554 c = c.parent
555 if c is not None:
556 if givewarning:
557 system.warning("local import", modname)
558 return c.contents[prefix].fullName() + suffix
559 else:
560 return prefix + suffix
561
563 - def __init__(self, system, modfullname):
564 self.system = system
565 self.modfullname = modfullname
566
568 if node.names[0][0] == '*':
569 modname = expandModname(self.system, node.modname, False)
570 self.system.importstargraph.setdefault(
571 self.modfullname, []).append(modname)
572
574 mv = system.ModuleVistor(system, name)
575 walk(ast, mv)
576 while mv.morenodes:
577 obj, node = mv.morenodes.pop(0)
578 system.push(obj, node)
579 mv.visit(node)
580 system.pop(obj)
581
582
583 -def fromText(src, modname='<test>', system=None):
584 if system is None:
585 _system = System()
586 else:
587 _system = system
588 processModuleAst(parse(src), modname, _system)
589 if system is None:
590 _system.finalStateComputations()
591 return _system.rootobjects[0]
592
593
595 assert system.state in ['blank', 'preparse']
596 if os.path.basename(dirpath):
597 package = system.pushPackage(os.path.basename(dirpath), None)
598 else:
599 package = None
600 for fname in os.listdir(dirpath):
601 fullname = os.path.join(dirpath, fname)
602 if os.path.isdir(fullname) and os.path.exists(os.path.join(fullname, '__init__.py')) and fname != 'test':
603 preprocessDirectory(system, fullname)
604 elif fname.endswith('.py'):
605 modname = os.path.splitext(fname)[0]
606 mod = system.pushModule(modname, None)
607 mod.filepath = fullname
608 mod.processed = False
609 system.popModule()
610 if package:
611 system.popPackage()
612 system.state = 'preparse'
613
615 assert system.state in ['preparse']
616 modlist = list(system.objectsOfType(Module))
617 for mod in modlist:
618 system.push(mod.parent)
619 isf = ImportStarFinder(system, mod.fullName())
620 try:
621 ast = parseFile(mod.filepath)
622 except (SyntaxError, ValueError):
623 system.warning("cannot parse", mod.filepath)
624 walk(ast, isf)
625 system.pop(mod.parent)
626 system.state = 'importstarred'
627
629 assert system.state in ['preparse', 'importstarred']
630
631 modlist = list(system.objectsOfType(Module))
632 newlist = toposort([m.fullName() for m in modlist], system.importstargraph)
633
634 for mod in newlist:
635 mod = system.allobjects[mod]
636 system.push(mod.parent)
637 try:
638 ast = parseFile(mod.filepath)
639 except (SyntaxError, ValueError):
640 system.warning("cannot parse", mod.filepath)
641 processModuleAst(ast, mod.name, system)
642 mod.processed = True
643 system.pop(mod.parent)
644 system.state = 'parsed'
645
650
656
658
659 output = []
660 input = dict.fromkeys(input)
662 for j in edges.get(i, []):
663 if j in input:
664 del input[j]
665 p(j)
666 output.append(i)
667 while input:
668 p(input.popitem()[0])
669 return output
670
671
672 -def main(systemcls, argv):
673 if '-r' in argv:
674 argv.remove('-r')
675 assert len(argv) == 1
676 system = systemcls()
677 processDirectory(system, argv[0])
678 pickle.dump(system, open('da.out', 'wb'), pickle.HIGHEST_PROTOCOL)
679 print
680 print 'warning summary:'
681 for k, v in system.warnings.iteritems():
682 print k, len(v)
683 else:
684 system = systemcls()
685 for fname in argv:
686 modname = os.path.splitext(os.path.basename(fname))[0]
687 processModuleAst(parseFile(fname), modname, system)
688 system.report()
689
690
691
692 if __name__ == '__main__':
693 main(System, sys.argv[1:])
694