Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/statsmodels/tools/docstring.py : 78%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Substantially copied from NumpyDoc 1.0pre.
3"""
4import copy
5import inspect
6import re
7import textwrap
8from collections import namedtuple
9from collections.abc import Mapping
12def dedent_lines(lines):
13 """Deindent a list of lines maximally"""
14 return textwrap.dedent("\n".join(lines)).split("\n")
17def strip_blank_lines(l):
18 """Remove leading and trailing blank lines from a list of lines"""
19 while l and not l[0].strip():
20 del l[0]
21 while l and not l[-1].strip():
22 del l[-1]
23 return l
26class Reader(object):
27 """
28 A line-based string reader.
29 """
31 def __init__(self, data):
32 """
33 Parameters
34 ----------
35 data : str
36 String with lines separated by '\n'.
37 """
38 if isinstance(data, list):
39 self._str = data
40 else:
41 self._str = data.split('\n') # store string as list of lines
43 self.reset()
45 def __getitem__(self, n):
46 return self._str[n]
48 def reset(self):
49 self._l = 0 # current line nr
51 def read(self):
52 if not self.eof():
53 out = self[self._l]
54 self._l += 1
55 return out
56 else:
57 return ''
59 def seek_next_non_empty_line(self):
60 for l in self[self._l:]:
61 if l.strip():
62 break
63 else:
64 self._l += 1
66 def eof(self):
67 return self._l >= len(self._str)
69 def read_to_condition(self, condition_func):
70 start = self._l
71 for line in self[start:]:
72 if condition_func(line):
73 return self[start:self._l]
74 self._l += 1
75 if self.eof():
76 return self[start:self._l + 1]
77 return []
79 def read_to_next_empty_line(self):
80 self.seek_next_non_empty_line()
82 def is_empty(line):
83 return not line.strip()
85 return self.read_to_condition(is_empty)
87 def read_to_next_unindented_line(self):
88 def is_unindented(line):
89 return (line.strip() and (len(line.lstrip()) == len(line)))
91 return self.read_to_condition(is_unindented)
93 def peek(self, n=0):
94 if self._l + n < len(self._str):
95 return self[self._l + n]
96 else:
97 return ''
99 def is_empty(self):
100 return not ''.join(self._str).strip()
103class ParseError(Exception):
104 def __str__(self):
105 message = self.args[0]
106 if hasattr(self, 'docstring'):
107 message = "%s in %r" % (message, self.docstring)
108 return message
111Parameter = namedtuple('Parameter', ['name', 'type', 'desc'])
114class NumpyDocString(Mapping):
115 """Parses a numpydoc string to an abstract representation
117 Instances define a mapping from section title to structured data.
118 """
120 sections = {
121 'Signature': '',
122 'Summary': [''],
123 'Extended Summary': [],
124 'Parameters': [],
125 'Returns': [],
126 'Yields': [],
127 'Receives': [],
128 'Raises': [],
129 'Warns': [],
130 'Other Parameters': [],
131 'Attributes': [],
132 'Methods': [],
133 'See Also': [],
134 'Notes': [],
135 'Warnings': [],
136 'References': '',
137 'Examples': '',
138 'index': {}
139 }
141 def __init__(self, docstring):
142 orig_docstring = docstring
143 docstring = textwrap.dedent(docstring).split('\n')
145 self._doc = Reader(docstring)
146 self._parsed_data = copy.deepcopy(self.sections)
148 try:
149 self._parse()
150 except ParseError as e:
151 e.docstring = orig_docstring
152 raise
154 def __getitem__(self, key):
155 return self._parsed_data[key]
157 def __setitem__(self, key, val):
158 if key not in self._parsed_data:
159 self._error_location("Unknown section %s" % key)
160 else:
161 self._parsed_data[key] = val
163 def __iter__(self):
164 return iter(self._parsed_data)
166 def __len__(self):
167 return len(self._parsed_data)
169 def _is_at_section(self):
170 self._doc.seek_next_non_empty_line()
172 if self._doc.eof():
173 return False
175 l1 = self._doc.peek().strip() # e.g. Parameters
177 if l1.startswith('.. index::'):
178 return True
180 l2 = self._doc.peek(1).strip() # ---------- or ==========
181 return l2.startswith('-' * len(l1)) or l2.startswith('=' * len(l1))
183 def _strip(self, doc):
184 i = 0
185 j = 0
186 for i, line in enumerate(doc):
187 if line.strip():
188 break
190 for j, line in enumerate(doc[::-1]):
191 if line.strip():
192 break
194 return doc[i:len(doc) - j]
196 def _read_to_next_section(self):
197 section = self._doc.read_to_next_empty_line()
199 while not self._is_at_section() and not self._doc.eof():
200 if not self._doc.peek(-1).strip(): # previous line was empty
201 section += ['']
203 section += self._doc.read_to_next_empty_line()
205 return section
207 def _read_sections(self):
208 while not self._doc.eof():
209 data = self._read_to_next_section()
210 name = data[0].strip()
212 if name.startswith('..'): # index section
213 yield name, data[1:]
214 elif len(data) < 2:
215 yield StopIteration
216 else:
217 yield name, self._strip(data[2:])
219 def _parse_param_list(self, content, single_element_is_type=False):
220 r = Reader(content)
221 params = []
222 while not r.eof():
223 header = r.read().strip()
224 if ' : ' in header:
225 arg_name, arg_type = header.split(' : ')[:2]
226 else:
227 if single_element_is_type:
228 arg_name, arg_type = '', header
229 else:
230 arg_name, arg_type = header, ''
232 desc = r.read_to_next_unindented_line()
233 desc = dedent_lines(desc)
234 desc = strip_blank_lines(desc)
236 params.append(Parameter(arg_name, arg_type, desc))
238 return params
240 # See also supports the following formats.
241 #
242 # <FUNCNAME>
243 # <FUNCNAME> SPACE* COLON SPACE+ <DESC> SPACE*
244 # <FUNCNAME> ( COMMA SPACE+ <FUNCNAME>)+ (COMMA | PERIOD)? SPACE*
245 # <FUNCNAME> ( COMMA SPACE+ <FUNCNAME>)* SPACE* COLON SPACE+ <DESC> SPACE*
247 # <FUNCNAME> is one of
248 # <PLAIN_FUNCNAME>
249 # COLON <ROLE> COLON BACKTICK <PLAIN_FUNCNAME> BACKTICK
250 # where
251 # <PLAIN_FUNCNAME> is a legal function name, and
252 # <ROLE> is any nonempty sequence of word characters.
253 # Examples: func_f1 :meth:`func_h1` :obj:`~baz.obj_r` :class:`class_j`
254 # <DESC> is a string describing the function.
256 _role = r":(?P<role>\w+):"
257 _funcbacktick = r"`(?P<name>(?:~\w+\.)?[a-zA-Z0-9_\.-]+)`"
258 _funcplain = r"(?P<name2>[a-zA-Z0-9_\.-]+)"
259 _funcname = r"(" + _role + _funcbacktick + r"|" + _funcplain + r")"
260 _funcnamenext = _funcname.replace('role', 'rolenext')
261 _funcnamenext = _funcnamenext.replace('name', 'namenext')
262 _description = r"(?P<description>\s*:(\s+(?P<desc>\S+.*))?)?\s*$"
263 _func_rgx = re.compile(r"^\s*" + _funcname + r"\s*")
264 _line_rgx = re.compile(
265 r"^\s*" +
266 r"(?P<allfuncs>" + # group for all function names
267 _funcname +
268 r"(?P<morefuncs>([,]\s+" + _funcnamenext + r")*)" +
269 r")" + # end of "allfuncs"
270 # Some function lists have a trailing comma (or period)
271 r"(?P<trailing>[,\.])?" +
272 _description)
274 # Empty <DESC> elements are replaced with '..'
275 empty_description = '..'
277 def _parse_see_also(self, content):
278 """
279 func_name : Descriptive text
280 continued text
281 another_func_name : Descriptive text
282 func_name1, func_name2, :meth:`func_name`, func_name3
283 """
285 items = []
287 def parse_item_name(text):
288 """Match ':role:`name`' or 'name'."""
289 m = self._func_rgx.match(text)
290 if not m:
291 raise ParseError("%s is not a item name" % text)
292 role = m.group('role')
293 name = m.group('name') if role else m.group('name2')
294 return name, role, m.end()
296 rest = []
297 for line in content:
298 if not line.strip():
299 continue
301 line_match = self._line_rgx.match(line)
302 description = None
303 if line_match:
304 description = line_match.group('desc')
305 if line_match.group('trailing') and description:
306 self._error_location(
307 'Unexpected comma or period after function list at '
308 'index %d of line '
309 '"%s"' % (line_match.end('trailing'), line))
310 if not description and line.startswith(' '):
311 rest.append(line.strip())
312 elif line_match:
313 funcs = []
314 text = line_match.group('allfuncs')
315 while True:
316 if not text.strip():
317 break
318 name, role, match_end = parse_item_name(text)
319 funcs.append((name, role))
320 text = text[match_end:].strip()
321 if text and text[0] == ',':
322 text = text[1:].strip()
323 rest = list(filter(None, [description]))
324 items.append((funcs, rest))
325 else:
326 raise ParseError("%s is not a item name" % line)
327 return items
329 def _parse_index(self, section, content):
330 """
331 .. index: default
332 :refguide: something, else, and more
333 """
335 def strip_each_in(lst):
336 return [s.strip() for s in lst]
338 out = {}
339 section = section.split('::')
340 if len(section) > 1:
341 out['default'] = strip_each_in(section[1].split(','))[0]
342 for line in content:
343 line = line.split(':')
344 if len(line) > 2:
345 out[line[1]] = strip_each_in(line[2].split(','))
346 return out
348 def _parse_summary(self):
349 """Grab signature (if given) and summary"""
350 if self._is_at_section():
351 return
353 # If several signatures present, take the last one
354 while True:
355 summary = self._doc.read_to_next_empty_line()
356 summary_str = " ".join([s.strip() for s in summary]).strip()
357 compiled = re.compile(r'^([\w., ]+=)?\s*[\w\.]+\(.*\)$')
358 if compiled.match(summary_str):
359 self['Signature'] = summary_str
360 if not self._is_at_section():
361 continue
362 break
364 if summary is not None:
365 self['Summary'] = summary
367 if not self._is_at_section():
368 self['Extended Summary'] = self._read_to_next_section()
370 def _parse(self):
371 self._doc.reset()
372 self._parse_summary()
374 sections = list(self._read_sections())
375 section_names = set([section for section, content in sections])
377 has_returns = 'Returns' in section_names
378 has_yields = 'Yields' in section_names
379 # We could do more tests, but we are not. Arbitrarily.
380 if has_returns and has_yields:
381 msg = 'Docstring contains both a Returns and Yields section.'
382 raise ValueError(msg)
383 if not has_yields and 'Receives' in section_names:
384 msg = 'Docstring contains a Receives section but not Yields.'
385 raise ValueError(msg)
387 for (section, content) in sections:
388 if not section.startswith('..'):
389 section = (s.capitalize() for s in section.split(' '))
390 section = ' '.join(section)
391 if self.get(section):
392 self._error_location("The section %s appears twice"
393 % section)
395 if section in ('Parameters', 'Other Parameters', 'Attributes',
396 'Methods'):
397 self[section] = self._parse_param_list(content)
398 elif section in (
399 'Returns', 'Yields', 'Raises', 'Warns', 'Receives'):
400 self[section] = self._parse_param_list(
401 content, single_element_is_type=True)
402 elif section.startswith('.. index::'):
403 self['index'] = self._parse_index(section, content)
404 elif section == 'See Also':
405 self['See Also'] = self._parse_see_also(content)
406 else:
407 self[section] = content
409 def _error_location(self, msg):
410 if hasattr(self, '_obj'):
411 # we know where the docs came from:
412 try:
413 filename = inspect.getsourcefile(self._obj)
414 except TypeError:
415 filename = None
416 msg = msg + (" in the docstring of %s in %s."
417 % (self._obj, filename))
419 raise ValueError(msg)
421 # string conversion routines
423 def _str_header(self, name, symbol='-'):
424 return [name, len(name) * symbol]
426 def _str_indent(self, doc, indent=4):
427 out = []
428 for line in doc:
429 out += [' ' * indent + line]
430 return out
432 def _str_signature(self):
433 if self['Signature']:
434 return [self['Signature'].replace('*', r'\*')] + ['']
435 else:
436 return ['']
438 def _str_summary(self):
439 if self['Summary']:
440 return self['Summary'] + ['']
441 else:
442 return []
444 def _str_extended_summary(self):
445 if self['Extended Summary']:
446 return self['Extended Summary'] + ['']
447 else:
448 return []
450 def _str_param_list(self, name):
451 out = []
452 if self[name]:
453 out += self._str_header(name)
454 for param in self[name]:
455 parts = []
456 if param.name:
457 parts.append(param.name)
458 if param.type:
459 parts.append(param.type)
460 out += [' : '.join(parts)]
461 if param.desc and ''.join(param.desc).strip():
462 out += self._str_indent(param.desc)
463 out += ['']
464 return out
466 def _str_section(self, name):
467 out = []
468 if self[name]:
469 out += self._str_header(name)
470 out += self[name]
471 out += ['']
472 return out
474 def _str_see_also(self, func_role):
475 if not self['See Also']:
476 return []
477 out = []
478 out += self._str_header("See Also")
479 last_had_desc = True
480 for funcs, desc in self['See Also']:
481 assert isinstance(funcs, list)
482 links = []
483 for func, role in funcs:
484 if role:
485 link = ':%s:`%s`' % (role, func)
486 elif func_role:
487 link = ':%s:`%s`' % (func_role, func)
488 else:
489 link = "%s" % func
490 links.append(link)
491 link = ', '.join(links)
492 out += [link]
493 if desc:
494 out += self._str_indent([' '.join(desc)])
495 last_had_desc = True
496 else:
497 last_had_desc = False
498 out += self._str_indent([self.empty_description])
500 if last_had_desc:
501 out += ['']
502 return out
504 def _str_index(self):
505 idx = self['index']
506 out = []
507 output_index = False
508 default_index = idx.get('default', '')
509 if default_index:
510 output_index = True
511 out += ['.. index:: %s' % default_index]
512 for section, references in idx.items():
513 if section == 'default':
514 continue
515 output_index = True
516 out += [' :%s: %s' % (section, ', '.join(references))]
517 if output_index:
518 return out
519 else:
520 return ''
522 def __str__(self, func_role=''):
523 out = []
524 out += self._str_signature()
525 out += self._str_summary()
526 out += self._str_extended_summary()
527 for param_list in ('Parameters', 'Returns', 'Yields', 'Receives',
528 'Other Parameters', 'Raises', 'Warns'):
529 out += self._str_param_list(param_list)
530 out += self._str_section('Warnings')
531 out += self._str_see_also(func_role)
532 for s in ('Notes', 'References', 'Examples'):
533 out += self._str_section(s)
534 for param_list in ('Attributes', 'Methods'):
535 out += self._str_param_list(param_list)
536 out += self._str_index()
537 return '\n'.join(out)
540class Docstring(object):
541 """
542 Docstring modification.
544 Parameters
545 ----------
546 docstring : str
547 The docstring to modify.
548 """
550 def __init__(self, docstring):
551 self._ds = None
552 self._docstring = docstring
553 if docstring is None:
554 return
555 self._ds = NumpyDocString(docstring)
557 def remove_parameters(self, parameters):
558 """
559 Parameters
560 ----------
561 parameters : str, list[str]
562 The names of the parameters to remove.
563 """
564 if self._docstring is None:
565 # Protection against -oo execution
566 return
567 if isinstance(parameters, str):
568 parameters = [parameters]
569 repl = [param for param in self._ds['Parameters']
570 if param.name not in parameters]
571 if len(repl) + len(parameters) != len(self._ds['Parameters']):
572 raise ValueError('One or more parameters were not found.')
573 self._ds['Parameters'] = repl
575 def insert_parameters(self, after, parameters):
576 """
577 Parameters
578 ----------
579 after : {None, str}
580 If None, inset the parameters before the first parameter in the
581 docstring.
582 parameters : Parameter, list[Parameter]
583 A Parameter of a list of Parameters.
584 """
585 if self._docstring is None:
586 # Protection against -oo execution
587 return
588 if isinstance(parameters, Parameter):
589 parameters = [parameters]
590 if after is None:
591 self._ds['Parameters'] = parameters + self._ds['Parameters']
592 else:
593 loc = -1
594 for i, param in enumerate(self._ds['Parameters']):
595 if param.name == after:
596 loc = i+1
597 break
598 if loc < 0:
599 raise ValueError()
600 params = self._ds['Parameters'][:loc] + parameters
601 params += self._ds['Parameters'][loc:]
602 self._ds['Parameters'] = params
604 def replace_block(self, block_name, block):
605 """
606 Parameters
607 ----------
608 block_name : str
609 Name of the block to replace, e.g., 'Summary'.
610 block : object
611 The replacement block. The structure of the replacement block must
612 match how the block is stored by NumpyDocString.
613 """
614 if self._docstring is None:
615 # Protection against -oo execution
616 return
617 block_name = ' '.join(map(str.capitalize, block_name.split(' ')))
618 if block_name not in self._ds:
619 raise ValueError('{0} is not a block in the '
620 'docstring'.format(block_name))
621 if not isinstance(block, list):
622 block = [block]
623 self._ds[block_name] = block
625 def extract_parameters(self, parameters, indent=0):
626 if self._docstring is None:
627 # Protection against -oo execution
628 return
629 if isinstance(parameters, str):
630 parameters = [parameters]
631 ds_params = {param.name: param for param in self._ds['Parameters']}
632 missing = set(parameters).difference(ds_params.keys())
633 if missing:
634 raise ValueError('{0} were not found in the '
635 'docstring'.format(','.join(missing)))
636 final = [ds_params[param] for param in parameters]
637 ds = copy.deepcopy(self._ds)
638 for key in ds:
639 if key != 'Parameters':
640 ds[key] = [] if key != 'index' else {}
641 else:
642 ds[key] = final
643 out = str(ds).strip()
644 if indent:
645 out = textwrap.indent(out, ' ' * indent)
647 out = '\n'.join(out.split('\n')[2:])
648 return out
650 def __str__(self):
651 return str(self._ds)
654def remove_parameters(docstring, parameters):
655 """
656 Parameters
657 ----------
658 docstring : str
659 The docstring to modify.
660 parameters : str, list[str]
661 The names of the parameters to remove.
663 Returns
664 -------
665 str
666 The modified docstring.
667 """
668 if docstring is None:
669 return
670 ds = Docstring(docstring)
671 ds.remove_parameters(parameters)
672 return str(ds)