Source code for utool.util_inspect

# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import functools
import inspect
import os
import os.path
import re
import six
import sys
import types
from os.path import dirname
from six.moves import builtins
from collections import OrderedDict
from six.moves import range, zip  # NOQA
from utool import util_regex
from utool import util_arg
from utool import util_inject
from utool._internal import meta_util_six
print, rrr, profile = util_inject.inject2(__name__, '[inspect]')


VERBOSE_INSPECT, VERYVERB_INSPECT = util_arg.get_module_verbosity_flags('inspect')


LIB_PATH = dirname(os.__file__)


#@profile
[docs]def check_module_usage(modpath_partterns): """ Args: modpath_partterns (list): CommandLine: python -m utool.util_inspect --exec-check_module_usage --show utprof.py -m utool.util_inspect --exec-check_module_usage --show python -m utool.util_inspect --exec-check_module_usage --pat="['auto*', 'user_dialogs.py', 'special_query.py', 'qt_inc_automatch.py', 'devcases.py']" python -m utool.util_inspect --exec-check_module_usage --pat="preproc_detectimg.py" python -m utool.util_inspect --exec-check_module_usage --pat="neighbor_index.py" Ignore: Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> modpath_partterns = ['_grave*'] >>> modpath_partterns = ['auto*', 'user_dialogs.py', 'special_query.py', 'qt_inc_automatch.py', 'devcases.py'] >>> modpath_partterns = ut.get_argval('--pat', type_=list, default=['*']) >>> modpath_partterns = ['neighbor_index.py'] >>> result = check_module_usage(modpath_partterns) >>> print(result) """ import utool as ut #dpath = '~/code/ibeis/ibeis/algo/hots' dpath = '.' modpaths = ut.flatten([ut.glob(dpath, pat) for pat in modpath_partterns]) modnames = ut.lmap(ut.get_modname_from_modpath, modpaths) def get_funcnames_from_modpath(modpath): import jedi source = ut.read_from(modpath) #script = jedi.Script(source=source, source_path=modpath, line=source.count('\n') + 1) definition_list = jedi.names(source) funcname_list = [definition.name for definition in definition_list if definition.type == 'function'] if True: classdef_list = [definition for definition in definition_list if definition.type == 'class'] defined_methods = ut.flatten([definition.defined_names() for definition in classdef_list]) funcname_list += [method.name for method in defined_methods if method.type == 'function' and not method.name.startswith('_')] return funcname_list cache = {} func_call_graph = ut.ddict(dict) importance_dict = {} # Find places where the module was imported modname_fpath_lists = [] for modname in modnames: patterns = ut.possible_import_patterns(modname) patterns = [re.escape(pat) for pat in patterns] # do modname grep with all possible import patterns grepres = ut.grep_projects(patterns, new=True, verbose=True, cache=cache) modname_fpath_lists += [grepres.found_fpath_list] grep_results = ut.ddict(dict) # Extract public members from each module progiter = ut.ProgIter(list(zip(modnames, modpaths, modname_fpath_lists))) for modname, modpath, found_import_fpath_list in progiter: #progiter.prog_hook( funcname_list = get_funcnames_from_modpath(modpath) for funcname in ut.ProgIter(funcname_list, lbl='funcs'): pattern = '\\b' + funcname + '\\b', # Search which module uses each public member #found_fpath_list, found_lines_list = ut.grep_projects(pattern, new=True, verbose=False, cache=cache) grepres = ut.grep_projects( pattern, new=True, verbose=False, cache=cache, fpath_list=found_import_fpath_list) parent_modnames = ut.lmap(ut.get_modname_from_modpath, grepres.found_fpath_list) grepres.found_modnames = parent_modnames parent_numlines = ut.lmap(len, grepres.found_lines_list) _callgraph = dict(zip(parent_modnames, parent_numlines)) # Remove self references #ut.delete_keys(_callgraph, modnames) grep_results[modname][funcname] = grepres func_call_graph[modname][funcname] = _callgraph print('func_call_graph = %s' % (ut.repr3(func_call_graph),)) import copy func_call_graph2 = copy.deepcopy(func_call_graph) #ignore_modnames = [] ignore_modnames = ['ibeis.algo.hots.multi_index', 'ibeis.algo.hots._neighbor_experiment'] num_callers = ut.ddict(dict) for modname, modpath in list(zip(modnames, modpaths)): subdict = func_call_graph2[modname] for funcname in subdict.keys(): _callgraph = subdict[funcname] ut.delete_keys(_callgraph, modnames) ut.delete_keys(_callgraph, ignore_modnames) num_callers[modname][funcname] = sum(_callgraph.values()) print(ut.dict_str(num_callers[modname], sorted_=True, key_order_metric='val')) # Check external usage unused_external = [] grep_results2 = copy.deepcopy(grep_results) for modname, grepres_subdict in grep_results2.items(): for funcname, grepres_ in grepres_subdict.items(): idxs = ut.find_list_indexes(grepres_.found_modnames, modnames) idxs += ut.find_list_indexes(grepres_.found_modnames, ignore_modnames) idxs = list(ut.filter_Nones(idxs)) ut.delete_items_by_index(grepres_, idxs) ut.delete_items_by_index(grepres_.found_modnames, idxs) if len(grepres_) > 0: print(grepres_.make_resultstr()) else: unused_external += [funcname] print('internal grep') # Check internal usage unused_internal = [] grep_results2 = copy.deepcopy(grep_results) for modname, grepres_subdict in grep_results2.items(): for funcname, grepres_ in grepres_subdict.items(): idxs = ut.filter_Nones(ut.find_list_indexes(grepres_.found_modnames, [modname])) idxs_ = ut.index_complement(idxs, len(grepres_.found_modnames)) ut.delete_items_by_index(grepres_, idxs_) ut.delete_items_by_index(grepres_.found_modnames, idxs_) grepres_.hack_remove_pystuff() #self = grepres_ if len(grepres_) > 0: #print(modname) #print(funcname) #print(grepres_.extended_regex_list) print(grepres_.make_resultstr()) else: unused_internal += [funcname] # HACK: how to write ut.parfor # returns a 0 lenth iterator so the for loop is never run. Then uses code # introspection to determine the content of the for loop body executes code # using the values of the local variables in a parallel / distributed # context. for modname, modpath in zip(modnames, modpaths): pattern = '\\b' + modname + '\\b', grepres = ut.grep_projects(pattern, new=True, verbose=False, cache=cache) parent_modnames = ut.lmap(ut.get_modname_from_modpath, grepres.found_fpath_list) parent_numlines = ut.lmap(len, grepres.found_lines_list) importance = dict(zip(parent_modnames, parent_numlines)) ut.delete_keys(importance, modnames) importance_dict[modname] = importance print('importance_dict = %s' % (ut.repr3(importance_dict),)) combo = reduce(ut.dict_union, importance_dict.values()) print('combined %s' % (ut.repr3(combo),)) # print(ut.repr3(found_fpath_list))
[docs]def help_members(obj): r""" Args: obj (class or module): CommandLine: python -m utool.util_inspect --exec-help_members Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> obj = ut.DynStruct >>> result = help_members(obj) >>> print(result) """ import utool as ut attr_list = [getattr(obj, attrname) for attrname in dir(obj)] type2_items = ut.group_items(attr_list, list(map(ut.type_str, map(type, attr_list)))) memtypes = ['instancemethod'] # , 'method-wrapper'] func_mems = ut.dict_subset(type2_items, memtypes, []) #other_mems = ut.delete_keys(type2_items.copy(), memtypes) func_list = ut.flatten(func_mems.values()) defsig_list = [] num_unbound_args_list = [] num_args_list = [] for func in func_list: #args = ut.get_func_argspec(func).args argspec = ut.get_func_argspec(func) args = argspec.args unbound_args = get_unbound_args(argspec) defsig = ut.func_defsig(func) defsig_list.append(defsig) num_unbound_args_list.append(len(unbound_args)) num_args_list.append(len(args)) group = ut.hierarchical_group_items(defsig_list, [num_unbound_args_list, num_args_list]) print(repr(obj)) print(ut.repr3(group, strvals=True))
[docs]def get_dev_hints(): VAL_FIELD = util_regex.named_field('val', '.*') VAL_BREF = util_regex.bref_field('val') registered_hints = OrderedDict([ # General IBEIS hints ('ibs.*' , ('ibeis.IBEISController', 'image analysis api')), ('testres', ('ibeis.TestResult', 'test result object')), ('qreq_' , ('ibeis.QueryRequest', 'query request object with hyper-parameters')), ('cm' , ('ibeis.ChipMatch', 'object of feature correspondences and scores')), ('qparams*', ('ibeis.QueryParams', 'query hyper-parameters')), ('qaid2_cm.*' , ('dict', 'dict of ``ChipMatch`` objects')), ('vecs' , ('ndarray[uint8_t, ndim=2]', 'descriptor vectors')), ('maws' , ('ndarray[float32_t, ndim=1]', 'multiple assignment weights')), ('words' , ('ndarray[uint8_t, ndim=2]', 'aggregate descriptor cluster centers')), ('word' , ('ndarray[uint8_t, ndim=1]', 'aggregate descriptor cluster center')), ('rvecs' , ('ndarray[uint8_t, ndim=2]', 'residual vector')), ('fm', ('list', 'list of feature matches as tuples (qfx, dfx)')), ('fs', ('list', 'list of feature scores')), ('aid_list' , ('list', 'list of annotation rowids')), ('aids' , ('list', 'list of annotation rowids')), ('aid_list[0-9]' , ('list', 'list of annotation ids')), ('ensure' , ('bool', 'eager evaluation if True')), ('qaid' , ('int', 'query annotation id')), ('aid[0-9]?', ('int', 'annotation id')), ('daids' , ('list', 'database annotation ids')), ('qaids' , ('list', 'query annotation ids')), ('use_cache', ('bool', 'turns on disk based caching')), ('qreq_vsmany_', ('QueryRequest', 'persistant vsmany query request')), ('qnid' , ('int', 'query name id')), # ('gfpath[0-9]?' , ('str', 'image file path string')), ('path[0-9]?' , ('str', 'path to file or directory')), ('n' , ('int', '')), ('ext' , ('str', 'extension')), ('_path' , ('str', 'path string')), ('path_' , ('str', 'path string')), ('.*_dpath' , ('str', 'directory path string')), ('.*_fpath' , ('str', 'file path string')), ('bbox' , ('tuple', 'bounding box in the format (x, y, w, h)')), ('theta' , ('float', 'angle in radians')), ('ori_thresh' , ('float', 'angle in radians')), ('xy_thresh_sqrd' , ('float', '')), ('xy_thresh' , ('float', '')), ('scale_thresh' , ('float', '')), # Pipeline hints ('qaid2_nns', ('dict', 'maps query annotid to (qfx2_idx, qfx2_dist)')), ('qaid2_nnvalid0', ('dict', 'maps query annotid to qfx2_valid0')), ('qfx2_valid0', ('ndarray', 'maps query feature index to K matches non-impossibility flags')), ('filt2_weights', ('dict', 'maps filter names to qfx2_weight ndarray')), ('qaid2_filtweights', ('dict', 'mapping to weights computed by filters like lnnbnn and ratio')), ('qaid2_nnfiltagg', ('dict', 'maps to nnfiltagg - tuple(qfx2_score, qfx2_valid)')), ('qaid2_nnfilts', ('dict', 'nonaggregate feature scores and validities for each feature NEW')), ('nnfiltagg', ('tuple', '(qfx2_score_agg, qfx2_valid_agg)')), ('nnfilts', ('tuple', '(filt_list, qfx2_score_list, qfx2_valid_list)')), ('qfx2_idx', ('ndarray[int32_t, ndims=2]', 'mapping from query feature index to db neighbor index')), ('K' , ('int', None)), ('Knorm' , ('int', None)), # SMK Hints ('smk_alpha', ('float', 'selectivity power')), ('smk_thresh', ('float', 'selectivity threshold')), ('query_sccw', ('float', 'query self-consistency-criterion')), ('data_sccw', ('float', 'data self-consistency-criterion')), ('invindex', ('InvertedIndex', 'object for fast vocab lookup')), # Plotting hints ('[qd]?rchip[0-9]?', ('ndarray[uint8_t, ndim=2]', 'rotated annotation image data')), ('[qd]?chip[0-9]?', ('ndarray[uint8_t, ndim=2]', 'annotation image data')), ('kp', ('ndarray[float32_t, ndim=1]', 'a single keypoint')), ('[qd]?kpts[0-9]?', ('ndarray[float32_t, ndim=2]', 'keypoints')), ('[qd]?vecs[0-9]?', ('ndarray[uint8_t, ndim=2]', 'descriptor vectors')), ('H', ('ndarray[float64_t, ndim=2]', 'homography/perspective matrix')), ('invV_mats2x2', ('ndarray[float32_t, ndim=3]', 'keypoint shapes')), ('invVR_mats2x2', ('ndarray[float32_t, ndim=3]', 'keypoint shape and rotations')), ('invV_mats', ('ndarray[float32_t, ndim=3]', 'keypoint shapes (possibly translation)')), ('invVR_mats', ('ndarray[float32_t, ndim=3]', 'keypoint shape and rotations (possibly translation)')), ('img\d*', ('ndarray[uint8_t, ndim=2]', 'image data')), ('img_in', ('ndarray[uint8_t, ndim=2]', 'image data')), ('arr', ('ndarray', '')), ('arr_', ('ndarray', '')), ('X', ('ndarray', 'data')), ('y', ('ndarray', 'labels')), ('imgBGR', ('ndarray[uint8_t, ndim=2]', 'image data in opencv format (blue, green, red)')), ('pnum', ('tuple', 'plot number')), ('fnum', ('int', 'figure number')), ('title', ('str', '')), ('text', ('str', '')), ('text_', ('str', '')), # Matching Hints ('ratio_thresh' , ('float', None)), # utool hints ('func' , ('function', 'live python function')), ('funcname' , ('str', 'function name')), ('modname' , ('str', 'module name')), ('argname_list' , ('str', 'list of argument names')), ('return_name' , ('str', 'return variable name')), ('dict_' , ('dict_', 'a dictionary')), ('examplecode' , ('str', None)), # Numpy Hints ('shape' , ('tuple', 'array dimensions')), ('chipshape' , ('tuple', 'height, width')), ('rng' , ('RandomState', 'random number generator')), # Opencv hings ('dsize' , ('tuple', 'width, height')), ('chipsize' , ('tuple', 'width, height')), # Standard Python Hints for my coding style ('.*_fn' , ('func', None)), ('str_' , ('str', None)), ('num_.*' , ('int', None)), ('.*_str' , ('str', None)), ('.*_?list_?' , ('list', None)), ('.*_?dict_?' , ('dict', None)), ('dict_?\d?' , ('dict', None)), ('.*_tup' , ('tuple', None)), ('.*_sublist' , ('list', None)), ('fpath[0-9]?' , ('str', 'file path string')), ('chip[A-Z]*' , ('ndarray', 'cropped image')), ('verbose', ('bool', 'verbosity flag')), # Other hints for my coding style ('wx2_' , ('dict', None)), ('qfx2_' + VAL_FIELD, ('ndarray', 'mapping from query feature index to ' + VAL_BREF)), ('.*x2_.*' , ('ndarray', None)), ('.+[^3]2_.*' , ('dict', None)), ('dpath' , ('str', 'directory path')), ('dname' , ('str', 'directory name')), ('fpath' , ('str', 'file path')), ('fname' , ('str', 'file name')), ('pattern' , ('str', '')), ]) return registered_hints
[docs]def infer_arg_types_and_descriptions(argname_list, defaults): """ Args: argname_list (list): defaults (list): Returns: tuple : (argtype_list, argdesc_list) CommandLine: python -m utool.util_inspect --test-infer_arg_types_and_descriptions Ignore: python -c "import utool; print(utool.auto_docstr('ibeis.algo.hots.pipeline', 'build_chipmatches'))" Example: >>> # ENABLE_DOCTEST >>> import utool >>> argname_list = ['ibs', 'qaid', 'fdKfds', 'qfx2_foo'] >>> defaults = None >>> tup = utool.infer_arg_types_and_descriptions(argname_list, defaults) >>> argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup """ #import utool as ut from utool import util_dev # hacks for IBEIS if True or util_dev.is_developer(): registered_hints = get_dev_hints() # key = regex pattern # val = hint=tuple(type_, desc_) if defaults is None: defaults = [] default_types = [type(val).__name__.replace('NoneType', 'None') for val in defaults] num_defaults = len(defaults) num_nodefaults = len(argname_list) - num_defaults argtype_list = ['?'] * (num_nodefaults) + default_types # defaults aligned with argtype_list and argdesc_list argdefault_list = [None] * num_nodefaults + list(defaults) hasdefault_list = [False] * num_nodefaults + [True] * num_defaults argdesc_list = ['' for _ in range(len(argname_list))] # use hints to build better docstrs for argx in range(len(argname_list)): #if argtype_list[argx] == '?' or argtype_list[argx] == 'None': argname = argname_list[argx] if argname is None: #print('warning argname is None') continue for regex, hint in six.iteritems(registered_hints): matchobj = re.match('^' + regex + '$', argname, flags=re.MULTILINE | re.DOTALL) if matchobj is not None: type_ = hint[0] desc_ = hint[1] if type_ is not None: if argtype_list[argx] == '?' or argtype_list[argx] == 'None': argtype_list[argx] = type_ if desc_ is not None: desc_ = matchobj.expand(desc_) argdesc_list[argx] = ' ' + desc_ break # append defaults to descriptions for argx in range(len(argdesc_list)): if hasdefault_list[argx]: if isinstance(argdefault_list[argx], types.ModuleType): defaultrepr = argdefault_list[argx].__name__ else: defaultrepr = repr(argdefault_list[argx]) #import utool as ut #ut.embed() argdesc_list[argx] += '(default = %s)' % (defaultrepr,) return argtype_list, argdesc_list, argdefault_list, hasdefault_list
[docs]def get_module_owned_functions(module): """ Replace with iter_module_doctesable (but change that name to be something better) returns functions actually owned by the module module = vtool.distance """ import utool as ut list_ = [] for key, val in ut.iter_module_doctestable(module): if hasattr(val, '__module__'): belongs = val.__module__ == module.__name__ elif hasattr(val, 'func_globals'): belongs = val.func_globals['__name__'] == module.__name__ if belongs: list_.append(val) return list_
[docs]def zzz_profiled_is_no(): pass
@profile
[docs]def zzz_profiled_is_yes(): pass
[docs]def iter_module_doctestable(module, include_funcs=True, include_classes=True, include_methods=True, include_builtin=True, include_inherited=False, debug_key=None): r""" Yeilds doctestable live object form a modules TODO: change name to iter_module_members Replace with iter_module_doctesable (but change that name to be something better) Args: module (module): live python module include_funcs (bool): include_classes (bool): include_methods (bool): include_builtin (bool): (default = True) include_inherited (bool): (default = False) Yeilds: tuple (str, callable): (funcname, func) doctestable CommandLine: python -m utool --tf iter_module_doctestable \ --modname=ibeis.algo.hots.chip_match --modname=ibeis.control.IBEISControl --modname=ibeis.control.SQLDatabaseControl --modname=ibeis.control.manual_annot_funcs --modname=ibeis.control.manual_annot_funcs --modname=ibeis.expt.test_result --modname=utool.util_progress --debug-key=build_msg_fmtstr_time2 --modname=utool.util_progress --debug-key=ProgressIter Debug: # fix profile with doctest utprof.py -m utool --tf iter_module_doctestable --modname=utool.util_inspect --debugkey=zzz_profiled_is_yes Example1: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> modname = ut.get_argval('--modname', type_=str, default=None) >>> kwargs = ut.argparse_funckw(iter_module_doctestable) >>> module = ut.util_tests if modname is None else ut.import_modname(modname) >>> debug_key = ut.get_argval('--debugkey', type_=str, default=None) >>> kwargs['debug_key'] = debug_key >>> doctestable_list = list(iter_module_doctestable(module, **kwargs)) >>> func_names = sorted(ut.get_list_column(doctestable_list, 0)) >>> print(ut.list_str(func_names)) """ import ctypes types.BuiltinFunctionType valid_func_types = [ types.FunctionType, types.BuiltinFunctionType, classmethod, staticmethod, #types.MethodType, types.BuiltinMethodType, ] if include_builtin: valid_func_types += [ types.BuiltinFunctionType ] if six.PY2: valid_class_types = (types.ClassType, types.TypeType,) else: valid_class_types = six.class_types scalar_types = ([dict, list, tuple, set, frozenset, bool, float, int] + list(six.string_types)) scalar_types += list(six.string_types) other_types = [functools.partial, types.ModuleType, ctypes.CDLL] if six.PY2: other_types += [types.InstanceType] invalid_types = tuple(scalar_types + other_types) valid_func_types = tuple(valid_func_types) #modpath = ut.get_modname_from_modpath(module.__file__) for key, val in six.iteritems(module.__dict__): # <DEBUG> if debug_key is not None and key == debug_key: print('DEBUG') print('debug_key = %r' % (debug_key,)) exec('item = val') # import utool as ut # ut.embed() # </DEBUG> if hasattr(val, '__module__'): # HACK: todo. figure out true parent module if val.__module__ == 'numpy': continue if val is None: pass elif isinstance(val, valid_func_types): if include_funcs: if not include_inherited and not is_defined_by_module(val, module): continue yield key, val elif isinstance(val, valid_class_types): class_ = val if not include_inherited and not is_defined_by_module(class_, module): continue if include_classes: # Yield the class itself yield key, val if include_methods: # Yield methods of the class for subkey, subval in six.iteritems(class_.__dict__): if isinstance(subval, property): subval = subval.fget # <DEBUG> if debug_key is not None and subkey == debug_key: import utool as ut ut.embed() # </DEBUG> # Unbound methods are still typed as functions if isinstance(subval, valid_func_types): if not include_inherited and not is_defined_by_module(subval, module): continue if isinstance(subval, (staticmethod)): subval.__func__.__ut_parent_class__ = class_ if not isinstance(subval, types.BuiltinFunctionType) and not isinstance(subval, (classmethod, staticmethod)): # HACK: __ut_parent_class__ lets util_test have # more info on the func should return extra info # instead subval.__ut_parent_class__ = class_ yield subkey, subval elif isinstance(val, invalid_types): pass else: if util_arg.VERBOSE: print('[util_inspect] WARNING module %r class %r:' % (module, class_,)) print(' * Unknown if testable val=%r' % (val)) print(' * Unknown if testable type(val)=%r' % type(val)) elif isinstance(val, invalid_types): pass else: #import utool as ut if util_arg.VERBOSE: print('[util_inspect] WARNING in module %r:' % (module,)) print(' * Unknown if testable val=%r' % (val)) print(' * Unknown if testable type(val)=%r' % type(val))
[docs]def is_defined_by_module(item, module): """ Check if item is directly defined by a module. This check may be prone to errors. """ flag = False if isinstance(item, types.ModuleType): if not hasattr(item, '__file__'): flag = False else: item_modpath = os.path.realpath(dirname(item.__file__)) mod_fpath = module.__file__.replace('.pyc', '.py') if not mod_fpath.endswith('__init__.py'): flag = False else: modpath = os.path.realpath(dirname(mod_fpath)) modpath = modpath.replace('.pyc', '.py') flag = item_modpath.startswith(modpath) elif hasattr(item, '_utinfo'): # Capture case where there is a utool wrapper orig_func = item._utinfo['orig_func'] flag = is_defined_by_module(orig_func, module) else: if isinstance(item, staticmethod): # static methods are a wrapper around a function item = item.__func__ try: func_globals = meta_util_six.get_funcglobals(item) func_module_name = func_globals['__name__'] if func_module_name == 'line_profiler': if item.func_name in dir(module) and len(item.func_name) > 8: flag = True elif func_module_name == module.__name__: flag = True except AttributeError: if hasattr(item, '__module__'): flag = item.__module__ == module.__name__ return flag
[docs]def get_func_modname(func): if hasattr(func, '_utinfo'): # Capture case where there is a utool wrapper orig_func = func._utinfo['orig_func'] return get_func_modname(orig_func) #try: func_globals = meta_util_six.get_funcglobals(func) modname = func_globals['__name__'] return modname #except AttributeError: # pass #pass
[docs]def is_bateries_included(item): """ Returns if a value is a python builtin function Args: item (object): Returns: bool: flag References: http://stackoverflow.com/questions/23149218/check-if-a-python-function-is-builtin CommandLine: python -m utool._internal.meta_util_six --exec-is_builtin Example: >>> # DISABLE_DOCTEST >>> from utool._internal.meta_util_six import * # NOQA >>> item = zip >>> flag = is_bateries_included(item) >>> result = ('flag = %s' % (str(flag),)) >>> print(result) """ flag = False if hasattr(item, '__call__') and hasattr(item, '__module__'): if item.__module__ is not None: module = sys.modules[item.__module__] if module == builtins: flag = True elif hasattr(module, '__file__'): flag = LIB_PATH == dirname(module.__file__) return flag
[docs]def list_class_funcnames(fname, blank_pats=[' #']): """ list_class_funcnames Args: fname (str): filepath blank_pats (list): defaults to ' #' Returns: list: funcname_list Example: >>> from utool.util_inspect import * # NOQA >>> fname = 'util_class.py' >>> blank_pats = [' #'] >>> funcname_list = list_class_funcnames(fname, blank_pats) >>> print(funcname_list) """ with open(fname, 'r') as file_: lines = file_.readlines() funcname_list = [] #full_line_ = '' for lx, line in enumerate(lines): #full_line_ += line if any([line.startswith(pat) for pat in blank_pats]): funcname_list.append('') if line.startswith(' def '): def_x = line.find('def') rparen_x = line.find('(') funcname = line[(def_x + 3):rparen_x] #print(funcname) funcname_list.append(funcname) return funcname_list
[docs]def list_global_funcnames(fname, blank_pats=[' #']): """ list_global_funcnames Args: fname (str): filepath blank_pats (list): defaults to ' #' Returns: list: funcname_list Example: >>> from utool.util_inspect import * # NOQA >>> fname = 'util_class.py' >>> blank_pats = [' #'] >>> funcname_list = list_global_funcnames(fname, blank_pats) >>> print(funcname_list) """ with open(fname, 'r') as file_: lines = file_.readlines() funcname_list = [] #full_line_ = '' for lx, line in enumerate(lines): #full_line_ += line if any([line.startswith(pat) for pat in blank_pats]): funcname_list.append('') if line.startswith('def '): def_x = line.find('def') rparen_x = line.find('(') funcname = line[(def_x + 3):rparen_x] #print(funcname) funcname_list.append(funcname) return funcname_list
[docs]def inherit_kwargs(inherit_func): """ TODO move to util_decor inherit_func = inspect_pdfs func = encoder.visualize.im_func """ import utool as ut keys, is_arbitrary = ut.get_kwargs(inherit_func) if is_arbitrary: keys += ['**kwargs'] kwargs_append = '\n'.join(keys) #from six.moves import builtins #builtins.print(kwargs_block) def _wrp(func): if func.__doc__ is None: func.__doc__ = '' # TODO append to kwargs block if it exists kwargs_block = 'Kwargs:\n' + ut.indent(kwargs_append) func.__doc__ += kwargs_block return func return _wrp
[docs]def filter_valid_kwargs(func, dict_): import utool as ut keys, is_arbitrary = ut.get_kwargs(func) if is_arbitrary: valid_dict_ = dict_ else: key_subset = ut.dict_keysubset(dict_, keys) valid_dict_ = ut.dict_subset(dict_, key_subset) return valid_dict_
[docs]def dummy_func(arg1, arg2, arg3=None, arg4=[1, 2, 3], arg5={}, **kwargs): """ test func for kwargs parseing """ foo = kwargs.get('foo', None) bar = kwargs.pop('bar', 4) foobar = str(foo) + str(bar) return foobar
[docs]def get_kwdefaults2(func, parse_source=False): return get_kwdefaults(func, parse_source=True)
[docs]def get_kwdefaults(func, parse_source=False): r""" Args: func (func): Returns: dict: CommandLine: python -m utool.util_inspect --exec-get_kwdefaults Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> func = dummy_func >>> parse_source = True >>> kwdefaults = get_kwdefaults(func, parse_source) >>> print('kwdefaults = %s' % (ut.dict_str(kwdefaults),)) """ #import utool as ut #with ut.embed_on_exception_context: argspec = inspect.getargspec(func) kwdefaults = {} if argspec.args is None or argspec.defaults is None: pass else: args = argspec.args defaults = argspec.defaults #kwdefaults = OrderedDict(zip(argspec.args[::-1], argspec.defaults[::-1])) kwpos = len(args) - len(defaults) kwdefaults = OrderedDict(zip(args[kwpos:], defaults)) if parse_source and argspec.keywords: # TODO parse for kwargs.get/pop keyword_defaults = parse_func_kwarg_keys(func, with_vals=True) for key, val in keyword_defaults: assert key not in kwdefaults, 'parsing error' kwdefaults[key] = val return kwdefaults
[docs]def get_argnames(func): argspec = inspect.getargspec(func) argnames = argspec.args return argnames
[docs]def get_funcname(func): return meta_util_six.get_funcname(func)
[docs]def set_funcname(func, newname): return meta_util_six.set_funcname(func, newname)
[docs]def get_imfunc(func): return meta_util_six.get_imfunc(func)
[docs]def get_funcglobals(func): return meta_util_six.get_funcglobals(func)
[docs]def get_funcdoc(func): return meta_util_six.get_funcdoc(func)
[docs]def get_funcfpath(func): return func.func_code.co_filename
[docs]def set_funcdoc(func, newdoc): return meta_util_six.set_funcdoc(func, newdoc)
[docs]def get_docstr(func_or_class): """ Get the docstring from a live object """ import utool as ut try: docstr_ = func_or_class.func_doc except AttributeError: docstr_ = func_or_class.__doc__ if docstr_ is None: docstr_ = '' docstr = ut.unindent(docstr_) return docstr
[docs]def prettyprint_parsetree(pt): """ pip install astdump pip install codegen """ #import astdump import astor #import codegen #import ast #astdump.indented(pt) #print(ast.dump(pt, include_attributes=True)) print(astor.dump(pt))
[docs]def find_child_kwarg_funcs(sourcecode, target_kwargs_name='kwargs'): r""" CommandLine: python3 -m utool.util_inspect --exec-find_child_kwarg_funcs Example: >>> # ENABLE_DOCTEST >>> import utool as ut >>> sourcecode = ut.codeblock( ''' warped_patch1_list, warped_patch2_list = list(zip(*ut.ichunks(data, 2))) interact_patches(labels, warped_patch1_list, warped_patch2_list, flat_metadata, **kwargs) import sys sys.badcall(**kwargs) def foo(): bar(**kwargs) ut.holymoly(**kwargs) baz() def biz(**kwargs): foo2(**kwargs) ''') >>> child_funcnamess = ut.find_child_kwarg_funcs(sourcecode) >>> print('child_funcnamess = %r' % (child_funcnamess,)) >>> assert 'foo2' not in child_funcnamess, 'foo2 should not be found' >>> assert 'bar' in child_funcnamess, 'bar should be found' Notes: """ import ast sourcecode = 'from __future__ import print_function\n' + sourcecode pt = ast.parse(sourcecode) child_funcnamess = [] debug = False or VERYVERB_INSPECT if debug: print('\nInput:') print('target_kwargs_name = %r' % (target_kwargs_name,)) print('\nSource:') print(sourcecode) import astor print('\nParse:') print(astor.dump(pt)) class KwargParseVisitor(ast.NodeVisitor): """ TODO: understand ut.update_existing and dict update """ def visit_FunctionDef(self, node): if debug: print('\nVISIT FunctionDef node = %r' % (node,)) print('node.args.kwarg = %r' % (node.args.kwarg,)) if six.PY2: kwarg_name = node.args.kwarg else: if node.args.kwarg is None: kwarg_name = None else: kwarg_name = node.args.kwarg.arg #import utool as ut #ut.embed() if kwarg_name != target_kwargs_name: # target kwargs is still in scope ast.NodeVisitor.generic_visit(self, node) def visit_Call(self, node): if debug: print('\nVISIT Call node = %r' % (node,)) #print(ut.dict_str(node.__dict__,)) if isinstance(node.func, ast.Attribute): funcname = node.func.value.id + '.' + node.func.attr elif isinstance(node.func, ast.Name): funcname = node.func.id else: raise NotImplementedError( 'do not know how to parse: node.func = %r' % (node.func,)) kwargs = node.kwargs kwargs_name = None if kwargs is None else kwargs.id if kwargs_name == target_kwargs_name: child_funcnamess.append(funcname) if debug: print('funcname = %r' % (funcname,)) print('kwargs_name = %r' % (kwargs_name,)) ast.NodeVisitor.generic_visit(self, node) try: KwargParseVisitor().visit(pt) except Exception: pass #import utool as ut #if ut.SUPER_STRICT: # raise return child_funcnamess #print('child_funcnamess = %r' % (child_funcnamess,))
[docs]def is_valid_python(code, reraise=True, ipy_magic_workaround=False): """ References: http://stackoverflow.com/questions/23576681/python-check-syntax """ import ast try: if ipy_magic_workaround: code = '\n'.join(['pass' if re.match(r'\s*%[a-z]*', line) else line for line in code.split('\n')]) ast.parse(code) except SyntaxError: if reraise: import utool as ut print('Syntax Error') ut.print_python_code(code) raise return False return True
[docs]def parse_return_type(sourcecode): r""" parse_return_type Args: sourcecode (?): Returns: tuple: (return_type, return_name, return_header) Ignore: testcase automated_helpers query_vsone_verified CommandLine: python -m utool.util_inspect --exec-parse_return_type python -m utool.util_inspect --test-parse_return_type Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> sourcecode = ut.codeblock( ... 'def foo(tmp=False):\n' ... ' bar = True\n' ... ' return bar\n' ... ) >>> returninfo = parse_return_type(sourcecode) >>> result = ut.repr2(returninfo) >>> print(result) ('?', 'bar', 'Returns', '') Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> sourcecode = ut.codeblock( ... 'def foo(tmp=False):\n' ... ' return True\n' ... ) >>> returninfo = parse_return_type(sourcecode) >>> result = ut.repr2(returninfo) >>> print(result) ('bool', 'True', 'Returns', '') Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> sourcecode = ut.codeblock( ... 'def foo(tmp=False):\n' ... ' for i in range(2): \n' ... ' yield i\n' ... ) >>> returninfo = parse_return_type(sourcecode) >>> result = ut.repr2(returninfo) >>> print(result) ('?', 'i', 'Yields', '') Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> sourcecode = ut.codeblock( ... 'def foo(tmp=False):\n' ... ' if tmp is True:\n' ... ' return (True, False)\n' ... ' elif tmp is False:\n' ... ' return 1\n' ... ' else:\n' ... ' bar = baz()\n' ... ' return bar\n' ... ) >>> returninfo = parse_return_type(sourcecode) >>> result = ut.repr2(returninfo) >>> print(result) ('tuple', '(True, False)', 'Returns', '') """ import ast return_type, return_name, return_header = (None, None, None) if sourcecode is None: return return_type, return_name, return_header, None sourcecode = 'from __future__ import print_function\n' + sourcecode pt = ast.parse(sourcecode) import utool as ut debug = ut.get_argflag('--debug-parse-return') #debug = True if debug: import astor print('\nSource:') print(sourcecode) print('\nParse:') print(astor.dump(pt)) print('... starting') def print_visit(type_, node): if debug: import utool as ut print('+---') print('\nVISIT %s node = %r' % (type_, node,)) print('node.__dict__ = ' + ut.repr2(node.__dict__, nl=True)) print('L___') def get_node_name_and_type(node): if isinstance(node, ast.Tuple): tupnode_list = node.elts tupleid = '(%s)' % (', '.join([str(get_node_name_and_type(tupnode)[1]) for tupnode in tupnode_list])) node_type = 'tuple' node_name = tupleid elif isinstance(node, ast.Dict): node_type = 'dict' node_name = None elif isinstance(node, ast.Name): node_name = node.id node_type = '?' if node_name in ['True', 'False']: node_type = 'bool' elif node_name == 'None': node_type = 'None' elif six.PY3 and isinstance(node, ast.NameConstant): node_name = str(node.value) node_type = '?' if node_name in ['True', 'False', True, False]: node_type = 'bool' elif node_name in ['None', None]: node_type = 'None' else: node_name = None node_type = '?' #node_type = 'ADD_TO_GET_NODE_NAME_AND_TYPE: ' + str(type(node.value)) return node_type, node_name class ReturnVisitor(ast.NodeVisitor): def init(self): self.found_nodes = [] self.return_header = None def visit_FunctionDef(self, node): print_visit('FunctionDef', node) # TODO: ignore subfunction return types ast.NodeVisitor.generic_visit(self, node) def visit_Return(self, node): print_visit('Return', node) ast.NodeVisitor.generic_visit(self, node) return_value = node.value print_visit('ReturnValue', return_value) self.found_nodes.append(return_value) self.return_header = 'Returns' def visit_Yield(self, node): print_visit('Yield', node) ast.NodeVisitor.generic_visit(self, node) return_value = node.value print_visit('YieldValue', return_value) self.found_nodes.append(return_value) self.return_header = 'Yields' try: self = ReturnVisitor() self.init() self.visit(pt) return_header = self.return_header if len(self.found_nodes) > 0: # hack rectify multiple return values node = self.found_nodes[0] return_type, return_name = get_node_name_and_type(node) else: return_name = None return_type = 'None' except Exception: if debug: raise return_desc = '' if return_type == '?': tup = infer_arg_types_and_descriptions([return_name], []) argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup return_type = argtype_list[0] return_desc = argdesc_list[0] return return_type, return_name, return_header, return_desc #def parse_return_type_OLD(sourcecode): # import utool as ut # import ast # if ut.VERBOSE: # print('[utool] parsing return types') # if sourcecode is None: # return_type, return_name, return_header = (None, None, None) # return return_type, return_name, return_header, None # #source_lines = sourcecode.splitlines() # sourcecode = 'from __future__ import print_function\n' + sourcecode # try: # pt = ast.parse(sourcecode) # except Exception: # return_type, return_name, return_header = (None, None, None) # #raise # return return_type, return_name, return_header, None # #print(sourcecode) # #ut.printex(ex, 'Error Parsing') # assert isinstance(pt, ast.Module), str(type(pt)) # Try = ast.Try if six.PY3 else ast.TryExcept # def find_function_nodes(pt): # function_nodes = [] # for node in pt.body: # if isinstance(node, ast.FunctionDef): # function_nodes.append(node) # return function_nodes # function_nodes = find_function_nodes(pt) # assert len(function_nodes) == 1 # func_node = function_nodes[0] # def find_return_node(node): # if isinstance(node, list): # candidates = [] # node_list = node # for node in node_list: # candidate = find_return_node(node) # if candidate is not None: # candidates.append(candidate) # if len(candidates) > 0: # return candidates[0] # elif isinstance(node, (ast.Return, ast.Yield)): # return node # elif isinstance(node, (ast.If, Try)): # return find_return_node(node.body) # else: # pass # #print(type(node)) # if ut.VERBOSE: # print('[utool] parsing return types') # returnnode = find_return_node(func_node.body) # # Check return or yeild # if isinstance(returnnode, ast.Yield): # return_header = 'Yeilds' # elif isinstance(returnnode, ast.Return): # return_header = 'Returns' # else: # return_header = None # # Get more return info # def get_node_name_and_type(node): # node_name = None # node_type = '?' # if node is None: # node_type = 'None' # elif isinstance(node.value, ast.Tuple): # tupnode_list = node.value.elts # def get_tuple_membername(tupnode): # if hasattr(tupnode, 'id'): # return tupnode.id # elif hasattr(tupnode, 'value'): # return 'None' # else: # return 'None' # pass # tupleid = '(%s)' % (', '.join([str(get_tuple_membername(tupnode)) for tupnode in tupnode_list])) # node_type = 'tuple' # node_name = tupleid # #node_name = ast.dump(node) # elif isinstance(node.value, ast.Dict): # node_type = 'dict' # node_name = None # elif isinstance(node.value, ast.Name): # node_name = node.value.id # if node_name == 'True': # node_name = 'True' # node_type = 'bool' # else: # #node_type = 'ADD_TO_GET_NODE_NAME_AND_TYPE: ' + str(type(node.value)) # node_type = '?' # return node_type, node_name # return_type, return_name = get_node_name_and_type(returnnode) # if return_type == '?': # tup = infer_arg_types_and_descriptions([return_name], []) # argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup # return_type = argtype_list[0] # return_desc = argdesc_list[0] # else: # return_desc = '' # return return_type, return_name, return_header, return_desc
[docs]def exec_func_src(func, globals_=None, locals_=None, key_list=None, sentinal=None, update=None): """ execs a func and returns requested local vars. Does not modify globals unless update=True (or in IPython) SeeAlso: ut.execstr_funckw """ import utool as ut sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True) if update is None: update = ut.inIPython() if globals_ is None: globals_ = ut.get_parent_globals() if locals_ is None: locals_ = ut.get_parent_locals() if sentinal is not None: sourcecode = ut.replace_between_tags(sourcecode, '', sentinal) globals_new = globals_.copy() if locals_ is not None: globals_new.update(locals_) orig_globals = globals_new.copy() #six.exec_(sourcecode, globals_new, locals_) six.exec_(sourcecode, globals_new) # Draw intermediate steps if key_list is None: #return locals_ # Remove keys created in function execution ut.delete_keys(globals_new, orig_globals.keys()) if update: # update input globals? globals_.update(globals_new) # ~~ TODO autodetermine the key_list from the function vars return globals_new else: #var_list = ut.dict_take(locals_, key_list) var_list = ut.dict_take(globals_new, key_list) return var_list
[docs]def get_func_sourcecode(func, stripdef=False, stripret=False, strip_docstr=False, strip_comments=False, remove_linenums=None): """ wrapper around inspect.getsource but takes into account utool decorators strip flags are very hacky as of now Args: func (function): stripdef (bool): stripret (bool): (default = False) strip_docstr (bool): (default = False) strip_comments (bool): (default = False) remove_linenums (None): (default = None) CommandLine: python -m utool.util_inspect --test-get_func_sourcecode Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> # build test data >>> func = get_func_sourcecode >>> stripdef = True >>> stripret = True >>> sourcecode = get_func_sourcecode(func, stripdef) >>> # verify results >>> print(result) """ import utool as ut #try: inspect.linecache.clearcache() # HACK: fix inspect bug sourcefile = inspect.getsourcefile(func) #except IOError: # sourcefile = None if hasattr(func, '_utinfo'): #if 'src' in func._utinfo: # sourcecode = func._utinfo['src'] #else: func2 = func._utinfo['orig_func'] sourcecode = get_func_sourcecode(func2) elif sourcefile is not None and sourcefile != '<string>': try_limit = 2 for num_tries in range(try_limit): try: #print(func) sourcecode = inspect.getsource(func) break #print(sourcecode) except (IndexError, OSError, SyntaxError) as ex: ut.printex(ex, 'Error getting source', keys=['sourcefile', 'func']) if False: # VERY HACK: try to reload the module and get a redefined # version of the function import imp modname = get_func_modname(func) funcname = ut.get_funcname(func) module = sys.modules[modname] # TODO: ut.reload_module() module = imp.reload(module) func = module.__dict__[funcname] else: # Fix inspect bug in python2.7 inspect.linecache.clearcache() if num_tries + 1 != try_limit: tries_left = try_limit - num_tries - 1 print('Attempting %d more time(s)' % (tries_left)) else: raise else: sourcecode = None #orig_source = sourcecode #print(orig_source) if stripdef: # hacky sourcecode = ut.unindent(sourcecode) #sourcecode = ut.unindent(ut.regex_replace('def [^)]*\\):\n', '', sourcecode)) #nodef_source = ut.regex_replace('def [^:]*\\):\n', '', sourcecode) regex_decor = '^@.' + ut.REGEX_NONGREEDY regex_defline = '^def [^:]*\\):\n' nodef_source = ut.regex_replace('(' + regex_decor + ')?' + regex_defline, '', sourcecode) sourcecode = ut.unindent(nodef_source) #print(sourcecode) pass if stripret: r""" \s is a whitespace char """ return_ = ut.named_field('return', 'return .*$') prereturn = ut.named_field('prereturn', r'^\s*') return_bref = ut.bref_field('return') prereturn_bref = ut.bref_field('prereturn') regex = prereturn + return_ repl = prereturn_bref + 'pass # ' + return_bref #import re #print(re.search(regex, sourcecode, flags=re.MULTILINE )) #print(re.search('return', sourcecode, flags=re.MULTILINE | re.DOTALL )) #print(re.search(regex, sourcecode)) sourcecode_ = re.sub(regex, repl, sourcecode, flags=re.MULTILINE) #print(sourcecode_) sourcecode = sourcecode_ pass if strip_docstr or strip_comments: # pip install pyminifier # References: http://code.activestate.com/recipes/576704/ #from pyminifier import minification, token_utils def remove_docstrings_or_comments(source): """ TODO: commit clean version to pyminifier """ import tokenize from six.moves import StringIO io_obj = StringIO(source) out = '' prev_toktype = tokenize.INDENT last_lineno = -1 last_col = 0 for tok in tokenize.generate_tokens(io_obj.readline): token_type = tok[0] token_string = tok[1] start_line, start_col = tok[2] end_line, end_col = tok[3] if start_line > last_lineno: last_col = 0 if start_col > last_col: out += (' ' * (start_col - last_col)) # Remove comments: if strip_comments and token_type == tokenize.COMMENT: pass elif strip_docstr and token_type == tokenize.STRING: if prev_toktype != tokenize.INDENT: # This is likely a docstring; double-check we're not inside an operator: if prev_toktype != tokenize.NEWLINE: if start_col > 0: out += token_string else: out += token_string prev_toktype = token_type last_col = end_col last_lineno = end_line return out sourcecode = remove_docstrings_or_comments(sourcecode) #sourcecode = minification.remove_comments_and_docstrings(sourcecode) #tokens = token_utils.listified_tokenizer(sourcecode) #minification.remove_comments(tokens) #minification.remove_docstrings(tokens) #token_utils.untokenize(tokens) if remove_linenums is not None: source_lines = sourcecode.strip('\n').split('\n') ut.delete_items_by_index(source_lines, remove_linenums) sourcecode = '\n'.join(source_lines) return sourcecode #else: #return get_func_sourcecode(func._utinfo['src'])
[docs]def get_unbound_args(argspec): try: args = argspec.args except Exception: func = argspec argspec = get_func_argspec(func) args = argspec.args args = argspec.args defaults = argspec.defaults if defaults is not None: kwpos = len(args) - len(defaults) unbound_args = args[:kwpos] else: unbound_args = args return unbound_args
[docs]def get_func_argspec(func): """ wrapper around inspect.getargspec but takes into account utool decorators """ if hasattr(func, '_utinfo'): argspec = func._utinfo['orig_argspec'] return argspec if isinstance(func, property): func = func.fget argspec = inspect.getargspec(func) return argspec
[docs]def get_kwargs(func): """ Args: func (function): Returns: tuple: keys, is_arbitrary keys (list): kwargs keys is_arbitrary (bool): has generic **kwargs CommandLine: python -m utool.util_inspect --test-get_kwargs Ignore: def func1(a, b, c): pass def func2(a, b, c, *args): pass def func3(a, b, c, *args, **kwargs): pass def func4(a, b=1, c=2): pass def func5(a, b=1, c=2, *args): pass def func6(a, b=1, c=2, **kwargs): pass def func7(a, b=1, c=2, *args, **kwargs): pass for func in [locals()['func' + str(x)] for x in range(1, 8)]: print(inspect.getargspec(func)) Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> # build test data >>> func = '?' >>> result = get_kwargs(func) >>> # verify results >>> print(result) """ #if argspec.keywords is None: import utool as ut argspec = inspect.getargspec(func) if argspec.defaults is not None: num_args = len(argspec.args) num_keys = len(argspec.defaults) keys = ut.take(argspec.args, range(num_args - num_keys, num_args)) else: keys = [] is_arbitrary = argspec.keywords is not None RECURSIVE = False if RECURSIVE and argspec.keywords is not None: pass # TODO: look inside function at the functions that the kwargs object is being # passed to return keys, is_arbitrary
[docs]def lookup_attribute_chain(attrname, namespace): """ >>> attrname = funcname >>> namespace = mod.__dict__ >>> import utool as ut >>> globals_ = ut.util_inspect.__dict__ >>> attrname = 'KWReg.print_defaultkw' """ #subdict = meta_util_six.get_funcglobals(root_func) subtup = attrname.split('.') subdict = namespace for attr in subtup[:-1]: subdict = subdict[attr].__dict__ leaf_name = subtup[-1] leaf_attr = subdict[leaf_name] return leaf_attr
[docs]def recursive_parse_kwargs(root_func, path_=None): """ recursive kwargs parser FIXME: if docstr indentation is off, this fails Args: root_func (function): live python function path_ (None): (default = None) Returns: list: CommandLine: python -m utool.util_inspect --exec-recursive_parse_kwargs:0 python -m utool.util_inspect --exec-recursive_parse_kwargs:1 python -m utool.util_inspect --exec-recursive_parse_kwargs:2 --mod plottool --func draw_histogram python -m utool.util_inspect --exec-recursive_parse_kwargs:2 --mod vtool --func ScoreNormalizer.visualize python -m utool.util_inspect --exec-recursive_parse_kwargs:2 --mod ibeis.viz.viz_matches --func show_name_matches --verbinspect Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> root_func = iter_module_doctestable >>> path_ = None >>> result = ut.repr2(recursive_parse_kwargs(root_func), nl=1) >>> print(result) [ ('include_funcs', True), ('include_classes', True), ('include_methods', True), ('include_builtin', True), ('include_inherited', False), ('debug_key', None), ] Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> from ibeis.algo.hots import chip_match >>> import utool as ut >>> root_func = chip_match.ChipMatch.show_ranked_matches >>> path_ = None >>> result = ut.repr2(recursive_parse_kwargs(root_func)) >>> print(result) Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> modname = ut.get_argval('--mod', type_=str, default='plottool') >>> funcname = ut.get_argval('--func', type_=str, default='draw_histogram') >>> mod = ut.import_modname(modname) >>> root_func = lookup_attribute_chain(funcname, mod.__dict__) >>> path_ = None >>> parsed = recursive_parse_kwargs(root_func) >>> result = ut.repr2(parsed) >>> print(result) """ if VERBOSE_INSPECT: print('[inspect] recursive parse kwargs root_func = %r ' % (root_func,)) import utool as ut if path_ is None: path_ = [] if root_func in path_: return [] path_.append(root_func) spec = ut.get_func_argspec(root_func) # ADD MORE kwargs_list = list(ut.get_kwdefaults(root_func, parse_source=False).items()) #kwargs_list = [(kw,) for kw in ut.get_kwargs(root_func)[0]] sourcecode = ut.get_func_sourcecode(root_func, strip_docstr=True, stripdef=True) kwargs_list += ut.parse_kwarg_keys(sourcecode, with_vals=True) def hack_lookup_mod_attrs(attr): # HACKS TODO: have find_child_kwarg_funcs infer an attribute is a # module / function / type. In the module case, we can import it and # look it up. Maybe args, or returns can help infer type. Maybe just # register some known varnames. Maybe jedi has some better way to do # this. if attr == 'ut': subdict = ut.__dict__ elif attr == 'pt': import plottool as pt subdict = pt.__dict__ else: subdict = None return subdict def resolve_attr_subfunc(subfunc_name): # look up attriute chain #subdict = root_func.func_globals subdict = meta_util_six.get_funcglobals(root_func) subtup = subfunc_name.split('.') try: subdict = lookup_attribute_chain(subfunc_name, subdict) if ut.is_func_or_method(subdict): # Was subdict supposed to be named something else here? subfunc = subdict return subfunc except (KeyError, TypeError): for attr in subtup[:-1]: try: subdict = subdict[attr].__dict__ except (KeyError, TypeError): # limited support for class lookup if ut.is_method(root_func) and spec.args[0] == attr: subdict = root_func.im_class.__dict__ else: # FIXME TODO lookup_attribute_chain subdict = hack_lookup_mod_attrs(attr) if subdict is None: print('Unable to find attribute of attr=%r' % (attr,)) if ut.SUPER_STRICT: raise if subdict is not None: attr_name = subtup[-1] subfunc = subdict[attr_name] return subfunc def check_subfunc_name(subfunc_name): if isinstance(subfunc_name, tuple) or '.' in subfunc_name: subfunc = resolve_attr_subfunc(subfunc_name) else: # try to directly take func from globals func_globals = meta_util_six.get_funcglobals(root_func) try: subfunc = func_globals[subfunc_name] except KeyError: print('Unable to find function definition subfunc_name=%r' % (subfunc_name,)) if ut.SUPER_STRICT: raise subfunc = None if subfunc is not None: subkw_list = recursive_parse_kwargs(subfunc) have_keys = set(ut.get_list_column(kwargs_list, 0)) new_subkw = [item for item in subkw_list if item[0] not in have_keys] else: new_subkw = [] return new_subkw if spec.keywords is not None: if VERBOSE_INSPECT: print('[inspect] Checking spec.keywords=%r' % (spec.keywords,)) subfunc_name_list = ut.find_child_kwarg_funcs(sourcecode, spec.keywords) if VERBOSE_INSPECT: print('[inspect] Checking subfunc_name_list=%r' % (subfunc_name_list,)) for subfunc_name in subfunc_name_list: new_subkw = check_subfunc_name(subfunc_name) kwargs_list.extend(new_subkw) return kwargs_list
[docs]def parse_func_kwarg_keys(func, with_vals=False): """ hacky inference of kwargs keys """ sourcecode = get_func_sourcecode(func, strip_docstr=True, strip_comments=True) kwkeys = parse_kwarg_keys(sourcecode, with_vals=with_vals) return kwkeys
[docs]def get_func_kwargs(func, stripdef=False, stripret=False, strip_docstr=False, remove_linenums=None): """ func = ibeis.run_experiment """ import utool as ut argspec = ut.get_func_argspec(func) header_kw = dict(zip(argspec.args[::-1], argspec.defaults[::-1])) # TODO if argspec.keywords is not None: # parse our keywords from func body if possible # possibly recursively pass return header_kw
[docs]def parse_kwarg_keys(source, keywords='kwargs', with_vals=False): r""" very hacky way to infer some of the kwarg keys TODO: use a code parse tree here. Use hints. Find other docstrings of functions that are called with kwargs. Find the name of the kwargs variable. Args: source (str): Returns: list: kwarg_keys CommandLine: python -m utool.util_inspect --exec-parse_kwarg_keys Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> source = "\n kwargs.get('foo', None)\n kwargs.pop('bar', 3)\n kwargs.pop('str', '3fd')\n kwargs.pop('str', '3f\'d')\n \"kwargs.get('baz', None)\"" >>> print(source) >>> kwarg_keys = parse_kwarg_keys(source, with_vals=True) >>> result = ('kwarg_keys = %s' % (str(kwarg_keys),)) >>> print(result) """ #source = ut.get_func_sourcecode(func, strip_docstr=True, strip_comments=True) import re import utool as ut keyname = ut.named_field('keyname', ut.REGEX_VARNAME) esc = re.escape #default = ut.named_field('default', '[\'\"A-Za-z_][A-Za-z0-9_\'\"]*') itemgetter = ut.regex_or(['get', 'pop']) pattern = esc(keywords + '.') + itemgetter + esc("('") + keyname + esc("',") if with_vals: WS = ut.REGEX_WHITESPACE valname = WS + ut.named_field('valname', ut.REGEX_RVAL) + WS + esc(')') pattern += valname #not_quotes = '^' + ut.positive_lookbehind(r'[^\'\"]*') #not_quotes = ut.regex_or(['^', r'\n']) + r'[^\'\"]*' #not_quotes = r'[^\'\"]*' not_quotes = r'^[^\'\"]*' pattern = not_quotes + pattern regex = re.compile(pattern, flags=re.MULTILINE) #print('pattern = %s' % (pattern,)) #print(pattern) groupdict_list = [match.groupdict() for match in regex.finditer(source)] kwarg_keys = [groupdict_['keyname'] for groupdict_ in groupdict_list] if with_vals: kwarg_vals = [ut.smart_cast2(groupdict_['valname']) for groupdict_ in groupdict_list] return list(zip(kwarg_keys, kwarg_vals)) else: return kwarg_keys
[docs]class KWReg(object): """ Helper to register keywords for complex keyword parsers """ def __init__(kwreg, enabled=False): kwreg.keys = [] kwreg.defaults = [] kwreg.enabled = enabled def __call__(kwreg, key, default): if kwreg.enabled: kwreg.keys.append(key) kwreg.defaults.append(default) return key, default @property def defaultkw(kwreg): return dict(zip(kwreg.keys, kwreg.defaults))
[docs] def print_defaultkw(kwreg): print(ut.dict_str(kwreg.defaultkw))
[docs]def infer_function_info(func): r""" Infers information for make_default_docstr # TODO: Interleave old documentation with new documentation Args: func (function): live python function CommandLine: python -m utool --tf infer_function_info:0 python -m utool --tf infer_function_info:1 --funcname=ibeis_cnn.models.siam.ignore_hardest_cases Ignore: import ibeis func = ibeis.control.IBEISControl.IBEISController.query_chips Example0: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> func = ut.infer_function_info >>> #func = ut.Timer.tic >>> func = ut.make_default_docstr >>> funcinfo = infer_function_info(func) >>> result = ut.dict_str(funcinfo.__dict__) >>> print(result) Example1: >>> # SCRIPT >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> funcname = ut.get_argval('--funcname') >>> # Parse out custom function >>> modname = '.'.join(funcname.split('.')[0:-1]) >>> script = 'import {modname}\nfunc = {funcname}'.format( >>> modname=modname, funcname=funcname) >>> globals_, locals_ = {}, {} >>> exec(script, globals_, locals_) >>> func = locals_['func'] >>> funcinfo = infer_function_info(func) >>> result = ut.dict_str(funcinfo.__dict__) >>> print(result) """ import utool as ut import re if isinstance(func, property): func = func.fget try: doc_shortdesc = '' doc_longdesc = '' known_arginfo = ut.ddict(dict) if True: current_doc = inspect.getdoc(func) docstr_blocks = ut.parse_docblocks_from_docstr(current_doc) docblock_types = ut.get_list_column(docstr_blocks, 0) docblock_types = [re.sub('Example[0-9]', 'Example', type_) for type_ in docblock_types] docblock_dict = ut.group_items(docstr_blocks, docblock_types) if '' in docblock_dict: docheaders = docblock_dict[''] docheaders_lines = ut.get_list_column(docheaders, 1) docheaders_order = ut.get_list_column(docheaders, 2) docheaders_lines = ut.sortedby(docheaders_lines, docheaders_order) doc_shortdesc = '\n'.join(docheaders_lines) if 'Args' in docblock_dict: argblocks = docblock_dict['Args'] if len(argblocks) != 1: print('Warning: should only be one args block') else: argblock = argblocks[0][1] assert argblock.startswith('Args:\n') argsblock_ = argblock[len('Args:\n'):] arglines = re.split(r'^ \b', argsblock_, flags=re.MULTILINE) arglines = [line for line in arglines if len(line) > 0] esc = re.escape def escparen(pat): return esc('(') + pat + esc(')') argname = ut.named_field('argname', ut.REGEX_VARNAME) argtype_ = ut.named_field('argtype', '.' + ut.REGEX_NONGREEDY) argtype = escparen(argtype_) argdesc = ut.named_field('argdesc', '.*') WS = ut.REGEX_WHITESPACE argpattern = ( WS + argname + WS + argtype + WS + ':' + WS + argdesc) for argline in arglines: m = re.match(argpattern, argline, flags=re.MULTILINE | re.DOTALL) try: groupdict_ = m.groupdict() except Exception as ex: print('---') print('argline = \n%s' % (argline,)) print('---') raise Exception('Unable to parse argline=%s' % (argline,)) #print('groupdict_ = %s' % (ut.dict_str(groupdict_),)) argname = groupdict_['argname'] known_arginfo[argname]['argdesc'] = groupdict_['argdesc'].rstrip('\n') # TODO: record these in a file for future reference # and potential guessing if groupdict_['argtype'] != '?': known_arginfo[argname]['argtype'] = groupdict_['argtype'] is_class = isinstance(func, six.class_types) needs_surround = current_doc is None or len(current_doc) == 0 if is_class: argfunc = func.__init__ else: argfunc = func argspec = ut.get_func_argspec(argfunc) (argname_list, varargs, varkw, defaults) = argspec # See util_inspect tup = ut.infer_arg_types_and_descriptions(argname_list, defaults) argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup # Put in user parsed info for index, argname in enumerate(argname_list): if argname in known_arginfo: arginfo = known_arginfo[argname] if 'argdesc' in arginfo: argdesc_list[index] = arginfo['argdesc'] if 'argtype' in arginfo: argtype_list[index] = arginfo['argtype'] if not is_class: # Move source down to base indentation, but remember original indentation sourcecode = get_func_sourcecode(func) #kwarg_keys = ut.parse_kwarg_keys(sourcecode) kwarg_items = ut.recursive_parse_kwargs(func) kwarg_keys = ut.get_list_column(kwarg_items, 0) #kwarg_keys = ut.unique_ordered(kwarg_keys) kwarg_keys = ut.setdiff_ordered(kwarg_keys, argname_list) else: sourcecode = None kwarg_keys = [] if sourcecode is not None: num_indent = ut.get_indentation(sourcecode) sourcecode = ut.unindent(sourcecode) returninfo = ut.parse_return_type(sourcecode) else: num_indent = 0 returninfo = None, None, None, '' return_type, return_name, return_header, return_desc = returninfo modname = func.__module__ funcname = ut.get_funcname(func) except Exception as ex: #print('dealing with infer function error') #print('has utinfo? ' + str(hasattr(func, '_utinfo'))) #sourcefile = inspect.getsourcefile(func) # NOQA ut.printex(ex, 'Error Infering Function Info', keys=[ 'func', 'sourcefile', 'sourcecode', 'argspec', ]) raise class FunctionInfo(object): def __init__(self): pass funcinfo = FunctionInfo() funcinfo.needs_surround = needs_surround funcinfo.argname_list = argname_list funcinfo.argtype_list = argtype_list funcinfo.argdesc_list = argdesc_list funcinfo.argdefault_list = argdefault_list funcinfo.hasdefault_list = hasdefault_list funcinfo.kwarg_keys = kwarg_keys funcinfo.varargs = varargs funcinfo.varkw = varkw funcinfo.defaults = defaults funcinfo.num_indent = num_indent funcinfo.return_type = return_type funcinfo.return_name = return_name funcinfo.return_header = return_header funcinfo.return_desc = return_desc funcinfo.modname = modname funcinfo.funcname = funcname funcinfo.doc_shortdesc = doc_shortdesc funcinfo.doc_longdesc = doc_longdesc funcinfo.ismethod = hasattr(func, 'im_class') return funcinfo
[docs]def parse_callname(searchline, sentinal='def '): """ Parses the function or class name from a signature line originally part of the vim plugin """ rparen_pos = searchline.find('(') if rparen_pos > 0: callname = searchline[len(sentinal):rparen_pos].strip(' ') return callname return None
[docs]def find_pattern_above_row(pattern, line_list, row, maxIter=50): """ originally part of the vim plugin """ import re # Janky way to find function name ix = 0 while True: pos = row - ix if maxIter is not None and ix > maxIter: break if pos < 0: break raise AssertionError('end of buffer') searchline = line_list[pos] if re.match(pattern, searchline) is not None: return searchline, pos ix += 1
[docs]def find_pyclass_above_row(line_list, row): """ originally part of the vim plugin """ # Get text posision pattern = '^class [a-zA-Z_]' classline, classpos = find_pattern_above_row(pattern, line_list, row, maxIter=None) return classline, classpos
[docs]def find_pyfunc_above_row(line_list, row, orclass=False): """ originally part of the vim plugin CommandLine: python -m utool.util_inspect --test-find_pyfunc_above_row Example: >>> # ENABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> func = find_pyfunc_above_row >>> fpath = meta_util_six.get_funcglobals(func)['__file__'].replace('.pyc', '.py') >>> line_list = ut.read_from(fpath, aslines=True) >>> row = meta_util_six.get_funccode(func).co_firstlineno + 1 >>> pyfunc, searchline = find_pyfunc_above_row(line_list, row) >>> result = pyfunc >>> print(result) find_pyfunc_above_row Example: >>> # DISABLE_DOCTEST >>> from utool.util_inspect import * # NOQA >>> import utool as ut >>> fpath = ut.util_inspect.__file__.replace('.pyc', '.py') >>> line_list = ut.read_from(fpath, aslines=True) >>> row = 1608 >>> pyfunc, searchline = find_pyfunc_above_row(line_list, row, orclass=True) >>> result = pyfunc >>> print(result) find_pyfunc_above_row """ searchlines = [] # for debugging funcname = None # Janky way to find function name func_sentinal = 'def ' method_sentinal = ' def ' class_sentinal = 'class ' for ix in range(200): func_pos = row - ix searchline = line_list[func_pos] cleanline = searchline.strip(' ') searchlines.append(cleanline) if searchline.startswith(func_sentinal): # and cleanline.endswith(':'): # Found a valid function name funcname = parse_callname(searchline, func_sentinal) if funcname is not None: break if orclass and searchline.startswith(class_sentinal): # Found a valid class name (as funcname) funcname = parse_callname(searchline, class_sentinal) if funcname is not None: break if searchline.startswith(method_sentinal): # and cleanline.endswith(':'): # Found a valid function name funcname = parse_callname(searchline, method_sentinal) if funcname is not None: classline, classpos = find_pyclass_above_row(line_list, func_pos) classname = parse_callname(classline, class_sentinal) if classname is not None: funcname = '.'.join([classname, funcname]) break else: funcname = None return funcname, searchlines
if __name__ == '__main__': """ CommandLine: python -c "import utool, utool.util_inspect; utool.doctest_funcs(utool.util_inspect, allexamples=True)" python -c "import utool, utool.util_inspect; utool.doctest_funcs(utool.util_inspect)" python -m utool.util_inspect --enableall python -m utool.util_inspect --enableall --test-iter-module-doctestable:1 python -m utool.util_inspect --allexamples """ import multiprocessing multiprocessing.freeze_support() # for win32 import utool as ut # NOQA ut.doctest_funcs()