Source code for utool.util_project

# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function  # , unicode_literals
from six.moves import zip, filter, filterfalse, map, range  # NOQA
import six  # NOQA
#from os.path import split, dirname, join
from os.path import dirname, join
from utool import util_class  # NOQA
from utool import util_dev
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_project]')


__GLOBAL_PROFILE__ = None


[docs]def ensure_text(fname, text, repo_dpath='.', force=False, locals_={}, chmod=None): """ Args: fname (str): file name text (str): repo_dpath (str): directory path string(default = '.') force (bool): (default = False) locals_ (dict): (default = {}) Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> result = setup_repo() >>> print(result) """ print('Ensuring fname=%r' % (fname)) import utool as ut fpath = join(repo_dpath, fname) if force or not ut.checkpath(fpath, verbose=2, n=5): text_ = ut.remove_codeblock_syntax_sentinals(text) text_ = text_.format(**locals_) + '\n' ut.writeto(fpath, text_) try: if chmod: ut.chmod(fpath, chmod) except Exception as ex: ut.printex(ex, iswarning=True)
[docs]def setup_repo(): r""" Creates default structure for a new repo CommandLine: python -m utool --tf setup_repo --repo=dtool --codedir=~/code python -m utool --tf setup_repo --repo=dtool --codedir=~/code python -m utool --tf setup_repo --repo=ibeis-flukematch-module --codedir=~/code --modname=ibeis_flukematch python -m utool --tf setup_repo --repo=mtgmonte --codedir=~/code --modname=mtgmonte python -m utool --tf setup_repo --repo=pydarknet --codedir=~/code --modname=pydarknet python -m utool --tf setup_repo Python: ipython import utool as ut ut.rrrr(0); ut.setup_repo() Example: >>> # SCRIPT >>> from utool.util_git import * # NOQA >>> import utool as ut >>> result = setup_repo() >>> print(result) """ print('\n [setup_repo]!') # import os from functools import partial import utool as ut # import os code_dpath = ut.truepath(ut.get_argval('--code-dir', default='~/code')) _code_dpath = ut.unexpanduser(code_dpath) repo_fname = (ut.get_argval(('--repo', '--repo-name'), type_=str)) repo_dpath = join(code_dpath, repo_fname) modname = ut.get_argval('--modname', default=repo_fname) ut.ensuredir(repo_dpath, verbose=True) _regencmd = 'python -m utool --tf setup_repo --repo={repo_fname} --codedir={_code_dpath} --modname={modname}' regencmd = _regencmd.format(**locals()) with ut.ChdirContext(repo_dpath): # os.chdir(repo_fname) locals_ = locals() force = True _ensure_text = partial(ensure_text, repo_dpath='.', force=False, locals_=locals_) _ensure_text( fname='todo.md', text=ut.codeblock( r''' # STARTBLOCK # {modname} TODO File * Add TODOS! # ENDBLOCK ''') ) _ensure_text( fname='README.md', text=ut.codeblock( r''' # STARTBLOCK # {modname} README FILE # ENDBLOCK ''') ) _ensure_text( fname='setup.py', chmod='+x', text=ut.codeblock( r''' # STARTBLOCK #!/usr/bin/env python2.7 """ Initially Generated By: {regencmd} """ from __future__ import absolute_import, division, print_function, unicode_literals from setuptools import setup try: from utool import util_setup except ImportError: print('ERROR: setup requires utool') raise INSTALL_REQUIRES = [ #'cython >= 0.21.1', #'numpy >= 1.9.0', #'scipy >= 0.16.0', ] CLUTTER_PATTERNS = [ # Patterns removed by python setup.py clean ] if __name__ == '__main__': kwargs = util_setup.setuptools_setup( setup_fpath=__file__, name='{modname}', packages=util_setup.find_packages(), version=util_setup.parse_package_for_version('{modname}'), license=util_setup.read_license('LICENSE'), long_description=util_setup.parse_readme('README.md'), ext_modules=util_setup.find_ext_modules(), cmdclass=util_setup.get_cmdclass(), #description='description of module', #url='https://github.com/<username>/{repo_fname}.git', #author='<author>', #author_email='<author_email>', keywords='', install_requires=INSTALL_REQUIRES, clutter_patterns=CLUTTER_PATTERNS, #package_data={{'build': ut.get_dynamic_lib_globstrs()}}, #build_command=lambda: ut.std_build_command(dirname(__file__)), classifiers=[], ) setup(**kwargs) # ENDBLOCK ''' ) ) _ensure_text( fname='.gitignore', text=ut.codeblock( r''' # STARTBLOCK *.py[cod] # C extensions *.so # Packages *.egg *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg lib lib64 __pycache__ # Installer logs pip-log.txt # Print Logs logs # Unit test / coverage reports .coverage .tox nosetests.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject .DS_Store *.dump.txt *.sqlite3 # profiler *.lprof *.prof *.flann *.npz # utool output _test_times.txt failed.txt *.orig _doc test_times.txt failed_doctests.txt # ENDBLOCK ''' ) ) _ensure_text( fname=join(repo_dpath, modname, '__init__.py'), text=ut.codeblock( r''' """ Initially Generated By: {regencmd} """ __version__ = '0.0.0' ''' ) ) _ensure_text( fname=join(repo_dpath, modname, '__main__.py'), chmod='+x', text=ut.codeblock( r''' # STARTBLOCK #!/usr/bin/env python2.7 # -*- coding: utf-8 -*- """ Initially Generated By: {regencmd} """ from __future__ import absolute_import, division, print_function, unicode_literals def {modname}_main(): ignore_prefix = [] ignore_suffix = [] import utool as ut # allows for --tf ut.main_function_tester('{modname}', ignore_prefix, ignore_suffix) if __name__ == '__main__': """ Usage: python -m {modname} --tf <funcname> """ print('Running {modname} main') {modname}_main() # ENDBLOCK ''' ) ) _ensure_text( fname='run_tests.py', chmod='+x', text=ut.codeblock( r''' # STARTBLOCK #!/usr/bin/env python2.7 """ Initially Generated By: {regencmd} """ from __future__ import absolute_import, division, print_function import sys import utool as ut def run_tests(): # Build module list and run tests import sys ut.change_term_title('RUN {modname} TESTS') exclude_doctests_fnames = set([ ]) exclude_dirs = [ '_broken', 'old', 'tests', 'timeits', '_scripts', '_timeits', '_doc', 'notebook', ] dpath_list = ['{modname}'] doctest_modname_list = ut.find_doctestable_modnames( dpath_list, exclude_doctests_fnames, exclude_dirs) for modname in doctest_modname_list: exec('import ' + modname, globals()) module_list = [sys.modules[name] for name in doctest_modname_list] nPass, nTotal, failed_cmd_list = ut.doctest_module_list(module_list) if nPass != nTotal: return 1 else: return 0 if __name__ == '__main__': import multiprocessing multiprocessing.freeze_support() retcode = run_tests() sys.exit(retcode) # ENDBLOCK ''' ) ) ut.ensuredir(join(repo_dpath, modname), verbose=True) #@util_class.ReloadingMetaclass
[docs]class UserProfile(object): def __init__(self): self.project_dpaths = None self.project_include_patterns = None self.project_exclude_dirs = None # def __str__(self): # return
[docs]def ibeis_user_profile(): import utool as ut import sys addpath = True module_fpath = ut.truepath('~/local/init/__REPOS1__.py') if addpath: module_dpath = dirname(module_fpath) sys.path.append(module_dpath) __REPOS1__ = ut.import_module_from_fpath(module_fpath) self = UserProfile() #self.project_dpaths = __REPOS1__.PROJECT_REPOS self.project_dpaths = __REPOS1__.IBEIS_REPOS self.project_dpaths += [ut.truepath('~/latex/crall-candidacy-2015/')] self.project_include_patterns = [ '*.py', '*.cxx', '*.cpp', '*.hxx', '*.hpp', '*.c', '*.h', '*.vim' #'*.py', # '*.cxx', '*.cpp', '*.hxx', '*.hpp', '*.c', '*.h', '*.vim' ] self.project_exclude_dirs = [ '_graveyard', '_broken', 'CompilerIdCXX', 'CompilerIdC', 'build', 'old', ] return self
[docs]def ensure_user_profile(user_profile=None): r""" Args: user_profile (UserProfile): (default = None) Returns: UserProfile: user_profile CommandLine: python -m utool.util_project --exec-ensure_user_profile --show Example: >>> # DISABLE_DOCTEST >>> from utool.util_project import * # NOQA >>> import utool as ut >>> user_profile = None >>> user_profile = ensure_user_profile(user_profile) >>> result = ('user_profile = %s' % (ut.repr2(user_profile),)) >>> print(ut.repr3(user_profile.project_dpaths)) >>> print(result) """ global __GLOBAL_PROFILE__ if __GLOBAL_PROFILE__ is None: import utool as ut if ut.is_developer(): __GLOBAL_PROFILE__ = ibeis_user_profile() if user_profile is None: user_profile = __GLOBAL_PROFILE__ return user_profile
@profile
[docs]def grep_projects(tofind_list, user_profile=None, verbose=True, new=False, **kwargs): r""" Greps the projects defined in the current UserProfile Args: tofind_list (list): user_profile (None): (default = None) Kwargs: user_profile CommandLine: python -m utool --tf grep_projects grep_projects Example: >>> # DISABLE_DOCTEST >>> from utool.util_project import * # NOQA >>> import utool as ut >>> import sys >>> tofind_list = ut.get_argval('--find', type_=list, >>> default=[sys.argv[-1]]) >>> grep_projects(tofind_list) """ import utool as ut user_profile = ensure_user_profile(user_profile) kwargs = kwargs.copy() colored = kwargs.pop('colored', True) grepkw = {} grepkw['exclude_dirs'] = user_profile.project_exclude_dirs grepkw['dpath_list'] = user_profile.project_dpaths grepkw['include_patterns'] = user_profile.project_include_patterns grepkw.update(kwargs) msg_list1 = [] msg_list2 = [] print_ = msg_list1.append print_('Greping Projects') print_('tofind_list = %s' % (ut.list_str(tofind_list, nl=True),)) #print_('grepkw = %s' % ut.dict_str(grepkw, nl=True)) if verbose: print('\n'.join(msg_list1)) #with ut.Timer('greping', verbose=True): grep_result = ut.grep(tofind_list, **grepkw) found_fpath_list, found_lines_list, found_lxs_list = grep_result # HACK, duplicate behavior. TODO: write grep print result function reflags = grepkw.get('reflags', 0) _exprs_flags = [ut.extend_regex2(expr, reflags) for expr in tofind_list] extended_regex_list = ut.take_column(_exprs_flags, 0) reflags_list = ut.take_column(_exprs_flags, 1) # HACK # pat = ut.util_regex.regex_or(extended_regex_list) reflags = reflags_list[0] # from utool import util_regex resultstr = ut.make_grep_resultstr(grep_result, extended_regex_list, reflags, colored=colored) msg_list2.append(resultstr) print_ = msg_list2.append #for fpath, lines, lxs in zip(found_fpath_list, found_lines_list, # found_lxs_list): # print_('----------------------') # print_('found %d line(s) in %r: ' % (len(lines), fpath)) # name = split(fpath)[1] # max_line = len(lines) # ndigits = str(len(str(max_line))) # for (lx, line) in zip(lxs, lines): # line = line.replace('\n', '') # print_(('%s : %' + ndigits + 'd |%s') % (name, lx, line)) # iter_ = zip(found_fpath_list, found_lines_list, found_lxs_list) # for fpath, lines, lxs in iter_: # print_('----------------------') # print_('found %d line(s) in %r: ' % (len(lines), fpath)) # name = split(fpath)[1] # max_line = len(lines) # ndigits = str(len(str(max_line))) # for (lx, line) in zip(lxs, lines): # line = line.replace('\n', '') # colored_line = ut.highlight_regex( # line.rstrip('\n'), pat, reflags=reflags) # print_(('%s : %' + ndigits + 'd |%s') % (name, lx, colored_line)) print_('====================') print_('found_fpath_list = ' + ut.list_str(found_fpath_list)) print_('') #print_('gvim -o ' + ' '.join(found_fpath_list)) if verbose: print('\n'.join(msg_list2)) msg_list = msg_list1 + msg_list2 if new: return GrepResult(found_fpath_list, found_lines_list, found_lxs_list, extended_regex_list, reflags) else: return msg_list
[docs]class GrepResult(util_dev.NiceRepr): def __init__(self, found_fpath_list, found_lines_list, found_lxs_list, extended_regex_list, reflags): self.found_fpath_list = found_fpath_list self.found_lines_list = found_lines_list self.found_lxs_list = found_lxs_list self.extended_regex_list = extended_regex_list self.reflags = reflags def __nice__(self): return '(%d)' % (len(self),) def __len__(self): return len(self.found_fpath_list) def __delitem__(self, index): import utool as ut index = ut.ensure_iterable(index) ut.delete_items_by_index(self.found_fpath_list, index) ut.delete_items_by_index(self.found_lines_list, index) ut.delete_items_by_index(self.found_lxs_list, index)
[docs] def remove_results(self, indicies): del self[indicies]
[docs] def make_resultstr(self, colored=True): import utool as ut tup = (self.found_fpath_list, self.found_lines_list, self.found_lxs_list) return ut.make_grep_resultstr(tup, self.extended_regex_list, self.reflags, colored=colored)
[docs] def hack_remove_pystuff(self): import utool as ut # Hack of a method new_lines = [] for lines in self.found_lines_list: # remove comment results flags = [not line.strip().startswith('# ') for line in lines] lines = ut.compress(lines, flags) # remove doctest results flags = [not line.strip().startswith('>>> ') for line in lines] lines = ut.compress(lines, flags) # remove cmdline tests import re flags = [not re.search('--test-' + self.extended_regex_list[0], line) for line in lines] lines = ut.compress(lines, flags) flags = [not re.search('--exec-' + self.extended_regex_list[0], line) for line in lines] lines = ut.compress(lines, flags) flags = [not re.search('--exec-[a-zA-z]*\.' + self.extended_regex_list[0], line) for line in lines] lines = ut.compress(lines, flags) flags = [not re.search('--test-[a-zA-z]*\.' + self.extended_regex_list[0], line) for line in lines] lines = ut.compress(lines, flags) # remove func defs flags = [not re.search('def ' + self.extended_regex_list[0], line) for line in lines] lines = ut.compress(lines, flags) new_lines += [lines] self.found_lines_list = new_lines # compress self flags = [len(lines_) > 0 for lines_ in self.found_lines_list] idxs = ut.list_where(ut.not_list(flags)) del self[idxs] ## Grep my projects #def gp(r, regexp): # rob_nav._grep(r, [regexp], recursive=True, dpath_list=project_dpaths(), regex=True) ## Sed my projects #def sp(r, regexpr, repl, force=False): # rob_nav._sed(r, regexpr, repl, force=force, recursive=True, dpath_list=project_dpaths())
[docs]def sed_projects(regexpr, repl, force=False, recursive=True, user_profile=None, **kwargs): """ Args: regexpr (?): repl (?): force (bool): (default = False) recursive (bool): (default = True) user_profile (None): (default = None) CommandLine: python -m utool.util_project --exec-sed_projects Example: >>> # DISABLE_DOCTEST >>> from utool.util_project import * # NOQA >>> regexpr = ut.get_argval('--find', type_=str, default=sys.argv[-1]) >>> repl = ut.get_argval('--repl', type_=str, default=sys.argv[-2]) >>> force = False >>> recursive = True >>> user_profile = None >>> result = sed_projects(regexpr, repl, force, recursive, user_profile) >>> print(result) Ignore: regexpr = 'annotation match_scores' repl = 'draw_annot_scoresep' """ # FIXME: finishme import utool as ut user_profile = ensure_user_profile(user_profile) sedkw = {} sedkw['exclude_dirs'] = user_profile.project_exclude_dirs sedkw['dpath_list'] = user_profile.project_dpaths sedkw['include_patterns'] = user_profile.project_include_patterns sedkw.update(kwargs) msg_list1 = [] #msg_list2 = [] print_ = msg_list1.append print_('Seding Projects') print(' * regular expression : %r' % (regexpr,)) print(' * replacement : %r' % (repl,)) print_('sedkw = %s' % ut.dict_str(sedkw, nl=True)) print(' * recursive: %r' % (recursive,)) print(' * force: %r' % (force,)) # Walk through each directory recursively for fpath in ut.matching_fnames(sedkw['dpath_list'], sedkw['include_patterns'], sedkw['exclude_dirs'], recursive=recursive): ut.sedfile(fpath, regexpr, repl, force) #def extend_regex(regexpr): # regex_map = { # r'\<': r'\b(?=\w)', # r'\>': r'\b(?!\w)', # ('UNSAFE', r'\x08'): r'\b', # } # for key, repl in six.iteritems(regex_map): # if isinstance(key, tuple): # search = key[1] # else: # search = key # if regexpr.find(search) != -1: # if isinstance(key, tuple): # print('WARNING! Unsafe regex with: %r' % (key,)) # regexpr = regexpr.replace(search, repl) # return regexpr #regexpr = extend_regex(regexpr) #if '\x08' in regexpr: # print('Remember \\x08 != \\b') # print('subsituting for you for you') # regexpr = regexpr.replace('\x08', '\\b') # print(' * regular expression : %r' % (regexpr,))
if False: def ensure_vim_plugins(): """ python ~/local/init/ensure_vim_plugins.py '~/local/init/ensure_vim_plugins.py' '~/local/init/__REPOS1__.py' """ # TODO pass def find_module_callers(): """ TODO: attempt to build a call graph between module functions to make it easy to see what can be removed and what cannot. """ import utool as ut from os.path import normpath mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_analyzer.py') mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_all.py') mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_organizer.py') module = ut.import_module_from_fpath(mod_fpath) user_profile = ut.ensure_user_profile() doctestables = list(ut.iter_module_doctestable(module, include_builtin=False)) grepkw = {} grepkw['exclude_dirs'] = user_profile.project_exclude_dirs grepkw['dpath_list'] = user_profile.project_dpaths grepkw['verbose'] = True usage_map = {} for funcname, func in doctestables: print('Searching for funcname = %r' % (funcname,)) found_fpath_list, found_lines_list, found_lxs_list = ut.grep([funcname], **grepkw) used_in = (found_fpath_list, found_lines_list, found_lxs_list) usage_map[funcname] = used_in external_usage_map = {} for funcname, used_in in usage_map.items(): (found_fpath_list, found_lines_list, found_lxs_list) = used_in isexternal_flag = [normpath(fpath) != normpath(mod_fpath) for fpath in found_fpath_list] ext_used_in = (ut.compress(found_fpath_list, isexternal_flag), ut.compress(found_lines_list, isexternal_flag), ut.compress(found_lxs_list, isexternal_flag)) external_usage_map[funcname] = ext_used_in for funcname, used_in in external_usage_map.items(): (found_fpath_list, found_lines_list, found_lxs_list) = used_in print('Calling modules: \n' + ut.repr2(ut.unique_ordered(ut.flatten([used_in[0] for used_in in external_usage_map.values()])), nl=True)) if __name__ == '__main__': """ CommandLine: python -m utool.util_project python -m utool.util_project --allexamples python -m utool.util_project --allexamples --noface --nosrc """ import multiprocessing multiprocessing.freeze_support() # for win32 import utool as ut # NOQA ut.doctest_funcs()