Source code for utool._internal.util_importer

# -*- coding: utf-8 -*-
"""
NEEDS CLEANUP SO IT EITHER DOES THE IMPORTS OR GENERATES THE FILE

python -c "import utool"

"""
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
import multiprocessing
import textwrap
#import types

#DEBUG_IMPORTS = '--debug-imports' in sys.argv
#----------
# EXECUTORS
#----------

def __excecute_imports(module, modname, IMPORTS, verbose=False):
    """ Module Imports """
    # level: -1 is a the Python2 import strategy
    # level:  0 is a the Python3 absolute import
    if verbose:
        print('[UTIL_IMPORT] EXECUTING %d IMPORT TUPLES' % (len(IMPORTS),))
    level = 0
    for name in IMPORTS:
        if level == -1:
            tmp = __import__(name, globals(), locals(), fromlist=[], level=level)
        elif level == 0:
            # FIXME: should support unicode. Maybe just a python2 thing
            tmp = __import__(modname, globals(), locals(), fromlist=[str(name)], level=level)


def __execute_fromimport(module, modname, IMPORT_TUPLES, verbose=False):
    """ Module From Imports """
    if verbose:
        print('[UTIL_IMPORT] EXECUTING %d FROM IMPORT TUPLES' % (len(IMPORT_TUPLES),))
    FROM_IMPORTS = __get_from_imports(IMPORT_TUPLES)
    for name, fromlist in FROM_IMPORTS:
        full_modname = '.'.join((modname, name))
        tmp = __import__(full_modname, globals(), locals(), fromlist=fromlist, level=0)
        for attrname in fromlist:
            setattr(module, attrname, getattr(tmp, attrname))
    return FROM_IMPORTS


def __execute_fromimport_star(module, modname, IMPORT_TUPLES, ignore_list=[],
                              ignore_startswith=[], ignore_endswith=[],
                              verbose=False, veryverbose=False):
    """ Effectively import * statements

    The dynamic_import must happen before any * imports otherwise it wont catch
    anything.
    """
    if verbose:
        print('[UTIL_IMPORT] EXECUTE %d FROMIMPORT STAR TUPLES.' % (len(IMPORT_TUPLES),))
    FROM_IMPORTS = []
    # Explicitly ignore these special functions (usually stdlib functions)
    ignoreset = set(['print', 'print_', 'printDBG', 'rrr', 'profile',
                     'print_function', 'absoulte_import', 'division', 'zip',
                     'map', 'range', 'list', 'zip_longest', 'filter', 'filterfalse',
                     'dirname', 'realpath', 'join', 'exists', 'normpath',
                     'splitext', 'expanduser', 'relpath', 'isabs',
                     'commonprefix', 'basename', 'input', 'reduce',
                     #'OrderedDict',
                     #'product',
                     ] + ignore_list)
                     #'isdir', 'isfile', '
    for name, fromlist in IMPORT_TUPLES:
        #absname = modname + '.' + name
        child_module = sys.modules[modname + '.' + name]
        varset = set(vars(module))
        fromset = set(fromlist) if fromlist is not None else set()
        def valid_attrname(attrname):
            """
            Guess if the attrname is valid based on its name
            """
            is_forced  = attrname in fromset
            is_private = attrname.startswith('_')
            is_conflit = attrname in varset
            is_module  = attrname in sys.modules  # Isn't fool proof (next step is)
            is_ignore1 = attrname in ignoreset
            is_ignore2 = any([attrname.startswith(prefix) for prefix in ignore_startswith])
            is_ignore3 = any([attrname.endswith(suffix) for suffix in ignore_endswith])
            is_ignore  = any((is_ignore1, is_ignore2, is_ignore3))
            is_valid = not any((is_ignore, is_private, is_conflit, is_module))
            return (is_forced or is_valid)
        allattrs = dir(child_module)
        fromlist_ = [attrname for attrname in allattrs if valid_attrname(attrname)]
        #if verbose:
        #    print('[UTIL_IMPORT]     name=%r, len(allattrs)=%d' % (name, len(allattrs)))
        #if verbose:
        #    print('[UTIL_IMPORT]     name=%r, len(fromlist_)=%d' % (name, len(fromlist_)))
        valid_fromlist_ = []
        for attrname in fromlist_:
            attrval = getattr(child_module, attrname)
            try:
                # Disallow fromimport modules
                forced = attrname in fromset
                if not forced and getattr(attrval, '__name__') in sys.modules:
                    if veryverbose:
                        print('[UTIL_IMPORT] not importing: %r' % attrname)
                    continue
            except AttributeError:
                pass
            if veryverbose:
                print('[UTIL_IMPORT] importing: %r' % attrname)
            valid_fromlist_.append(attrname)
            setattr(module, attrname, attrval)
        if verbose:
            print('[UTIL_IMPORT]     name=%r, len(valid_fromlist_)=%d' % (name, len(valid_fromlist_)))
        FROM_IMPORTS.append((name, valid_fromlist_))
    return FROM_IMPORTS

#----------
# PARSERS
#----------

def __get_imports(IMPORT_TUPLES):
    """ Returns import names
    IMPORT_TUPLES are specified as
    (name, fromlist, ispackage)
    """
    IMPORTS = [tup[0] for tup in IMPORT_TUPLES]
    return IMPORTS


def __get_from_imports(IMPORT_TUPLES):
    """ Returns import names and fromlist
    IMPORT_TUPLES are specified as
    (name, fromlist, ispackage)
    """
    FROM_IMPORTS = [(tup[0], tup[1]) for tup in IMPORT_TUPLES
                    if tup[1] is not None and len(tup[1]) > 0]
    return FROM_IMPORTS

#----------
# STRING MAKERS
#----------

def _initstr(modname, IMPORTS, FROM_IMPORTS, inject_execstr, withheader=True):
    """ Calls the other string makers """
    header         = _make_module_header() if withheader else ''
    import_str     = _make_imports_str(IMPORTS, modname)
    fromimport_str = _make_fromimport_str(FROM_IMPORTS, modname)
    initstr = '\n'.join([str_ for str_ in [
        header,
        import_str,
        fromimport_str,
        inject_execstr,
    ] if len(str_) > 0])
    return initstr

def _make_module_header():
    return '\n'.join([
        '# flake8: noqa',
        'from __future__ import absolute_import, division, print_function, unicode_literals'])

def _make_imports_str(IMPORTS, rootmodname='.'):
    imports_fmtstr = 'from {rootmodname} import %s'.format(rootmodname=rootmodname)
    return '\n'.join([imports_fmtstr % (name,) for name in IMPORTS])

def _make_fromimport_str(FROM_IMPORTS, rootmodname='.'):
    from utool import util_str
    if rootmodname == '.':
        # dot is already taken care of in fmtstr
        rootmodname = ''
    def _pack_fromimport(tup):
        name, fromlist = tup[0], tup[1]
        from_module_str = 'from {rootmodname}.{name} import ('.format(rootmodname=rootmodname, name=name)
        newline_prefix = (' ' * len(from_module_str))
        if len(fromlist) > 0:
            rawstr = from_module_str + ', '.join(fromlist) + ',)'
        else:
            rawstr = ''

        # not sure why this isn't 76? >= maybe?
        packstr = util_str.pack_into(rawstr, textwidth=75,
                                     newline_prefix=newline_prefix,
                                    break_words=False)
        return packstr
    from_str = '\n'.join(map(_pack_fromimport, FROM_IMPORTS))
    return from_str

def _inject_execstr(modname, IMPORT_TUPLES):
    """ Injection and Reload String Defs """
    if modname == 'utool':
        # Special case import of the util_inject module
        injecter = 'util_inject'
        injecter_import = ''
    else:
        # Normal case implicit import of util_inject
        injecter_import = 'import utool'
        injecter = 'utool'
    injectstr_fmt = textwrap.dedent(
        r'''
        # STARTBLOCK
        {injecter_import}
        print, rrr, profile = {injecter}.inject2(__name__, '[{modname}]')


        def reassign_submodule_attributes(verbose=True):
            """
            why reloading all the modules doesnt do this I don't know
            """
            import sys
            if verbose and '--quiet' not in sys.argv:
                print('dev reimport')
            # Self import
            import {modname}
            # Implicit reassignment.
            seen_ = set([])
            for tup in IMPORT_TUPLES:
                if len(tup) > 2 and tup[2]:
                    continue  # dont import package names
                submodname, fromimports = tup[0:2]
                submod = getattr({modname}, submodname)
                for attr in dir(submod):
                    if attr.startswith('_'):
                        continue
                    if attr in seen_:
                        # This just holds off bad behavior
                        # but it does mimic normal util_import behavior
                        # which is good
                        continue
                    seen_.add(attr)
                    setattr({modname}, attr, getattr(submod, attr))


        def reload_subs(verbose=True):
            """ Reloads {modname} and submodules """
            if verbose:
                print('Reloading submodules')
            rrr(verbose=verbose)
            def wrap_fbrrr(mod):
                def fbrrr(*args, **kwargs):
                    """ fallback reload """
                    if verbose:
                        print('No fallback relaod for mod=%r' % (mod,))
                    # Breaks ut.Pref (which should be depricated anyway)
                    # import imp
                    # imp.reload(mod)
                return fbrrr
            def get_rrr(mod):
                if hasattr(mod, 'rrr'):
                    return mod.rrr
                else:
                    return wrap_fbrrr(mod)
            def get_reload_subs(mod):
                return getattr(mod, 'reload_subs', wrap_fbrrr(mod))
            {reload_body}
            rrr(verbose=verbose)
            try:
                # hackish way of propogating up the new reloaded submodule attributes
                reassign_submodule_attributes(verbose=verbose)
            except Exception as ex:
                print(ex)
        rrrr = reload_subs
        # ENDBLOCK
        ''')
    injectstr_fmt = injectstr_fmt.replace('# STARTBLOCK', '')
    injectstr_fmt = injectstr_fmt.replace('# ENDBLOCK', '')
    rrrdir_fmt  = '    get_reload_subs({modname})(verbose=verbose)'
    rrrfile_fmt = '    get_rrr({modname})(verbose=verbose)'

    def _reload_command(tup):
        if len(tup) > 2 and tup[2] is True:
            return rrrdir_fmt.format(modname=tup[0])
        else:
            return rrrfile_fmt.format(modname=tup[0])
    reload_body = '\n'.join(map(_reload_command, IMPORT_TUPLES)).strip()
    format_dict = {
        'modname': modname,
        'reload_body': reload_body,
        'injecter': injecter,
        'injecter_import': injecter_import,
    }
    inject_execstr = injectstr_fmt.format(**format_dict).strip()
    return inject_execstr

#----------
# PUBLIC FUNCTIONS
#----------

[docs]def dynamic_import(modname, IMPORT_TUPLES, developing=True, ignore_froms=[], dump=False, ignore_startswith=[], ignore_endswith=[], ignore_list=[], verbose=False): """ MAIN ENTRY POINT Dynamically import listed util libraries and their attributes. Create reload_subs function. Using __import__ like this is typically not considered good style However, it is better than import * and this will generate the good file text that can be used when the module is 'frozen" """ if verbose: print('[UTIL_IMPORT] Running Dynamic Imports for modname=%r ' % modname) # Get the module that will be imported into module = sys.modules[modname] # List of modules to be imported IMPORTS = __get_imports(IMPORT_TUPLES) # Import the modules __excecute_imports(module, modname, IMPORTS, verbose=verbose) # If developing do explicit import stars if developing: FROM_IMPORTS = __execute_fromimport_star(module, modname, IMPORT_TUPLES, ignore_list=ignore_list, ignore_startswith=ignore_startswith, ignore_endswith=ignore_endswith, verbose=verbose) else: FROM_IMPORTS = __execute_fromimport(module, modname, IMPORT_TUPLES, verbose=verbose) inject_execstr = _inject_execstr(modname, IMPORT_TUPLES) # If requested: print what the __init__ module should look like dump_requested = (('--dump-%s-init' % modname) in sys.argv or ('--print-%s-init' % modname) in sys.argv) or dump overwrite_requested = ('--update-%s-init' % modname) in sys.argv if verbose: print('[UTIL_IMPORT] Finished Dynamic Imports for modname=%r ' % modname) if dump_requested: is_main_proc = multiprocessing.current_process().name == 'MainProcess' if is_main_proc: from utool import util_str initstr = _initstr(modname, IMPORTS, FROM_IMPORTS, inject_execstr) print(util_str.indent(initstr)) # Overwrite the __init__.py file with new explicit imports if overwrite_requested: """ SeeAlso: util_inject.inject_python_code util_str.replace_between_tags """ is_main_proc = multiprocessing.current_process().name == 'MainProcess' if is_main_proc: from utool import util_str from os.path import join, exists initstr = _initstr(modname, IMPORTS, FROM_IMPORTS, inject_execstr, withheader=False) new_else = util_str.indent(initstr) #print(new_else) # Get path to init file so we can overwrite it init_fpath = join(module.__path__[0], '__init__.py') print('attempting to update: %r' % init_fpath) assert exists(init_fpath) new_lines = [] editing = False updated = False #start_tag = '# <AUTOGEN_INIT>' #end_tag = '# </AUTOGEN_INIT>' with open(init_fpath, 'r') as file_: #text = file_.read() lines = file_.readlines() for line in lines: if not editing: new_lines.append(line) if line.strip().startswith('# <AUTOGEN_INIT>'): new_lines.append('\n' + new_else + '\n # </AUTOGEN_INIT>\n') editing = True updated = True if line.strip().startswith('# </AUTOGEN_INIT>'): editing = False # TODO: #new_text = util_str.replace_between_tags(text, new_else, start_tag, end_tag) if updated: print('writing updated file: %r' % init_fpath) new_text = ''.join(new_lines) with open(init_fpath, 'w') as file_: file_.write(new_text) else: print('no write hook for file: %r' % init_fpath) return inject_execstr
[docs]def make_initstr(modname, IMPORT_TUPLES, verbose=False): """ Just creates the string representation. Does no importing. """ IMPORTS = __get_imports(IMPORT_TUPLES) FROM_IMPORTS = __get_from_imports(IMPORT_TUPLES) inject_execstr = _inject_execstr(modname, IMPORT_TUPLES) return _initstr(modname, IMPORTS, FROM_IMPORTS, inject_execstr)
[docs]def make_import_tuples(module_path, exclude_modnames=[]): """ Infer the IMPORT_TUPLES from a module_path """ from utool import util_path kwargs = dict(private=False, full=False) module_list = util_path.ls_modulefiles(module_path, noext=True, **kwargs) package_list = util_path.ls_moduledirs(module_path, **kwargs) exclude_set = set(exclude_modnames) module_import_tuples = [(modname, None) for modname in module_list if modname not in exclude_set] package_import_tuples = [(modname, None, True) for modname in package_list if modname not in exclude_set] IMPORT_TUPLES = (module_import_tuples + package_import_tuples) return IMPORT_TUPLES