Source code for utool._internal.util_importer

# -*- coding: utf-8 -*-
"""
NEEDS CLEANUP SO IT EITHER DOES THE IMPORTS OR GENERATES THE FILE

python -c "import utool"

"""
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
import multiprocessing
import textwrap

# import types

# DEBUG_IMPORTS = '--debug-imports' in sys.argv
# ----------
# EXECUTORS
# ----------


def __excecute_imports(module, modname, imports, verbose=False):
    """ Module Imports """
    # level: -1 is a the Python2 import strategy
    # level:  0 is a the Python3 absolute import
    if verbose:
        print('[UTIL_IMPORT] EXECUTING %d IMPORT TUPLES' % (len(imports),))
    level = 0
    for name in imports:
        if level == -1:
            tmp = __import__(name, globals(), locals(), fromlist=[], level=level)
        elif level == 0:
            # FIXME: should support unicode. Maybe just a python2 thing
            tmp = __import__(
                modname, globals(), locals(), fromlist=[str(name)], level=level
            )


def __execute_fromimport(module, modname, import_tuples, verbose=False):
    """ Module From Imports """
    if verbose:
        print('[UTIL_IMPORT] EXECUTING %d FROM IMPORT TUPLES' % (len(import_tuples),))
    from_imports = __get_from_imports(import_tuples)
    for name, fromlist in from_imports:
        full_modname = '.'.join((modname, name))
        tmp = __import__(full_modname, globals(), locals(), fromlist=fromlist, level=0)
        for attrname in fromlist:
            setattr(module, attrname, getattr(tmp, attrname))
    return from_imports


def __execute_fromimport_star(
    module,
    modname,
    import_tuples,
    ignore_list=[],
    ignore_startswith=[],
    ignore_endswith=[],
    check_not_imported=True,
    verbose=False,
    veryverbose=False,
):
    r"""
    Effectively import * statements

    The dynamic_import must happen before any * imports otherwise it wont catch
    anything.

    Ignore:
        ignore_startswith = []
        ignore_endswith = []
        check_not_imported = False
        verbose = True
        veryverbose = True
    """
    if verbose:
        print('[UTIL_IMPORT] EXECUTE %d FROMIMPORT STAR TUPLES.' % (len(import_tuples),))
    from_imports = []
    # Explicitly ignore these special functions (usually stdlib functions)
    ignoreset = set(
        [
            'print',
            'print_',
            'printDBG',
            'rrr',
            'profile',
            'print_function',
            'absolute_import',
            'division',
            'zip',
            'map',
            'range',
            'list',
            'zip_longest',
            'filter',
            'filterfalse',
            'dirname',
            'realpath',
            'join',
            'exists',
            'normpath',
            'splitext',
            'expanduser',
            'relpath',
            'isabs',
            'commonprefix',
            'basename',
            'input',
            'reduce',
            #'OrderedDict',
            #'product',
        ]
        + ignore_list
    )
    #'isdir', 'isfile', '

    # def is_defined_by_module2(item, module):
    #    belongs = False
    #    if hasattr(item, '__module__'):
    #        belongs = item.__module__ == module.__name__
    #    elif hasattr(item, 'func_globals'):
    #        belongs = item.func_globals['__name__'] == module.__name__
    #    return belongs

    for name, fromlist in import_tuples:
        # absname = modname + '.' + name
        child_module = sys.modules[modname + '.' + name]
        # Check if the variable already belongs to the module
        varset = set(vars(module)) if check_not_imported else set()
        fromset = set(fromlist) if fromlist is not None else set()

        def valid_attrname(attrname):
            """
            Guess if the attrname is valid based on its name
            """
            is_forced = attrname in fromset
            is_private = attrname.startswith('_')
            is_conflit = attrname in varset
            is_module = attrname in sys.modules  # Isn't fool proof (next step is)
            is_ignore1 = attrname in ignoreset
            is_ignore2 = any(
                [attrname.startswith(prefix) for prefix in ignore_startswith]
            )
            is_ignore3 = any([attrname.endswith(suffix) for suffix in ignore_endswith])
            is_ignore = any((is_ignore1, is_ignore2, is_ignore3))
            is_valid = not any((is_ignore, is_private, is_conflit, is_module))
            # is_valid = is_valid and is_defined_by_module2(getattr(child_module, attrname), child_module)
            return is_forced or is_valid

        allattrs = dir(child_module)
        fromlist_ = [attrname for attrname in allattrs if valid_attrname(attrname)]
        # if verbose:
        #    print('[UTIL_IMPORT]     name=%r, len(allattrs)=%d' % (name, len(allattrs)))
        # if verbose:
        #    print('[UTIL_IMPORT]     name=%r, len(fromlist_)=%d' % (name, len(fromlist_)))
        valid_fromlist_ = []
        for attrname in fromlist_:
            attrval = getattr(child_module, attrname)
            try:
                # Disallow fromimport modules
                forced = attrname in fromset
                if not forced and getattr(attrval, '__name__') in sys.modules:
                    if veryverbose:
                        print('[UTIL_IMPORT] not importing: %r' % attrname)
                    continue
            except AttributeError:
                pass
            if veryverbose:
                print('[UTIL_IMPORT] %s is importing: %r' % (modname, attrname))
            valid_fromlist_.append(attrname)
            setattr(module, attrname, attrval)
        if verbose:
            print(
                '[UTIL_IMPORT]     name=%r, len(valid_fromlist_)=%d'
                % (name, len(valid_fromlist_))
            )
        from_imports.append((name, valid_fromlist_))
    return from_imports


# ----------
# PARSERS
# ----------


def __get_from_imports(import_tuples):
    """ Returns import names and fromlist
    import_tuples are specified as
    (name, fromlist, ispackage)
    """
    from_imports = [
        (tup[0], tup[1])
        for tup in import_tuples
        if tup[1] is not None and len(tup[1]) > 0
    ]
    return from_imports


# ----------
# STRING MAKERS
# ----------


def _initstr(modname, imports, from_imports, inject_execstr, withheader=True):
    """ Calls the other string makers """
    header = _make_module_header() if withheader else ''
    import_str = _make_imports_str(imports, modname)
    fromimport_str = _make_fromimport_str(from_imports, modname)
    initstr = '\n'.join(
        [
            str_
            for str_ in [header, import_str, fromimport_str, inject_execstr,]
            if len(str_) > 0
        ]
    )
    return initstr


def _make_module_header():
    return '\n'.join(
        [
            '# flake8: noqa',
            'from __future__ import absolute_import, division, print_function, unicode_literals',
        ]
    )


def _make_imports_str(imports, rootmodname='.'):
    imports_fmtstr = 'from {rootmodname} import %s'.format(rootmodname=rootmodname)
    return '\n'.join([imports_fmtstr % (name,) for name in imports])


def _make_fromimport_str(from_imports, rootmodname='.'):
    from utool import util_str

    if rootmodname == '.':
        # dot is already taken care of in fmtstr
        rootmodname = ''

    def _pack_fromimport(tup):
        name, fromlist = tup[0], tup[1]
        from_module_str = 'from {rootmodname}.{name} import ('.format(
            rootmodname=rootmodname, name=name
        )
        newline_prefix = ' ' * len(from_module_str)
        if len(fromlist) > 0:
            rawstr = from_module_str + ', '.join(fromlist) + ',)'
        else:
            rawstr = ''

        # not sure why this isn't 76? >= maybe?
        packstr = util_str.pack_into(
            rawstr, textwidth=75, newline_prefix=newline_prefix, break_words=False
        )
        return packstr

    from_str = '\n'.join(map(_pack_fromimport, from_imports))
    return from_str


def _inject_execstr(modname, import_tuples):
    """ Injection and Reload String Defs """
    if modname == 'utool':
        # Special case import of the util_inject module
        injecter = 'util_inject'
        injecter_import = ''
    else:
        # Normal case implicit import of util_inject
        injecter_import = 'import utool'
        injecter = 'utool'
    injectstr_fmt = textwrap.dedent(
        r'''
        # STARTBLOCK
        {injecter_import}
        print, rrr, profile = {injecter}.inject2(__name__, '[{modname}]')


        def reassign_submodule_attributes(verbose=1):
            """
            Updates attributes in the __init__ modules with updated attributes
            in the submodules.
            """
            import sys
            if verbose and '--quiet' not in sys.argv:
                print('dev reimport')
            # Self import
            import {modname}
            # Implicit reassignment.
            seen_ = set([])
            for tup in IMPORT_TUPLES:
                if len(tup) > 2 and tup[2]:
                    continue  # dont import package names
                submodname, fromimports = tup[0:2]
                submod = getattr({modname}, submodname)
                for attr in dir(submod):
                    if attr.startswith('_'):
                        continue
                    if attr in seen_:
                        # This just holds off bad behavior
                        # but it does mimic normal util_import behavior
                        # which is good
                        continue
                    seen_.add(attr)
                    setattr({modname}, attr, getattr(submod, attr))


        def reload_subs(verbose=1):
            """ Reloads {modname} and submodules """
            if verbose:
                print('Reloading {modname} submodules')
            rrr(verbose > 1)
            def wrap_fbrrr(mod):
                def fbrrr(*args, **kwargs):
                    """ fallback reload """
                    if verbose > 0:
                        print('Auto-reload (using rrr) not setup for mod=%r' % (mod,))
                return fbrrr
            def get_rrr(mod):
                if hasattr(mod, 'rrr'):
                    return mod.rrr
                else:
                    return wrap_fbrrr(mod)
            def get_reload_subs(mod):
                return getattr(mod, 'reload_subs', wrap_fbrrr(mod))
            {reload_body}
            rrr(verbose > 1)
            try:
                # hackish way of propogating up the new reloaded submodule attributes
                reassign_submodule_attributes(verbose=verbose)
            except Exception as ex:
                print(ex)
        rrrr = reload_subs
        # ENDBLOCK
        '''
    )
    injectstr_fmt = injectstr_fmt.replace('# STARTBLOCK', '')
    injectstr_fmt = injectstr_fmt.replace('# ENDBLOCK', '')
    rrrdir_fmt = '    get_reload_subs({modname})(verbose=verbose)'
    rrrfile_fmt = '    get_rrr({modname})(verbose > 1)'

    def _reload_command(tup):
        if len(tup) > 2 and tup[2] is True:
            return rrrdir_fmt.format(modname=tup[0])
        else:
            return rrrfile_fmt.format(modname=tup[0])

    reload_body = '\n'.join(map(_reload_command, import_tuples)).strip()
    format_dict = {
        'modname': modname,
        'reload_body': reload_body,
        'injecter': injecter,
        'injecter_import': injecter_import,
    }
    inject_execstr = injectstr_fmt.format(**format_dict).strip()
    return inject_execstr


# ----------
# PUBLIC FUNCTIONS
# ----------


[docs]def dynamic_import( modname, import_tuples, developing=True, ignore_froms=[], dump=False, ignore_startswith=[], ignore_endswith=[], ignore_list=[], check_not_imported=True, return_initstr=False, verbose=False, ): """ MAIN ENTRY POINT Dynamically import listed util libraries and their attributes. Create reload_subs function. Using __import__ like this is typically not considered good style However, it is better than import * and this will generate the good file text that can be used when the module is 'frozen" Returns: str: init_inject_str - by default all imports are executed in this function and only the remainig code needed to be executed is returned to define the reload logic. str, str: init_inject_str, init_str - if return_initstr is True then also returns init_str defining the from imports. Ignore: ignore_startswith = [] ignore_endswith = [] check_not_imported = True verbose = True """ if verbose: print('[UTIL_IMPORT] Running Dynamic Imports for modname=%r ' % modname) # Get the module that will be imported into try: module = sys.modules[modname] except Exception: module = __import__(modname) # List of modules to be imported imports = [tup[0] for tup in import_tuples] # Import the modules __excecute_imports(module, modname, imports, verbose=verbose) # If developing do explicit import stars if developing: from_imports = __execute_fromimport_star( module, modname, import_tuples, ignore_list=ignore_list, ignore_startswith=ignore_startswith, ignore_endswith=ignore_endswith, check_not_imported=check_not_imported, verbose=verbose, ) else: from_imports = __execute_fromimport( module, modname, import_tuples, verbose=verbose ) inject_execstr = _inject_execstr(modname, import_tuples) # If requested: print what the __init__ module should look like dump_requested = ( ('--dump-%s-init' % modname) in sys.argv or ('--print-%s-init' % modname) in sys.argv ) or dump overwrite_requested = ('--update-%s-init' % modname) in sys.argv if verbose: print('[UTIL_IMPORT] Finished Dynamic Imports for modname=%r ' % modname) if dump_requested: is_main_proc = multiprocessing.current_process().name == 'MainProcess' if is_main_proc: from utool import util_str initstr = _initstr(modname, imports, from_imports, inject_execstr) print(util_str.indent(initstr)) # Overwrite the __init__.py file with new explicit imports if overwrite_requested: """ SeeAlso: util_inject.inject_python_code util_str.replace_between_tags """ is_main_proc = multiprocessing.current_process().name == 'MainProcess' if is_main_proc: from utool import util_str from os.path import join, exists initstr = _initstr( modname, imports, from_imports, inject_execstr, withheader=False ) new_else = util_str.indent(initstr) # print(new_else) # Get path to init file so we can overwrite it init_fpath = join(module.__path__[0], '__init__.py') print('attempting to update: %r' % init_fpath) assert exists(init_fpath) new_lines = [] editing = False updated = False # start_tag = '# <AUTOGEN_INIT>' # end_tag = '# </AUTOGEN_INIT>' with open(init_fpath, 'r') as file_: # text = file_.read() lines = file_.readlines() for line in lines: if not editing: new_lines.append(line) if line.strip().startswith('# <AUTOGEN_INIT>'): new_lines.append('\n' + new_else + '\n # </AUTOGEN_INIT>\n') editing = True updated = True if line.strip().startswith('# </AUTOGEN_INIT>'): editing = False # TODO: # new_text = util_str.replace_between_tags(text, new_else, start_tag, end_tag) if updated: print('writing updated file: %r' % init_fpath) new_text = ''.join(new_lines) with open(init_fpath, 'w') as file_: file_.write(new_text) else: print('no write hook for file: %r' % init_fpath) if return_initstr: initstr = _initstr(modname, imports, from_imports, '', withheader=False) return inject_execstr, initstr else: return inject_execstr
[docs]def make_initstr(modname, import_tuples, verbose=False): """ Just creates the string representation. Does no importing. """ imports = [tup[0] for tup in import_tuples] from_imports = __get_from_imports(import_tuples) inject_execstr = _inject_execstr(modname, import_tuples) return _initstr(modname, imports, from_imports, inject_execstr)
[docs]def make_import_tuples(module_path, exclude_modnames=[]): """ Infer the import_tuples from a module_path """ from utool import util_path kwargs = dict(private=False, full=False) module_list = util_path.ls_modulefiles(module_path, noext=True, **kwargs) package_list = util_path.ls_moduledirs(module_path, **kwargs) exclude_set = set(exclude_modnames) module_import_tuples = [ (modname, None) for modname in module_list if modname not in exclude_set ] package_import_tuples = [ (modname, None, True) for modname in package_list if modname not in exclude_set ] import_tuples = module_import_tuples + package_import_tuples return import_tuples