# -*- coding: utf-8 -*-
# Utools for setup.py files
from __future__ import absolute_import, division, print_function
import sys
import textwrap
from os.path import exists, join, dirname, split, splitext
import os
from utool import util_cplat
from utool import util_path
from utool import util_io
from utool import util_str
#from utool import util_dev
from utool.util_dbg import printex
from utool._internal import meta_util_arg
VERBOSE = meta_util_arg.VERBOSE
[docs]class SETUP_PATTERNS():
clutter_pybuild = [
'*.pyc',
'*.pyo',
]
clutter_cyth = [
'_*_cyth.o',
'_*_cyth_bench.py',
'run_cyth_benchmarks.sh',
'_*_cyth.c',
'_*_cyth.pyd',
'_*_cyth.pxd',
'_*_cyth.html',
'_*_cyth.pyx',
'_*_cyth.so',
'_*_cyth.dylib'
]
chmod_test = ['test_*.py']
[docs]def read_license(license_file):
try:
with open(license_file, 'r') as file_:
firstline = file_.readline()
license_type = firstline.replace('License', '').strip()
return license_type
except IOError:
print('Warning no LICENCE file')
return '???'
[docs]def build_pyo(project_dirs):
pyexe = util_cplat.python_executable()
for dir_ in project_dirs:
#command_args = [pyexe, '-O', '-m', 'compileall', dir_ + '\*.py']
command_args = [pyexe, '-O', '-m', 'compileall', dir_]
#command = ' '.join(command_args)
#os.system(command)
util_cplat.shell(command_args)
[docs]def setup_chmod(setup_fpath, setup_dir, chmod_patterns):
""" Gives files matching pattern the same chmod flags as setup.py """
#st_mode = os.stat(setup_fpath).st_mode
st_mode = 33277
for pattern in chmod_patterns:
for fpath in util_path.glob(setup_dir, pattern, recursive=True):
print('[setup] chmod fpath=%r' % fpath)
os.chmod(fpath, st_mode)
[docs]def assert_in_setup_repo(setup_fpath, name=''):
""" pass in __file__ from setup.py """
setup_dir, setup_fname = split(setup_fpath)
cwd = os.getcwd()
#repo_dname = split(setup_dir)[1]
#print('cwd = %r' % (cwd))
#print('repo_dname = %r' % repo_dname)
#print('setup_dir = %r' % (setup_dir))
#print('setup_fname = %r' % (setup_fname))
try:
assert setup_fname == 'setup.py', 'name is not setup.py'
#assert name == '' or repo_dname == name,
('name=%r' % name)
assert cwd == setup_dir, 'cwd is not setup_dir'
assert exists(setup_dir), 'setup dir does not exist'
assert exists(join(setup_dir, 'setup.py')), 'setup.py does not exist'
except AssertionError as ex:
printex(ex, 'ERROR!: setup.py must be run from repository root')
raise
[docs]def clean(setup_dir, clutter_patterns, clutter_dirs):
print('[setup] clean()')
clutter_patterns_ = [pat for pat in clutter_patterns if not pat.endswith('/')]
_clutter_dirs = [pat[:-1] for pat in clutter_patterns if pat.endswith('/')]
clutter_dirs_ = _clutter_dirs + clutter_dirs
util_path.remove_files_in_dir(setup_dir,
clutter_patterns_,
recursive=True,
verbose=VERBOSE)
for dir_ in clutter_dirs_:
util_path.delete(dir_, verbose=VERBOSE, print_exists=False)
#util_path.remove_files_in_dir(dir_)
#for fpath in cython_files:
# fname, ext = splitext(fpath)
# for libext in util_cplat.LIB_EXT_LIST:
# util_path.remove_file(fname + libext)
# util_path.remove_file(fname + '.c')
#def build_cython(cython_files):
# """ doesn't work """
# for fpath in cython_files:
# util_dev.compile_cython(fpath)
[docs]def translate_cyth():
import cyth
cyth.translate_all()
[docs]def get_numpy_include_dir():
try:
import numpy as np
return np.get_include()
except ImportError:
return ''
[docs]def find_ext_modules(disable_warnings=True):
from setuptools import Extension
import utool
from os.path import relpath
cwd = os.getcwd()
CYTH = 'cyth' in sys.argv
BEXT = 'bext' in sys.argv
BUILD = 'build' in sys.argv
BUILD_EXT = 'build_ext' in sys.argv
if any([BEXT, CYTH]):
translate_cyth() # translate cyth before finding ext modules
if not any([BEXT, BUILD, BUILD_EXT]):
# dont find modules if they are not being built
return []
#pyx_list = utool.glob(cwd, '*_cython.pyx', recursive=True)
pyx_list = utool.glob(cwd, '*.pyx', recursive=True)
if disable_warnings:
extra_compile_args = ['-Wno-format', '-Wno-unused-function']
else:
extra_compile_args = []
ext_modules = []
for pyx_abspath in pyx_list:
pyx_relpath = relpath(pyx_abspath, cwd)
pyx_modname, _ = splitext(pyx_relpath.replace('\\', '.').replace('/', '.'))
print('[find_ext] Found Module:')
print(' * pyx_modname = %r' % (pyx_modname,))
print(' * pyx_relpath = %r' % (pyx_relpath,))
extmod = Extension(pyx_modname, [pyx_relpath],
include_dirs=[get_numpy_include_dir()],
extra_compile_args=extra_compile_args)
ext_modules.append(extmod)
return ext_modules
[docs]def find_packages(recursive=True, maxdepth=None):
"""
Finds all directories with an __init__.py file in them
"""
import utool
if utool.VERBOSE:
print('[util_setup] find_packages(recursive=%r, maxdepth=%r)' % (recursive, maxdepth))
from os.path import relpath
cwd = os.getcwd()
init_files = utool.glob(cwd, '__init__.py', recursive=recursive, maxdepth=maxdepth)
package_paths = list(map(dirname, init_files))
package_relpaths = [relpath(path, cwd) for path in package_paths]
packages = []
for path in package_relpaths:
base = utool.dirsplit(path)[0]
if exists(join(base, '__init__.py')):
package = path.replace('/', '.').replace('\\', '.')
packages.append(package)
return packages
[docs]def get_cmdclass():
try:
from Cython.Distutils import build_ext
cmdclass = {'build_ext': build_ext}
return cmdclass
except Exception as ex:
print(ex)
print('WARNING: Cython is not installed. This is only a problem if you are building C extensions')
return {}
[docs]def parse_author():
""" TODO: this function should parse setup.py or a module for
the author variable
"""
return 'Jon Crall' # FIXME
[docs]def autogen_sphinx_apidoc():
r"""
autogen_sphinx_docs.py
Ignore:
C:\Python27\Scripts\autogen_sphinx_docs.py
autogen_sphinx_docs.py
pip uninstall sphinx
pip install sphinx
pip install sphinxcontrib-napoleon
pip install sphinx --upgrade
pip install sphinxcontrib-napoleon --upgrade
cd C:\Python27\Scripts
ls C:\Python27\Scripts
python -c "import sphinx; print(sphinx.__version__)"
CommandLine:
python -m utool.util_setup --exec-autogen_sphinx_apidoc
Example:
>>> # SCRIPT
>>> from utool.util_setup import * # NOQA
>>> autogen_sphinx_apidoc()
"""
# TODO: assert sphinx-apidoc exe is found
# TODO: make find_exe work?
import utool as ut
def build_sphinx_apidoc_cmdstr():
print('')
print('if this fails try: sudo pip install sphinx')
print('')
apidoc = 'sphinx-apidoc'
if ut.WIN32:
winprefix = 'C:/Python27/Scripts/'
sphinx_apidoc_exe = winprefix + apidoc + '.exe'
else:
sphinx_apidoc_exe = apidoc
apidoc_argfmt_list = [
sphinx_apidoc_exe,
'--force',
'--full',
'--maxdepth="{maxdepth}"',
'--doc-author="{author}"',
'--doc-version="{doc_version}"',
'--doc-release="{doc_release}"',
'--output-dir="_doc"',
#'--separate', # Put documentation for each module on its own page
'--private', # Include "_private" modules
'{pkgdir}',
]
outputdir = '_doc'
author = ut.parse_author()
packages = ut.find_packages(maxdepth=1)
assert len(packages) != 0, 'directory must contain at least one package'
if len(packages) > 1:
assert len(packages) == 1,\
('FIXME I dont know what to do with more than one root package: %r'
% (packages,))
pkgdir = packages[0]
version = ut.parse_package_for_version(pkgdir)
modpath = dirname(ut.truepath(pkgdir))
apidoc_fmtdict = {
'author': author,
'maxdepth': '8',
'pkgdir': pkgdir,
'doc_version': version,
'doc_release': version,
'outputdir': outputdir,
}
ut.assert_exists('setup.py')
ut.ensuredir('_doc')
apidoc_fmtstr = ' '.join(apidoc_argfmt_list)
apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict)
print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir,))
if ut.VERBOSE:
print(ut.dict_str(apidoc_fmtdict))
return apidoc_cmdstr, modpath, outputdir
def build_conf_replstr():
#
# Make custom edits to conf.py
# FIXME:
#ext_search_text = ut.unindent(
# r'''
# extensions = [
# [^\]]*
# ]
# ''')
ext_search_text = r'extensions = \[[^/]*\]'
# TODO: http://sphinx-doc.org/ext/math.html#module-sphinx.ext.pngmath
#'sphinx.ext.mathjax',
exclude_modules = [] # ['ibeis.all_imports']
ext_repl_text = ut.codeblock(
'''
MOCK_MODULES = {exclude_modules}
if len(MOCK_MODULES) > 0:
import mock
for mod_name in MOCK_MODULES:
sys.modules[mod_name] = mock.Mock()
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.viewcode',
# For LaTeX
'sphinx.ext.pngmath',
# For Google Sytle Docstrs
# https://pypi.python.org/pypi/sphinxcontrib-napoleon
'sphinxcontrib.napoleon',
#'sphinx.ext.napoleon',
]
'''
).format(exclude_modules=str(exclude_modules))
#theme_search = 'html_theme = \'default\''
theme_search = 'html_theme = \'[a-zA-Z_1-3]*\''
theme_repl = ut.codeblock(
'''
import sphinx_rtd_theme
html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
''')
head_text = ut.codeblock(
'''
from sphinx.ext.autodoc import between
import sphinx_rtd_theme
import sys
import os
# Dont parse IBEIS args
os.environ['IBIES_PARSE_ARGS'] = 'OFF'
os.environ['UTOOL_AUTOGEN_SPHINX_RUNNING'] = 'ON'
sys.path.append('{modpath}')
sys.path.append(sys.path.insert(0, os.path.abspath("../")))
autosummary_generate = True
modindex_common_prefix = ['_']
'''
).format(modpath=ut.truepath(modpath))
tail_text = ut.codeblock(
'''
def setup(app):
# Register a sphinx.ext.autodoc.between listener to ignore everything
# between lines that contain the word IGNORE
app.connect('autodoc-process-docstring', between('^.*IGNORE.*$', exclude=True))
return app
'''
)
return (ext_search_text, ext_repl_text, theme_search, theme_repl, head_text, tail_text)
apidoc_cmdstr, modpath, outputdir = build_sphinx_apidoc_cmdstr()
ext_search_text, ext_repl_text, theme_search, theme_repl, head_text, tail_text = build_conf_replstr()
dry = ut.get_argflag('--dry')
if not dry:
# Execute sphinx-apidoc
ut.cmd(apidoc_cmdstr, shell=True)
# sphinx-apidoc outputs conf.py to <outputdir>, add custom commands
#
# Change dir to <outputdir>
print('chdir' + outputdir)
os.chdir(outputdir)
conf_fname = 'conf.py'
conf_text = ut.read_from(conf_fname)
conf_text = conf_text.replace('import sys', 'import sys # NOQA')
conf_text = conf_text.replace('import os', 'import os # NOQA')
conf_text = ut.regex_replace(theme_search, theme_repl, conf_text)
conf_text = ut.regex_replace(ext_search_text, ext_repl_text, conf_text)
conf_text = head_text + '\n' + conf_text + tail_text
ut.write_to(conf_fname, conf_text)
# Make the documentation
#if ut.LINUX:
# ut.cmd('make html', shell=True)
#if ut.WIN32:
#raw_input('waiting')
if not ut.get_argflag('--nomake'):
ut.cmd('make', 'html', shell=True)
else:
print(apidoc_cmdstr)
print('cd ' + outputdir)
print('manual edits of conf.py')
print('make html')
[docs]def presetup_commands(setup_fpath, kwargs):
if VERBOSE:
print('[setup] presetup_commands()')
name = kwargs.get('name', '')
# Parse args
project_dirs = kwargs.pop('project_dirs', None)
chmod_patterns = kwargs.pop('chmod_patterns', [])
clutter_dirs = kwargs.pop('clutter_dirs', None)
clutter_patterns = kwargs.pop('clutter_patterns', [])
build_command = kwargs.pop('build_command', NOOP)
# Augment patterns with builtin patterns
chmod_patterns += SETUP_PATTERNS.chmod_test if kwargs.pop('chmod_tests', True) else []
clutter_patterns += SETUP_PATTERNS.clutter_pybuild if kwargs.pop('clean_pybuild', True) else []
clutter_patterns += SETUP_PATTERNS.clutter_cyth if kwargs.pop('clean_pybuild', True) else []
setup_fpath = util_path.truepath(setup_fpath)
#
setup_dir = dirname(setup_fpath)
build_dir = join(setup_dir, 'build')
os.chdir(setup_dir) # change into setup directory
assert_in_setup_repo(setup_fpath, name)
# Augment clutter dirs
if clutter_dirs is None:
clutter_dirs = []
clutter_dirs += [
'build',
'dist',
name + '.egg-info',
'__pycache__'
]
if project_dirs is None:
project_dirs = util_path.ls_moduledirs(setup_dir)
# Execute pre-setup commands based on argv
#BEXT = 'bext' in sys.argv
#BUILD_EXT = 'build_ext' in sys.argv
#CYTH = 'cyth' in sys.argv
#if CYTH:
# translate_cyth()
for arg in iter(sys.argv[:]):
#print(arg)
# Clean clutter files
if arg in ['clean']:
clean(setup_dir, clutter_patterns, clutter_dirs)
#sys.exit(0)
if arg in ['build'] or (not exists(build_dir) and
'clean' not in sys.argv):
if VERBOSE:
print('[setup] Executing build command')
try:
build_command()
except Exception as ex:
printex(ex, 'Error calling buildcommand from cwd=%r\n' % os.getcwd())
raise
if arg in ['versions']:
print('+ ---')
print('Testing preqres package versions')
install_requires = kwargs.get('install_requires', [])
import pip
print('Checking install_requires = [\n%s\n]' % '\n'.join(install_requires))
pip.main(['show'] + [depline.split(' ')[0] for depline in install_requires])
print('L ___ Done Version Testing')
if arg in ['docs']:
# FIXME
autogen_sphinx_apidoc()
# Build optimized files
if arg in ['o', 'pyo']:
build_pyo(project_dirs)
# Cythonize files
#if arg in ['cython']:
# build_cython(cython_files)
# Chmod files
if arg in ['chmod']:
setup_chmod(setup_fpath, setup_dir, chmod_patterns)
try:
sys.argv.remove('docs')
sys.argv.remove('cyth')
#sys.argv.remove('--cyth-write')
except ValueError:
pass
try:
# SUPER HACK
# aliases bext to build_ext --inplace
sys.argv.remove('bext')
sys.argv.append('build_ext')
sys.argv.append('--inplace')
except ValueError:
pass
presetup = presetup_commands
[docs]def parse_package_for_version(name):
"""
Searches for a variable named __version__ in name's __init__.py file and
returns the value. This function parses the source text. It does not load
the module.
"""
from utool import util_regex
init_fpath = join(name, '__init__.py')
version_errmsg = textwrap.dedent(
'''
You must include a __version__ variable
in %s\'s __init__.py file.
Try something like:
__version__ = '1.0.0.dev1' ''' % (name,))
if not exists(init_fpath):
raise AssertionError(version_errmsg)
val_regex = util_regex.named_field('version', '[0-9a-zA-Z.]+')
regexstr = '__version__ *= *[\'"]' + val_regex
def parse_version(line):
# Helper
line = line.replace(' ', '').replace('\t', '')
match_dict = util_regex.regex_parse(regexstr, line)
if match_dict is not None:
return match_dict['version']
# Find the version in the text of the source
#version = 'UNKNOWN_VERSION'
with open(init_fpath, 'r') as file_:
for line in file_.readlines():
if line.startswith('__version__'):
version = parse_version(line)
if version is not None:
return version
raise AssertionError(version_errmsg)
def __infer_setup_kwargs(module, kwargs):
""" Implicitly build kwargs based on standard info """
# Get project name from the module
#if 'name' not in kwargs:
# kwargs['name'] = module.__name__
#else:
# raise AssertionError('must specify module name!')
name = kwargs['name']
# Our projects depend on utool
#if kwargs['name'] != 'utool':
# install_requires = kwargs.get('install_requires', [])
# if 'utool' not in install_requires:
# install_requires.append('utool')
# kwargs['install_requires'] = install_requires
packages = kwargs.get('packages', [])
if name not in packages:
packages.append(name)
kwargs['packages'] = packages
if 'version' not in kwargs:
version = parse_package_for_version(name)
kwargs['version'] = version
# Parse version
#if 'version' not in kwargs:
# if module is None:
# version_errmsg = 'You must include a version (preferably one that matches the __version__ variable in your modules init file'
# raise AssertionError(version_errmsg)
# else:
# Parse license
if 'license' not in kwargs:
try:
kwargs['license'] = read_license('LICENSE')
except IOError:
pass
# Parse readme
if 'long_description' not in kwargs:
kwargs['long_description'] = parse_readme()
[docs]def parse_readme(readmefile='README.md'):
return util_io.read_from(readmefile, verbose=False, strict=False)
if __name__ == '__main__':
"""
CommandLine:
python -m utool.util_setup
python -m utool.util_setup --allexamples
python -m utool.util_setup --allexamples --noface --nosrc
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()