# -*- coding: utf-8 -*-
"""
progress handler.
Old progress funcs needto be depricated ProgressIter and ProgChunks are pretty
much the only useful things here.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import time
import math
import datetime
from functools import partial
from utool import util_logging
from utool import util_inject
from utool import util_arg
from utool import util_time
from utool import util_iter
from utool import util_cplat
from six.moves import range, zip
import collections
import six # NOQA
print, rrr, profile = util_inject.inject2(__name__)
default_timer = util_time.default_timer
SILENT = util_arg.SILENT
VERBOSE = util_arg.VERBOSE
VALID_PROGRESS_TYPES = ['none', 'dots', 'fmtstr', 'simple']
AGGROFLUSH = util_arg.get_argflag('--aggroflush')
PROGGRESS_BACKSPACE = not util_arg.get_argflag(('--screen', '--progress-backspace'))
NO_PROGRESS = util_arg.get_argflag(('--no-progress', '--noprogress'))
FORCE_ALL_PROGRESS = util_arg.get_argflag(('--force-all-progress',))
# ('--screen' not in sys.argv and '--progress-backspace' not in sys.argv)
DEBUG_FREQ_ADJUST = util_arg.get_argflag('--debug-adjust-freq')
[docs]def test_progress():
"""
CommandLine:
python -m utool.util_progress --test-test_progress
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_progress import * # NOQA
>>> test_progress()
"""
import utool as ut
# import time
# ut.rrrr()
print('_________________')
# numiter = 50
# sleeptime = 1E-4
# sleeptime2 = 1E-2
numiter = 20
sleeptime = 1e-7
sleeptime2 = 1e-7
with ut.Timer():
for x in ut.ProgressIter(range(0, numiter), freq=8, adjust=True):
time.sleep(sleeptime)
print('_________________')
numiter = 50
sleeptime = 1e-4
with ut.Timer():
for x in ut.ProgressIter(range(0, numiter), freq=8, adjust=True):
time.sleep(sleeptime)
print('_________________')
print('No frequncy run:')
with ut.Timer():
for x in range(0, numiter):
time.sleep(sleeptime)
print('_________________')
numiter = 500
sleeptime = 8e-7
with ut.Timer():
for x in ut.ProgressIter(range(0, numiter), freq=8, adjust=True):
time.sleep(sleeptime)
print('_________________')
with ut.Timer():
for x in ut.ProgressIter(range(0, numiter), freq=200):
time.sleep(sleeptime)
print('_________________')
print('No frequncy run:')
with ut.Timer():
for x in range(0, numiter):
time.sleep(sleeptime)
print('_________________')
# Test nested iter
# progiter1 = ut.ProgressIter(range(0, 10), lbl='prog1', freq=1, adjust=False)
# for count1 in progiter1:
# progiter_partials = progiter1.get_subindexers(1)
# progiter2 = progiter_partials[0](range(0, 7), lbl='sub_prog1', freq=1, adjust=False)
# for count2 in progiter2:
# pass
for x in ut.ProgressIter(zip(range(10), range(10)), freq=8, adjust=True):
time.sleep(sleeptime)
# progiter3 = progiter_partials[1](range(0, 3), lbl='sub_prog2', freq=1, adjust=False)
# for count3 in progiter3:
# pass
print('Double backspace progress 1')
progiter1 = ut.ProgressIter(
range(0, 10), lbl='prog1', freq=1, adjust=False, backspace=False
)
for count1 in progiter1:
progiter2 = ut.ProgressIter(
range(0, 10), lbl='prog2', freq=1, adjust=False, backspace=True
)
for count2 in progiter2:
time.sleep(sleeptime2)
print('Double backspace progress 2')
progiter1 = ut.ProgressIter(
range(0, 10), lbl='prog1', freq=1, adjust=False, backspace=True
)
for count1 in progiter1:
progiter2 = ut.ProgressIter(
range(0, 10), lbl='prog2', freq=1, adjust=False, backspace=True
)
for count2 in progiter2:
time.sleep(sleeptime2)
[docs]def get_num_chunks(length, chunksize):
r"""
Returns the number of chunks that a list will be split into given a
chunksize.
Args:
length (int):
chunksize (int):
Returns:
int: n_chunks
CommandLine:
python -m utool.util_progress --exec-get_num_chunks:0
Example0:
>>> # ENABLE_DOCTEST
>>> from utool.util_progress import * # NOQA
>>> length = 2000
>>> chunksize = 256
>>> n_chunks = get_num_chunks(length, chunksize)
>>> result = ('n_chunks = %s' % (six.text_type(n_chunks),))
>>> print(result)
n_chunks = 8
"""
n_chunks = int(math.ceil(length / chunksize))
return n_chunks
[docs]def ProgChunks(list_, chunksize, nInput=None, **kwargs):
"""
Yeilds an iterator in chunks and computes progress
Progress version of ut.ichunks
Args:
list_ (list):
chunksize (?):
nInput (None): (default = None)
Kwargs:
length, freq
Returns:
ProgressIter: progiter_
CommandLine:
python -m utool.util_progress ProgChunks --show
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_progress import * # NOQA
>>> import utool as ut
>>> list_ = range(100)
>>> chunksize = 10
>>> nInput = None
>>> progiter_ = ProgChunks(list_, chunksize, nInput)
>>> iter_ = iter(progiter_)
>>> chunk = six.next(iter_)
>>> assert len(chunk) == 10
>>> rest = ut.flatten(list(progiter_))
>>> assert len(rest) == 90
"""
if nInput is None:
nInput = len(list_)
n_chunks = get_num_chunks(nInput, chunksize)
kwargs['length'] = n_chunks
if 'freq' not in kwargs:
kwargs['freq'] = 1
chunk_iter = util_iter.ichunks(list_, chunksize)
progiter_ = ProgressIter(chunk_iter, **kwargs)
return progiter_
[docs]def ProgPartial(*args, **kwargs):
return partial(ProgressIter, *args, **kwargs)
[docs]class ProgressIter(object):
"""
Wraps a for loop with progress reporting
lbl='Progress: ', length=0, flushfreq=4, startafter=-1, start=True,
repl=False, approx=False, disable=False, writefreq=1, with_time=False,
backspace=True, pad_stdout=False, wfreq=None, ffreq=None, freq=None,
total=None, num=None, with_totaltime=None
Referencs:
https://github.com/verigak/progress/blob/master/progress/__init__.py
Args:
iterable (): iterable normally passed to for loop
lbl (str): progress label
length (int):
flushfreq (int):
startafter (int):
start (bool):
repl (bool):
approx (bool):
enabled (bool):
writefreq (int):
with_totaltime (bool):
backspace (bool):
pad_stdout (bool):
autoadjust (bool): no adjusting frequency if True (default False)
wfreq (None): alias for write_freq
ffreq (None): alias for flush_freq
total (None): alias for length
num (None): alias for length
Timeit::
import utool as ut
setup = ut.codeblock(
'''
import utool as ut
from six.moves import range, zip
import time
def time_append(size):
start_time = time.time()
last_time = start_time
list2 = []
for x in range(size):
now_time = time.time()
between = now_time - last_time
last_time = now_time
list2.append(between)
def time_assign(size):
start_time = time.time()
last_time = start_time
list1 = ut.alloc_nones(size)
for x in range(size):
now_time = time.time()
between = now_time - last_time
last_time = now_time
list1[x] = between
def time_baseline(size):
start_time = time.time()
last_time = start_time
for x in range(size):
now_time = time.time()
between = now_time - last_time
last_time = now_time
def time_null(size):
for x in range(size):
pass
''')
input_sizes = [2 ** count for count in range(7, 12)]
stmt_list = ['time_assign', 'time_append', 'time_baseline', 'time_null']
input_sizes=[100, 1000, 10000]
ut.timeit_grid(stmt_list, setup, input_sizes=input_sizes, show=True)
CommandLine:
python -m utool.util_progress --test-ProgressIter
python -m utool.util_progress --test-ProgressIter:0
python -m utool.util_progress --test-ProgressIter:1
python -m utool.util_progress --test-ProgressIter:2
python -m utool.util_progress --test-ProgressIter:3
Example:
>>> # ENABLE_DOCTEST
>>> import utool as ut
>>> from six.moves import range
>>> num = 1000
>>> num2 = 10001
>>> results1 = [x for x in ut.ProgressIter(range(num), wfreq=10, adjust=True)]
>>> results4 = [x for x in ut.ProgressIter(range(num), wfreq=1, adjust=True)]
>>> results2 = [x for x in range(num)]
>>> results3 = [x for x in ut.progiter((y + 1 for y in range(num2)),
>>> ntotal=num2, wfreq=1000,
>>> backspace=True, adjust=True)]
>>> assert results1 == results2
Example1:
>>> # DISABLE_DOCTEST
>>> # SLOW_DOCTEST
>>> import utool as ut
>>> from six.moves import range
>>> num2 = 10001
>>> progiter = ut.ProgressIter(range(num2), lbl='testing primes',
>>> report_unit='seconds', freq=1,
>>> time_thresh=.1, adjust=True)
>>> [ut.get_nth_prime_bruteforce(29) for x in progiter]
Example2:
>>> # DISABLE_DOCTEST
>>> # SLOW_DOCTEST
>>> import utool as ut
>>> from six.moves import range
>>> num2 = 100001
>>> progiter = ut.ProgressIter(range(num2), lbl='testing primes',
>>> report_unit='seconds', freq=1,
>>> time_thresh=3, adjust=True, bs=True)
>>> [ut.get_nth_prime_bruteforce(29) for x in progiter]
Example3:
>>> # DISABLE_DOCTEST
>>> # SLOW_DOCTEST
>>> import utool as ut
>>> from six.moves import range
>>> import time
>>> crazy_time_list = [.001, .01, .0001] * 1000
>>> crazy_time_iter = (time.sleep(x) for x in crazy_time_list)
>>> progiter = ut.ProgressIter(crazy_time_iter, lbl='crazy times', length=len(crazy_time_list), freq=10)
>>> list(progiter)
"""
def __init__(self, iterable=None, *args, **kwargs):
self.iterable = iterable
if len(args) < 2 and 'nTotal' not in kwargs and 'length' not in kwargs:
try:
length = len(iterable)
kwargs['length'] = length
except Exception:
pass
self.use_rate = kwargs.pop('use_rate', True)
self.use_rate = True # Force
self.lbl = kwargs.get('lbl', 'lbl')
self.lbl = kwargs.get('label', self.lbl)
self.length = kwargs.get('nTotal', kwargs.get('length', 0))
# self.backspace = kwargs.get('backspace', True)
self.backspace = kwargs.get('backspace', kwargs.get('bs', False))
self.freq = kwargs.get('freq', 1)
self.invert_rate = kwargs.get('invert_rate', False)
self.auto_invert_rate = kwargs.get('auto_invert_rate', True)
self.verbose = kwargs.pop('verbose', True) # VERBOSE
# self.report_unit = kwargs.get('report_unit', 'minutes')
self.enabled = kwargs.get('enabled', True)
self.report_unit = kwargs.get('report_unit', 'seconds')
# autoadjust frequency of reporting
self.autoadjust = kwargs.get('autoadjust', kwargs.get('adjust', False))
self.time_thresh = kwargs.pop('time_thresh', None)
self.prog_hook = kwargs.pop('prog_hook', None)
self.prehack = kwargs.pop('prehack', None)
self.freq_est_strat = kwargs.pop('freq_est', 'between')
if 'separate' in kwargs:
print('WARNING separate no longer supported by ProgIter')
# FIXME: get these subinder things working
# ~/code/guitool/guitool/guitool_components.py
# self.substep_min = kwargs.pop('substep_min', 0)
# self.substep_size = kwargs.pop('substep_size', 1)
# self.level = kwargs.pop('level', 0)
self.parent_index = kwargs.pop('parent_index', 0)
self.parent_length = kwargs.pop('parent_length', 1)
self.parent_offset = self.parent_index * self.length
self._cursor_at_newline = True
# Window sizes for estimates
self.est_window = kwargs.pop('est_window', 64)
# self.start_offset = self.substep_min
self.stream = kwargs.pop('stream', None)
self.extra = ''
if FORCE_ALL_PROGRESS:
self.freq = 1
self.autoadjust = False
if self.prog_hook is not None:
# Sets the label of a progress bar to the ProgIter label
self.prog_hook.register_progiter(self)
# self.time_thresh_growth = kwargs.pop('time_thresh_growth', 1.0)
self.time_thresh_growth = kwargs.pop('time_thresh_growth', 1.0)
self.with_totaltime = False
if self.freq is None:
self.freq = 1
if self.use_rate:
# Hacky so hacky. this needs major cleanup
# saving args and kwargs so can wait on log_progress call
# not sure where it is called and dont want to break things
self.args = args
self.kwargs = kwargs
self.mark = None
self.end = None
# else:
# self.mark, self.end = log_progress(*args, **kwargs)
self.count = 0
def __call__(self, iterable):
self.iterable = iterable
return self
def __iter__(self):
if not self.enabled:
return iter(self.iterable)
if NO_PROGRESS:
# IF PROGRESS IS TURNED OFF
msg = 'Iterating ' + self.lbl + ' with no progress'
if self.verbose:
print(msg)
# with ut.Timer(msg):
return iter(self.iterable)
else:
# if self.use_rate:
# STANDARD CALL CASE
return self.iter_rate()
# else:
# return self.iter_without_rate()
# def get_subindexers(prog_iter, num_substeps):
# # FIXME and make this a method of progiter
# step_min = (((prog_iter.count - 1) / prog_iter.length) *
# prog_iter.substep_size + prog_iter.substep_min)
# step_size = (1.0 / prog_iter.length) * prog_iter.substep_size
# substep_size = step_size / num_substeps
# substep_min_list = [(step * substep_size) + step_min
# for step in range(num_substeps)]
# #level = prog_iter.level + 1
# DEBUG = False
# if DEBUG:
# with ut.Indenter(' ' * 4 * prog_iter.level):
# print('\n')
# print('+____<NEW SUBSTEPS>____')
# print('Making %d substeps for prog_iter.lbl = %s' % (
# num_substeps, prog_iter.lbl,))
# print(' * step_min = %.2f' % (step_min,))
# print(' * step_size = %.2f' % (step_size,))
# print(' * substep_size = %.2f' % (substep_size,))
# print(' * substep_min_list = %r' % (substep_min_list,))
# print(r'L____</NEW SUBSTEPS>____')
# print('\n')
# subprog_partial_list = [
# partial(ProgressIter,
# parent_length=prog_iter.length * num_substeps,
# parent_index=(prog_iter.count - 1) + (prog_iter.length * step))
# for step in range(num_substeps)]
# return subprog_partial_list
# def build_msg_fmtstr_time(self, lbl, invert_rate, backspace):
# with_wall = True
# tzname = time.tzname[0]
# if util_cplat.WIN32:
# tzname = tzname.replace('Eastern Standard Time', 'EST')
# msg_fmtstr_time = ''.join((
# 'rate=%3.3f seconds/iter, ' if invert_rate else 'rate=%4.2f Hz,',
# ' etr=%s,',
# ' ellapsed=%s,',
# ' wall=%s ' + tzname if with_wall else '',
# #'' if backspace else '\n',
# '\n' if backspace else '',
# ))
# return msg_fmtstr_time
[docs] @staticmethod
def build_msg_fmtstr_head_cols(length, lbl):
nTotal_ = '?' if length == 0 else six.text_type(length)
msg_head_columns = ['', lbl, ' {count:4d}/', nTotal_, '... ']
return msg_head_columns
[docs] @staticmethod
def build_msg_fmtstr2(lbl, length, invert_rate, backspace):
r"""
Args:
lbl (str):
invert_rate (bool):
backspace (bool):
Returns:
str: msg_fmtstr_time
CommandLine:
python -m utool.util_progress --exec-ProgressIter.build_msg_fmtstr2
Setup:
>>> from utool.util_progress import * # NOQA
>>> lbl = 'foo'
>>> invert_rate = True
>>> backspace = False
>>> length = None
Example:
>>> # DISABLE_DOCTEST
>>> msg_fmtstr_time = ProgressIter.build_msg_fmtstr2(lbl, length, invert_rate, backspace)
>>> result = ('%s' % (ut.repr2(msg_fmtstr_time),))
>>> print(result)
"""
with_wall = True
tzname = time.tzname[0]
if util_cplat.WIN32:
tzname = tzname.replace('Eastern Standard Time', 'EST')
# ansii/vt100 code for clearline
# CLEARLINE_L2 = '\33[2K'
# BEFORE_PROG = '\r\033[?25l'
CLEARLINE_EL0 = '\33[0K' # clear line to right
# CLEARLINE_EL1 = '\33[1K' # clear line to left
CLEARLINE_EL2 = '\33[2K' # clear line
# DECTCEM_HIDE = '\033[?25l' # hide cursor
CLEAR_BEFORE = '\r' + CLEARLINE_EL2 # + DECTCEM_HIDE
# FIXME: hideing cursor persists if the program crashes
CLEAR_AFTER = CLEARLINE_EL0
msg_head = ProgressIter.build_msg_fmtstr_head_cols(length, lbl)
if backspace:
msg_head = [CLEAR_BEFORE] + msg_head
msg_tail = [
(
'rate={rate:4.2f} iter/sec, '
if invert_rate
else 'rate={rate:4.2f} sec/iter, '
),
('' if length == 0 else ' etr={etr},'),
' ellapsed={ellapsed},',
(' wall={wall} ' + tzname if with_wall else ''),
# backslash-r is a carrage return and undoes all previous output on
# a written line
(' {extra}'),
CLEAR_AFTER if backspace else '\n',
]
msg_fmtstr_time = ''.join((msg_head + msg_tail))
return msg_fmtstr_time
[docs] def iter_rate(self):
"""
pun not intended
# TODO: record iteration times for analysis
# TODO Incorporate this better
# FIXME; pad_stdout into subfunctions
import dis
dis.dis(ut.ProgressIter.iter_rate)
"""
# class IterState(object):
# def __init__(state):
# state.freq = 1
# state.freq = 1
# pass
adjust = self.autoadjust
self._cursor_at_newline = not self.backspace
# SETUP VARIABLES
# HACK: reaquire logging print funcs in case they have changed
if self.stream is None:
self.write = util_logging._utool_write()
self.flush = util_logging._utool_flush()
else:
self.write = lambda msg: self.stream.write(msg) # NOQA
self.flush = lambda: self.stream.flush() # NOQA
length = self.length * self.parent_length # hack
freq = self.freq
self.count = 0
between_count = 0
last_count = 0
# how long iterations should be before a flush
# (used for freq adjustment)
time_thresh = (
self._get_timethresh_heuristics()
if self.time_thresh is None
else self.time_thresh
)
time_thresh_growth = self.time_thresh_growth
if time_thresh_growth > 1:
# time_thresh_growth is specified for very long processes
# print out the starting timestamp in that case
timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + time.tzname[0]
print('Start progress lbl= %s at %s' % (self.lbl, timestamp,))
# time_thresh = 0.5
max_between_time = -1.0
max_between_count = -1.0 # why is this different? # because frequency varies
# TODO: should be kept as a statistic that uses the max time from a
# list of iterations divided by the size of that list that will account
# for buffering issues
iters_per_second = 0
self.iters_per_second = float('nan')
self.est_seconds_left = 0
self.total_seconds = 0
# Write initial message
# force_newlines = not self.backspace
start_msg_fmt = ''.join(self.build_msg_fmtstr_head_cols(length, self.lbl))
self.msg_fmtstr = self.build_msg_fmtstr2(
self.lbl, length, self.invert_rate, self.backspace
)
try:
util_logging._utool_flush()()
except IOError as ex:
# There is some weird error when doing progress in IPython notebook
if util_arg.VERBOSE:
print('IOError flushing %s' % (ex,))
if not self.prehack:
if self.backspace:
self.display_message()
elif self.verbose:
start_msg = start_msg_fmt.format(count=self.parent_offset)
util_logging._utool_write()(start_msg + '\n')
self._cursor_at_newline = not self.backspace
try:
util_logging._utool_flush()()
except IOError as ex:
# There is some weird error when doing progress in IPython notebook
if util_arg.VERBOSE:
print('IOError flushing %s' % (ex,))
else:
self._cursor_at_newline = True
if self.prog_hook is not None:
self.prog_hook(self.count, length)
# TODO: on windows is time.clock better?
# http://exnumerus.blogspot.com/2011/02/how-to-quickly-plot-multiple-line.html
start_time = default_timer()
last_time = start_time
start = 1 + self.parent_offset
if self.freq_est_strat == 'between':
FREQ_EST = 0
elif self.freq_est_strat == 'absolute':
FREQ_EST = 1
else:
FREQ_EST = 1
USE_RECORD = True
USE_RECORD_WINDOWED_AVG = True
USE_RECORD_WINDOWED_WEIGHT = 0.9
# use last 64 times to compute a more stable average rate
measure_between_time = collections.deque([], maxlen=self.est_window)
# Wrap the for loop with a generator
for self.count, item in enumerate(self.iterable, start=start):
if self.prehack:
# hack to print before yeilding
# so much for efficiency
self.set_extra((self.lbl + '=' + self.prehack) % item)
self.display_message()
self.ensure_newline()
# GENERATE
yield item
if self.prehack or (self.count) % freq == 0:
now_time = default_timer()
between_time = now_time - last_time
between_count = self.count - last_count
total_seconds = now_time - start_time
self.total_seconds = total_seconds
if FREQ_EST == 0:
if USE_RECORD:
measure_between_time.append(
between_count / (float(between_time) + 1e-9)
)
if USE_RECORD_WINDOWED_AVG:
iters_per_second = None
for measure_between in measure_between_time:
if iters_per_second is None:
iters_per_second = measure_between
else:
iters_per_second = (
(USE_RECORD_WINDOWED_WEIGHT) * iters_per_second
+ (1.0 - USE_RECORD_WINDOWED_WEIGHT)
* measure_between
)
else:
iters_per_second = sum(measure_between_time) / len(
measure_between_time
)
else:
iters_per_second = between_count / (float(between_time) + 1e-9)
elif FREQ_EST == 1:
iters_per_second = (now_time - start_time) / self.count
self.iters_per_second = iters_per_second
# If the future is known
if length is None:
est_seconds_left = -1
else:
iters_left = length - self.count
est_seconds_left = iters_left / (1.0 / iters_per_second + 1e-9)
self.est_seconds_left = est_seconds_left
# /future
last_count = self.count
last_time = now_time
# ADJUST FREQ IF NEEDED
# Adjust frequency if printing too quickly
# so progress doesnt slow down actual function
# TODO: better adjust algorithm
time_thresh *= time_thresh_growth
if adjust and (
between_time < time_thresh or between_time > time_thresh * 2.0
):
max_between_time = max(max(max_between_time, between_time), 1e-9)
max_between_count = max(max_between_count, between_count)
# If progress was uniform and all time estimates were
# perfect this would be the new freq to achieve time_thresh
new_freq = max(
int(time_thresh * max_between_count / max_between_time), 1
)
if DEBUG_FREQ_ADJUST:
print('\n+---')
print('[prog] between_count = %r' % between_count)
print('[prog] between_time = %.8r' % between_time)
print('[prog] time_thresh = %r' % time_thresh)
print('[prog] max_between_count = %r' % max_between_count)
print('[prog] max_between_time = %.8r' % max_between_time)
print('[prog] Adusting frequency from: %r' % freq)
print('[prog] Adusting frequency to: %r' % new_freq)
print('L___')
# But things are not perfect. So, don't make drastic changes
max_freq_change_up = max(256, freq * 2)
max_freq_change_down = freq // 2
if (new_freq - freq) > max_freq_change_up:
freq += max_freq_change_up
elif (freq - new_freq) > max_freq_change_down:
freq -= max_freq_change_down
else:
freq = new_freq
if not self.prehack:
self.display_message()
# DO PROGRESS INFO
if self.prog_hook is not None:
# From the point of view of the progress iter, we are about
# to enter the body of a for loop. (But we may have
# executed the body implicitly in the yeild.... so it is
# ambiguous. In the second case 0 will be executed twice.
self.prog_hook(self.count, length)
if self.prehack:
self.set_extra('')
# --- end of main loop
# cleanup
if (self.count) % freq != 0:
# If the final line of progress was not written in the loop, write
# it here
self.est_seconds_left = 0
self.total_seconds = default_timer() - start_time
self.display_message()
if self.prog_hook is not None:
# From the point of view of the progress iter, we are about to
# enter the body of a for loop. (But we may have executed the
# body implicitly in the yeild.... so it is ambiguous. In the
# second case 0 will be executed twice.
self.prog_hook(self.count, length)
self.ensure_newline()
[docs] def display_message(self):
# HACK to be more like sklearn.extrnals ProgIter version
if self.verbose:
instant_invert_rate = self.iters_per_second < 0.1
if self.auto_invert_rate and self.invert_rate != instant_invert_rate:
self.invert_rate = instant_invert_rate
length = self.length * self.parent_length # hack
self.msg_fmtstr = self.build_msg_fmtstr2(
self.lbl, length, self.invert_rate, self.backspace
)
rate = (
1.0 / (self.iters_per_second + 1e-9)
if self.invert_rate
else self.iters_per_second
)
msg = self.msg_fmtstr.format(
count=self.count,
rate=rate,
etr=six.text_type(datetime.timedelta(seconds=int(self.est_seconds_left))),
ellapsed=six.text_type(
datetime.timedelta(seconds=int(self.total_seconds))
),
wall=time.strftime('%H:%M'),
extra=self.extra,
)
self.write(msg)
self._cursor_at_newline = not self.backspace
try:
self.flush()
except IOError as ex:
if util_arg.VERBOSE:
print('IOError flushing %s' % (ex,))
# print('self.flush = %r' % (self.flush,))
# import utool as ut
# ut.debug_logging_iostreams()
# ut.printex(ex)
# raise
pass
[docs] def ensure_newline(self):
"""
use before any custom printing when using the progress iter to ensure
your print statement starts on a new line instead of at the end of a
progress line
"""
DECTCEM_SHOW = '\033[?25h' # show cursor
AT_END = DECTCEM_SHOW + '\n'
if not self._cursor_at_newline:
self.write(AT_END)
self._cursor_at_newline = True
def _get_timethresh_heuristics(self):
"""
resonably decent hueristics for how much time to wait before
updating progress.
"""
if self.length > 1e5:
time_thresh = 2.5
elif self.length > 1e4:
time_thresh = 2.0
elif self.length > 1e3:
time_thresh = 1.0
else:
time_thresh = 0.5
return time_thresh
progiter = ProgressIter
[docs]class ProgIter(ProgressIter):
""" Thin wrapper with better arg positions """
def __init__(self, iterable, lbl='Prog', adjust=True, freq=1, bs=True, **kwargs):
import utool as ut
super(ut.ProgIter, self).__init__(
iterable, lbl=lbl, adjust=adjust, freq=freq, bs=bs, **kwargs
)
[docs]def progress_str(
max_val, lbl='Progress: ', repl=False, approx=False, backspace=PROGGRESS_BACKSPACE
):
r""" makes format string that prints progress: %Xd/MAX_VAL with backspaces
NOTE: \r can be used instead of backspaces. This function is not very
relevant because of that.
"""
# string that displays max value
max_str = six.text_type(max_val)
if approx:
# denote approximate maximum
max_str = '~' + max_str
dnumstr = six.text_type(len(max_str))
# string that displays current progress
cur_str = '%' + dnumstr + 'd'
# If user passed in the label
if repl:
_fmt_str = lbl.replace('<cur_str>', cur_str).replace('<max_str>', max_str)
else:
_fmt_str = lbl + cur_str + '/' + max_str
if backspace:
# put backspace characters into the progress string
# (looks nice on normal terminals)
# nBackspaces = len(_fmt_str) - len(dnumstr) + len(max_str)
# backspaces = '\b' * nBackspaces
# fmt_str = backspaces + _fmt_str
# FIXME: USE CARAGE RETURN INSTEAD OF BACKSPACES
fmt_str = '\r' + _fmt_str
else:
# FIXME: USE CARAGE RETURN INSTEAD OF BACKSPACES
# this looks better on terminals without backspaces
fmt_str = _fmt_str + '\n'
return fmt_str
[docs]def log_progress(
lbl='Progress: ',
length=0,
flushfreq=4,
startafter=-1,
start=True,
repl=False,
approx=False,
disable=False,
writefreq=1,
with_time=False,
backspace=True,
pad_stdout=False,
wfreq=None,
ffreq=None,
freq=None,
total=None,
num=None,
with_totaltime=None,
):
"""
DEPRICATE
FIXME: depricate for ProgressIter.
still used in util_dev
"""
global AGGROFLUSH
# Alias kwargs with simpler names
if num is not None:
length = num
if total is not None:
length = total
if wfreq is not None:
writefreq = wfreq
if ffreq is not None:
flushfreq = ffreq
if freq is not None:
writefreq = flushfreq = freq
if with_totaltime is not None:
with_time = with_totaltime
# flush frequency must be a multiple of write frequency
flushfreq = max(int(round(flushfreq / writefreq)), 1) * writefreq
if length < startafter or disable:
# Do not mark progress if only executing a small number of tasks
def mark_progress(*args):
pass
def end_progress(*args):
pass
return mark_progress, end_progress
else:
write_fn = util_logging._utool_write()
flush_fn = util_logging._utool_flush()
# build format string for displaying progress
fmt_str = progress_str(
length, lbl=lbl, repl=repl, approx=approx, backspace=backspace
)
if AGGROFLUSH:
# Progress function which automatically flushes
def mark_progress(count, flush_fn=flush_fn):
count_ = count + 1
write_fn(fmt_str % (count_))
flush_fn()
else:
# Progress function flushes every <flushfreq> times
def mark_progress(
count,
fmt_str=fmt_str,
flushfreq=flushfreq,
writefreq=writefreq,
write_fn=write_fn,
flush_fn=flush_fn,
):
count_ = count + 1
if count_ % writefreq == 0:
write_fn(fmt_str % count_)
if count_ % flushfreq == 0:
flush_fn()
if pad_stdout:
write_fn('\n')
write_fn('\n')
flush_fn()
if with_time:
tt = util_time.tic(lbl)
def end_progress(count_=length, write_fn=write_fn, flush_fn=flush_fn):
write_fn(fmt_str % (count_))
write_fn('\n')
flush_fn()
if with_time:
util_time.toc(tt)
if pad_stdout:
write_fn('\n\n')
flush_fn()
# mark_progress(0)
if start:
mark_progress(-1)
return mark_progress, end_progress
if __name__ == '__main__':
"""
CommandLine:
python -c "import utool, utool.util_progress; utool.doctest_funcs(utool.util_progress, allexamples=True)"
python -c "import utool, utool.util_progress; utool.doctest_funcs(utool.util_progress)"
python -m utool.util_progress
python -m utool.util_progress --allexamples
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()