Source code for utool.util_progress

# -*- coding: utf-8 -*-
"""
progress handler.

Old progress funcs needto be depricated ProgressIter and ProgChunks are pretty
much the only useful things here.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import time
import math
import datetime
from functools import partial
from utool import util_logging
from utool import util_inject
from utool import util_arg
from utool import util_time
from utool import util_iter
from utool import util_cplat
from six.moves import range, zip
import six  # NOQA
print, rrr, profile = util_inject.inject2(__name__, '[progress]')

default_timer = util_time.default_timer


QUIET = util_arg.QUIET
SILENT = util_arg.SILENT
VALID_PROGRESS_TYPES = ['none', 'dots', 'fmtstr', 'simple']
AGGROFLUSH = util_arg.get_argflag('--aggroflush')
PROGGRESS_BACKSPACE = not util_arg.get_argflag(('--screen', '--progress-backspace'))
NO_PROGRESS = util_arg.get_argflag(('--no-progress', '--noprogress'))
FORCE_ALL_PROGRESS = util_arg.get_argflag(('--force-all-progress',))
#('--screen' not in sys.argv and '--progress-backspace' not in sys.argv)

DEBUG_FREQ_ADJUST = util_arg.get_argflag('--debug-adjust-freq')


#PROGRESS_WRITE = sys.stdout.write
#PROGRESS_FLUSH = sys.stdout.flush

# FIXME: if this is loaded before logging beings
# progress will not be logged
#PROGRESS_WRITE = util_logging.__UTOOL_WRITE__
PROGRESS_WRITE = print
PROGRESS_FLUSH = util_logging.__UTOOL_FLUSH__


[docs]def test_progress(): """ CommandLine: python -m utool.util_progress --test-test_progress Example: >>> # ENABLE_DOCTEST >>> from utool.util_progress import * # NOQA >>> test_progress() """ import utool as ut #import time #ut.rrrr() print('_________________') numiter = 50 sleeptime = 1E-4 with ut.Timer(): for x in ut.ProgressIter(range(0, numiter), freq=8, adjust=True): time.sleep(sleeptime) print('_________________') numiter = 50 sleeptime = 1E-4 with ut.Timer(): for x in ut.ProgressIter(range(0, numiter), freq=8, adjust=True): time.sleep(sleeptime) print('_________________') print('No frequncy run:') with ut.Timer(): for x in range(0, numiter): time.sleep(sleeptime) print('_________________') numiter = 500 sleeptime = 8E-7 with ut.Timer(): for x in ut.ProgressIter(range(0, numiter), freq=8, adjust=True): time.sleep(sleeptime) print('_________________') with ut.Timer(): for x in ut.ProgressIter(range(0, numiter), freq=200): time.sleep(sleeptime) print('_________________') print('No frequncy run:') with ut.Timer(): for x in range(0, numiter): time.sleep(sleeptime) print('_________________') # Test nested iter progiter1 = ut.ProgressIter(range(0, 10), lbl='prog1', freq=1, adjust=False) for count1 in progiter1: progiter_partials = progiter1.get_subindexers(1) progiter2 = progiter_partials[0](range(0, 7), lbl='sub_prog1', freq=1, adjust=False) for count2 in progiter2: pass for x in ut.ProgressIter(zip(range(10), range(10)), freq=8, adjust=True): time.sleep(sleeptime) #progiter3 = progiter_partials[1](range(0, 3), lbl='sub_prog2', freq=1, adjust=False) #for count3 in progiter3: # pass
[docs]def get_nTotalChunks(nTotal, chunksize): """ Returns the number of chunks that a list will be split into given a chunksize. Args: nTotal (int): chunksize (int): Returns: int: nTotalChunks SeeAlso: util_iter.ichunks CommandLine: python -m utool.util_progress --exec-get_nTotalChunks Example: >>> # DISABLE_DOCTEST >>> from utool.util_progress import * # NOQA >>> nTotal = 2000 >>> chunksize = 256 >>> nTotalChunks = get_nTotalChunks(nTotal, chunksize) >>> result = ('nTotalChunks = %s' % (six.text_type(nTotalChunks),)) >>> print(result) nTotalChunks = 8 """ nTotalChunks = int(math.ceil(nTotal / chunksize)) return nTotalChunks
[docs]def ProgChunks(list_, chunksize, nInput=None, **kwargs): """ Yeilds an iterator in chunks and computes progress Progress version of ut.ichunks """ if nInput is None: nInput = len(list_) nTotalChunks = get_nTotalChunks(nInput, chunksize) kwargs['nTotal'] = nTotalChunks if 'freq' not in kwargs: kwargs['freq'] = 1 chunk_iter = util_iter.ichunks(list_, chunksize) progiter_ = ProgressIter(chunk_iter, **kwargs) return progiter_
[docs]def ProgPartial(*args, **kwargs): return partial(ProgressIter, *args, **kwargs)
[docs]class ProgressIter(object): """ Wraps a for loop with progress reporting lbl='Progress: ', nTotal=0, flushfreq=4, startafter=-1, start=True, repl=False, approx=False, disable=False, writefreq=1, with_time=False, backspace=True, pad_stdout=False, wfreq=None, ffreq=None, freq=None, total=None, num=None, with_totaltime=None Referencs: https://github.com/verigak/progress/blob/master/progress/__init__.py Args: iterable (): iterable normally passed to for loop lbl (str): progress label nTotal (int): flushfreq (int): startafter (int): start (bool): repl (bool): approx (bool): enabled (bool): writefreq (int): with_totaltime (bool): backspace (bool): pad_stdout (bool): autoadjust (bool): no adjusting frequency if True (default False) wfreq (None): alias for write_freq ffreq (None): alias for flush_freq total (None): alias for nTotal num (None): alias for nTotal Timeit:: import utool as ut setup = ut.codeblock( ''' import utool as ut from six.moves import range, zip import time def time_append(size): start_time = time.time() last_time = start_time list2 = [] for x in range(size): now_time = time.time() between = now_time - last_time last_time = now_time list2.append(between) def time_assign(size): start_time = time.time() last_time = start_time list1 = ut.alloc_nones(size) for x in range(size): now_time = time.time() between = now_time - last_time last_time = now_time list1[x] = between def time_baseline(size): start_time = time.time() last_time = start_time for x in range(size): now_time = time.time() between = now_time - last_time last_time = now_time def time_null(size): for x in range(size): pass ''') input_sizes = [2 ** count for count in range(7, 12)] stmt_list = ['time_assign', 'time_append', 'time_baseline', 'time_null'] input_sizes=[100, 1000, 10000] ut.timeit_grid(stmt_list, setup, input_sizes=input_sizes, show=True) CommandLine: python -m utool.util_progress --test-ProgressIter python -m utool.util_progress --test-ProgressIter:0 python -m utool.util_progress --test-ProgressIter:1 python -m utool.util_progress --test-ProgressIter:2 Example: >>> # ENABLE_DOCTEST >>> import utool as ut >>> from six.moves import range >>> num = 1000 >>> num2 = 10001 >>> results1 = [x for x in ut.ProgressIter(range(num), wfreq=10, adjust=True)] >>> results4 = [x for x in ut.ProgressIter(range(num), wfreq=1, adjust=True)] >>> results2 = [x for x in range(num)] >>> results3 = [x for x in ut.progiter((y + 1 for y in range(num2)), >>> nTotal=num2, wfreq=1000, >>> backspace=True, adjust=True)] >>> assert results1 == results2 Example1: >>> # SLOW_DOCTEST >>> import utool as ut >>> from six.moves import range >>> num2 = 10001 >>> progiter = ut.ProgressIter(range(num2), lbl='testing primes', >>> report_unit='seconds', freq=1, >>> time_thresh=.1, adjust=True) >>> results1 = [ut.get_nth_prime_bruteforce(29) for x in progiter] >>> print(ut.truncate_str(ut.repr2(results1))) Example2: >>> # SLOW_DOCTEST >>> import utool as ut >>> from six.moves import range >>> num2 = 100001 >>> progiter = ut.ProgressIter(range(num2), lbl='testing primes', >>> report_unit='seconds', freq=1, >>> time_thresh=3, adjust=True) >>> results1 = [ut.get_nth_prime_bruteforce(29) for x in progiter] >>> print(ut.truncate_str(ut.repr2(results1))) """ def __init__(self, iterable=None, *args, **kwargs): self.iterable = iterable if len(args) < 2 and 'nTotal' not in kwargs: try: nTotal = len(iterable) kwargs['nTotal'] = nTotal except Exception: pass self.use_rate = kwargs.pop('use_rate', True) self.lbl = kwargs.get('lbl', 'lbl') self.nTotal = kwargs.get('nTotal', 0) self.backspace = kwargs.get('backspace', True) self.freq = kwargs.get('freq', 1) self.invert_rate = kwargs.get('invert_rate', False) #self.report_unit = kwargs.get('report_unit', 'minutes') self.enabled = kwargs.get('enabled', True) self.report_unit = kwargs.get('report_unit', 'seconds') # autoadjust frequency of reporting self.autoadjust = kwargs.get('autoadjust', kwargs.get('adjust', False)) self.time_thresh = kwargs.pop('time_thresh', None) self.prog_hook = kwargs.pop('prog_hook', None) self.separate = kwargs.pop('separate', False) # FIXME: get these subinder things working # ~/code/guitool/guitool/guitool_components.py self.substep_min = kwargs.pop('substep_min', 0) self.substep_size = kwargs.pop('substep_size', 1) self.level = kwargs.pop('level', 0) self.parent_index = kwargs.pop('parent_index', 0) self.parent_nTotal = kwargs.pop('parent_nTotal', 1) self.parent_offset = self.parent_index * self.nTotal #self.start_offset = self.substep_min if FORCE_ALL_PROGRESS: self.freq = 1 self.autoadjust = False if self.prog_hook is not None: self.prog_hook.register_progiter(self) #self.time_thresh_growth = kwargs.pop('time_thresh_growth', 1.0) self.time_thresh_growth = kwargs.pop('time_thresh_growth', 1.0) self.with_totaltime = False if self.freq is None: self.freq = 1 if self.use_rate: # Hacky so hacky. this needs major cleanup # saving args and kwargs so can wait on log_progress call # not sure where it is called and dont want to break things self.args = args self.kwargs = kwargs self.mark = None self.end = None else: self.mark, self.end = log_progress(*args, **kwargs) self.count = 0 def __call__(self, iterable): self.iterable = iterable return self def __iter__(self): if not self.enabled: return iter(self.iterable) if NO_PROGRESS: # IF PROGRESS IS TURNED OFF msg = 'Iterating ' + self.lbl + ' with no progress' if not util_arg.QUIET: print(msg) #with ut.Timer(msg): return iter(self.iterable) else: if self.use_rate: # STANDARD CALL CASE return self.iter_rate() else: return self.iter_without_rate()
[docs] def get_subindexers(prog_iter, num_substeps): # FIXME and make this a method of progiter step_min = (((prog_iter.count - 1) / prog_iter.nTotal) * prog_iter.substep_size + prog_iter.substep_min) step_size = (1.0 / prog_iter.nTotal) * prog_iter.substep_size substep_size = step_size / num_substeps substep_min_list = [(step * substep_size) + step_min for step in range(num_substeps)] #level = prog_iter.level + 1 DEBUG = False if DEBUG: with ut.Indenter(' ' * 4 * prog_iter.level): print('\n') print('+____<NEW SUBSTEPS>____') print('Making %d substeps for prog_iter.lbl = %s' % ( num_substeps, prog_iter.lbl,)) print(' * step_min = %.2f' % (step_min,)) print(' * step_size = %.2f' % (step_size,)) print(' * substep_size = %.2f' % (substep_size,)) print(' * substep_min_list = %r' % (substep_min_list,)) print(r'L____</NEW SUBSTEPS>____') print('\n') subprog_partial_list = [ partial(ProgressIter, parent_nTotal=prog_iter.nTotal * num_substeps, parent_index=(prog_iter.count - 1) + (prog_iter.nTotal * step)) for step in range(num_substeps)] return subprog_partial_list #def build_msg_fmtstr_time(self, lbl, invert_rate, backspace): # with_wall = True # tzname = time.tzname[0] # if util_cplat.WIN32: # tzname = tzname.replace('Eastern Standard Time', 'EST') # msg_fmtstr_time = ''.join(( # 'rate=%3.3f seconds/iter, ' if invert_rate else 'rate=%4.2f Hz,', # ' etr=%s,', # ' ellapsed=%s,', # ' wall=%s ' + tzname if with_wall else '', # #'' if backspace else '\n', # '\n' if backspace else '', # )) # return msg_fmtstr_time #def build_msg_fmtstr(self, nTotal, lbl, invert_rate, backspace): # msg_fmtstr = (self.build_msg_fmtstr_index(nTotal, lbl) + # self.build_msg_fmtstr_time(lbl, invert_rate, backspace)) # return msg_fmtstr
@staticmethod
[docs] def build_msg_fmtstr_head_cols(nTotal, lbl): nTotal_ = '?' if nTotal == 0 else six.text_type(nTotal) msg_head_columns = ['\r', lbl, ' {count:4d}/', nTotal_ , '... '] return msg_head_columns
@staticmethod
[docs] def build_msg_fmtstr2(lbl, nTotal, invert_rate, backspace): r""" Args: lbl (str): invert_rate (bool): backspace (bool): Returns: str: msg_fmtstr_time CommandLine: python -m utool.util_progress --exec-ProgressIter.build_msg_fmtstr2 Setup: >>> from utool.util_progress import * # NOQA >>> lbl = 'foo' >>> invert_rate = True >>> backspace = False >>> nTotal = None Example: >>> # DISABLE_DOCTEST >>> msg_fmtstr_time = ProgressIter.build_msg_fmtstr2(lbl, nTotal, invert_rate, backspace) >>> result = ('%s' % (ut.repr2(msg_fmtstr_time),)) >>> print(result) """ with_wall = True tzname = time.tzname[0] if util_cplat.WIN32: tzname = tzname.replace('Eastern Standard Time', 'EST') msg_head = ProgressIter.build_msg_fmtstr_head_cols(nTotal, lbl) msg_tail = [ ('rate={rate:3.3f} seconds/iter, ' if invert_rate else 'rate={rate:4.2f} Hz,'), ('' if nTotal == 0 else ' etr={etr},'), ' ellapsed={ellapsed},', (' wall={wall} ' + tzname if with_wall else ''), #'' if backspace else '\n', '\n' if backspace else '', ] msg_fmtstr_time = ''.join((msg_head + msg_tail)) return msg_fmtstr_time
@profile
[docs] def iter_rate(self): """ pun not intended # TODO: record iteration times for analysis # TODO Incorporate this better # FIXME; pad_stdout into subfunctions import dis dis.dis(ut.ProgressIter.iter_rate) """ #class IterState(object): # def __init__(state): # state.freq = 1 # state.freq = 1 # pass adjust = self.autoadjust # SETUP VARIABLES # HACK: reaquire logging print funcs in case they have changed PROGRESS_WRITE = util_logging.__UTOOL_WRITE__ PROGRESS_FLUSH = util_logging.__UTOOL_FLUSH__ nTotal = self.nTotal * self.parent_nTotal # hack freq = self.freq self.count = 0 between_count = 0 last_count = 0 # how long iterations should be before a flush # (used for freq adjustment) time_thresh = (self._get_timethresh_heuristics() if self.time_thresh is None else self.time_thresh) time_thresh_growth = self.time_thresh_growth if time_thresh_growth > 1: # time_thresh_growth is specified for very long processes # print out the starting timestamp in that case timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + time.tzname[0] print('Start progress lbl= %s at %s' % (self.lbl, timestamp,)) #time_thresh = 0.5 max_between_time = -1.0 max_between_count = -1.0 # why is this different? # because frequency varies # TODO: should be kept as a statistic that uses the max time from a # list of iterations divided by the size of that list that will account # for buffering issues iters_per_second = -1 #est_timeunit_left = -1 # Write initial message force_newlines = not self.backspace print_sep = self.separate if print_sep: print('---------') PROGRESS_WRITE( ''.join(self.build_msg_fmtstr_head_cols( nTotal, self.lbl)).format(count=self.parent_offset)) #PROGRESS_WRITE(self.build_msg_fmtstr_index(nTotal, self.lbl) % (self.parent_offset)) if force_newlines: PROGRESS_WRITE('\n') PROGRESS_FLUSH() if self.prog_hook is not None: self.prog_hook(self.count, nTotal) # Prepare for iteration msg_fmtstr = self.build_msg_fmtstr2(self.lbl, nTotal, self.invert_rate, self.backspace) #msg_fmtstr = self.build_msg_fmtstr(nTotal, self.lbl, # self.invert_rate, self.backspace) # TODO: on windows is time.clock better? # http://exnumerus.blogspot.com/2011/02/how-to-quickly-plot-multiple-line.html start_time = default_timer() last_time = start_time start = 1 + self.parent_offset # Wrap the for loop with a generator for self.count, item in enumerate(self.iterable, start=start): # GENERATE yield item # DO PROGRESS INFO if (self.count) % freq == 0: # UPDATE INFO now_time = default_timer() between_time = (now_time - last_time) between_count = self.count - last_count total_seconds = (now_time - start_time) iters_per_second = between_count / (float(between_time) + 1E-9) # If the future is known if nTotal is None: est_seconds_left = -1 else: iters_left = nTotal - self.count est_seconds_left = iters_left / (iters_per_second + 1E-9) # /future last_count = self.count last_time = now_time # ADJUST FREQ IF NEEDED # Adjust frequency if printing too quickly # so progress doesnt slow down actual function # TODO: better adjust algorithm time_thresh *= time_thresh_growth if adjust and (between_time < time_thresh or between_time > time_thresh * 2.0): max_between_time = max(max(max_between_time, between_time), 1E-9) max_between_count = max(max_between_count, between_count) # If progress was uniform and all time estimates were # perfect this would be the new freq to achieve time_thresh new_freq = max(int(time_thresh * max_between_count / max_between_time), 1) if DEBUG_FREQ_ADJUST: print('\n+---') print('[prog] between_count = %r' % between_count) print('[prog] between_time = %.8r' % between_time) print('[prog] time_thresh = %r' % time_thresh) print('[prog] max_between_count = %r' % max_between_count) print('[prog] max_between_time = %.8r' % max_between_time) print('[prog] Adusting frequency from: %r' % freq) print('[prog] Adusting frequency to: %r' % new_freq) print('L___') # But things are not perfect. So, don't make drastic changes max_freq_change_up = max(256, freq * 2) max_freq_change_down = freq // 2 if (new_freq - freq) > max_freq_change_up: freq += max_freq_change_up elif (freq - new_freq) > max_freq_change_down: freq -= max_freq_change_down else: freq = new_freq #msg = msg_fmtstr % ( # self.count, # 1.0 / iters_per_second if self.invert_rate else iters_per_second, # six.text_type(datetime.timedelta(seconds=int(est_seconds_left))), # six.text_type(datetime.timedelta(seconds=int(total_seconds))), # time.strftime('%H:%M'), #) msg = msg_fmtstr.format( count=self.count, rate=1.0 / iters_per_second if self.invert_rate else iters_per_second, etr=six.text_type(datetime.timedelta(seconds=int(est_seconds_left))), ellapsed=six.text_type(datetime.timedelta(seconds=int(total_seconds))), wall=time.strftime('%H:%M'), ) #est_timeunit_left, #total_timeunit) if print_sep: print('---------') PROGRESS_WRITE(msg) if force_newlines: PROGRESS_WRITE('\n') PROGRESS_FLUSH() if self.prog_hook is not None: self.prog_hook(self.count, nTotal) if (self.count) % freq != 0: # If the final line of progress was not written in the loop, write it here est_seconds_left = 0 now_time = default_timer() total_seconds = (now_time - start_time) #msg = msg_fmtstr % ( # self.count, 1.0 / iters_per_second if self.invert_rate else iters_per_second, # six.text_type(datetime.timedelta(seconds=int(est_seconds_left))), # six.text_type(datetime.timedelta(seconds=int(total_seconds))), # time.strftime('%H:%M'), #) msg = msg_fmtstr.format( count=self.count, rate=1.0 / iters_per_second if self.invert_rate else iters_per_second, etr=six.text_type(datetime.timedelta(seconds=int(est_seconds_left))), ellapsed=six.text_type(datetime.timedelta(seconds=int(total_seconds))), wall=time.strftime('%H:%M'), ) PROGRESS_WRITE(msg) PROGRESS_WRITE('\n') PROGRESS_FLUSH() #cumrate = 1E-9 #self.nTotal = len(self.iterable) # for: # if: # if: # print('') # freq = max(int(1.3 * between_count * time_thresh / between_time), 1) # # There has to be a standard way to do this. # # Refer to: https://github.com/verigak/progress/blob/master/progress/__init__.py # freq = max(int((between_count * between_time) / time_thresh), 1) # freq = max(int((between_count) / time_thresh), 1) # freq = max(int((between_time) / time_thresh), 1) # freq = max(int(time_thresh / between_count), 1) # print('[prog] Adusting frequency to: %r' % freq) # print('') # cumrate += between_time # rate = (self.count + 1.0) / float(cumrate) # if False and __debug__: # print('<!!!!!!!!!!!!!>') # print('iters_left = %r' % iters_left) # print('between_time = %r' % between_time) # print('between_count = %r' % between_count) # print('est_seconds_left = %r' % est_seconds_left) # print('iters_per_second = %r' % iters_per_second) # print('</!!!!!!!!!!!!!>') #PROGRESS_WRITE('\n') #msg = msg_fmtstr % (self.count + 1, iters_per_second, est_timeunit_left) #print('freq = %r' % freq) #self.end(self.count + 1) #def make_substep_progiters():
[docs] def iter_without_rate(self): if self.mark is None: # Continuation of hacking self.mark, self.end = log_progress(*self.args, **self.kwargs) mark = self.mark # Wrap the for loop with a generator self.count = -1 for self.count, item in enumerate(self.iterable): mark(self.count) yield item self.end(self.count + 1)
[docs] def mark_current(self): self.mark(self.count)
def _get_timethresh_heuristics(self): """ resonably decent hueristics for how much time to wait before updating progress. """ if self.nTotal > 1E5: time_thresh = 2.5 elif self.nTotal > 1E4: time_thresh = 2.0 elif self.nTotal > 1E3: time_thresh = 1.0 else: time_thresh = 0.5 return time_thresh
progiter = ProgressIter
[docs]class ProgIter(ProgressIter): # Thin wrapper with better arg positions def __init__(self, iterable, lbl='Prog', adjust=True, freq=1, **kwargs): super(ProgIter, self).__init__(iterable, lbl=lbl, adjust=adjust, freq=freq, **kwargs)
[docs]def progress_str(max_val, lbl='Progress: ', repl=False, approx=False, backspace=PROGGRESS_BACKSPACE): r""" makes format string that prints progress: %Xd/MAX_VAL with backspaces NOTE: \r can be used instead of backspaces. This function is not very relevant because of that. """ # string that displays max value max_str = six.text_type(max_val) if approx: # denote approximate maximum max_str = '~' + max_str dnumstr = six.text_type(len(max_str)) # string that displays current progress cur_str = '%' + dnumstr + 'd' # If user passed in the label if repl: _fmt_str = lbl.replace('<cur_str>', cur_str).replace('<max_str>', max_str) else: _fmt_str = lbl + cur_str + '/' + max_str if backspace: # put backspace characters into the progress string # (looks nice on normal terminals) #nBackspaces = len(_fmt_str) - len(dnumstr) + len(max_str) #backspaces = '\b' * nBackspaces #fmt_str = backspaces + _fmt_str # FIXME: USE CARAGE RETURN INSTEAD OF BACKSPACES fmt_str = '\r' + _fmt_str else: # FIXME: USE CARAGE RETURN INSTEAD OF BACKSPACES # this looks better on terminals without backspaces fmt_str = _fmt_str + '\n' return fmt_str
[docs]def log_progress(lbl='Progress: ', nTotal=0, flushfreq=4, startafter=-1, start=True, repl=False, approx=False, disable=False, writefreq=1, with_time=False, backspace=True, pad_stdout=False, wfreq=None, ffreq=None, freq=None, total=None, num=None, with_totaltime=None): """ FIXME: depricate for ProgressIter. Still used in smk funcs Returns two functions (mark_progress, end_progress) which will handle logging progress in a for loop. flush frequency must be a multiple of write frequency Args: lbl (str): progress label nTotal (int): flushfreq (int): startafter (int): start (bool): repl (bool): approx (bool): disable (bool): writefreq (int): with_totaltime (bool): backspace (bool): pad_stdout (bool): wfreq (None): alias for write_freq ffreq (None): alias for flush_freq freq (None): alias for flush_freq and write_freq (prefered) total (None): alias for nTotal num (None): alias for nTotal Example: >>> import utool, time >>> from six.moves import range >>> # Define a dummy task >>> spam = 42.0 >>> nTotal = 1000 >>> iter_ = (num for num in range(0, nTotal * 2, 2)) >>> # Create progress functions ... mark_, end_ = utool.log_progress('prog ', nTotal, flushfreq=17) \b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bprog 0/1000 >>> for count, item in enumerate(iter_): #doctest: +ELLIPSIS ... # Call with enumerate to keep track of a count variable ... time.sleep(.001) ... spam += item + count ... mark_(count) \b...prog 1000/1000 >>> # Mark completion >>> end_() <BLANKLINE> """ # utool.auto_docstr('utool.util_progress', 'log_progress') # python -c "import utool; utool.print_auto_docstr('utool.util_progress', 'log_progress')" # # In reference to above docstr: # I don't completely understand why some of the >>> and ... had to be where # they are, but doctest gets very angry if its not in this format # TODO: Option to display rate of progress # TODO: Option to display estimated time remaining global AGGROFLUSH # Alias kwargs with simpler names if num is not None: nTotal = num if total is not None: nTotal = total if wfreq is not None: writefreq = wfreq if ffreq is not None: flushfreq = ffreq if freq is not None: writefreq = flushfreq = freq if with_totaltime is not None: with_time = with_totaltime # flush frequency must be a multiple of write frequency flushfreq = max(int(round(flushfreq / writefreq)), 1) * writefreq if nTotal < startafter or disable: # Do not mark progress if only executing a small number of tasks def mark_progress(*args): pass def end_progress(*args): pass return mark_progress, end_progress else: write_fn = PROGRESS_WRITE flush_fn = PROGRESS_FLUSH # build format string for displaying progress fmt_str = progress_str(nTotal, lbl=lbl, repl=repl, approx=approx, backspace=backspace) if AGGROFLUSH: # Progress function which automatically flushes def mark_progress(count, flush_fn=flush_fn): count_ = count + 1 write_fn(fmt_str % (count_)) flush_fn() else: # Progress function flushes every <flushfreq> times def mark_progress(count, fmt_str=fmt_str, flushfreq=flushfreq, writefreq=writefreq, write_fn=write_fn, flush_fn=flush_fn): count_ = count + 1 if count_ % writefreq == 0: write_fn(fmt_str % count_) if count_ % flushfreq == 0: flush_fn() if pad_stdout: write_fn('\n') write_fn('\n') flush_fn() if with_time: tt = util_time.tic(lbl) def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn): write_fn(fmt_str % (count_)) write_fn('\n') flush_fn() if with_time: util_time.toc(tt) if pad_stdout: write_fn('\n\n') flush_fn() #mark_progress(0) if start: mark_progress(-1) return mark_progress, end_progress
[docs]def simple_progres_func(verbosity, msg, progchar='.'): def mark_progress0(*args): pass def mark_progress1(*args): PROGRESS_WRITE(progchar) def mark_progress2(*args): print(msg % args) if verbosity == 0: mark_progress = mark_progress0 elif verbosity == 1: mark_progress = mark_progress1 elif verbosity == 2: mark_progress = mark_progress2 return mark_progress #def prog_func(*args, **kwargs): # return progress_func(*args, **kwargs) ## TODO: Return start_prog, make_prog, end_prog #def progress_func(max_val=0, lbl='Progress: ', mark_after=-1, # flush_after=4, spacing=0, line_len=80, # progress_type='fmtstr', mark_start=False, repl=False, # approx=False, override_quiet=False): # """ DEPRICATE # Returns: # a function that marks progress taking the iteration count as a # parameter. Prints if max_val > mark_at. Prints dots if max_val not # specified or simple=True # """ # write_fn = PROGRESS_WRITE # #write_fn = print_ # #print('STARTING PROGRESS: VERBOSE=%r QUIET=%r' % (VERBOSE, QUIET)) # # Tell the user we are about to make progress # if SILENT or (QUIET and not override_quiet) or (progress_type in ['simple', 'fmtstr'] and max_val < mark_after): # return lambda count: None, lambda: None # # none: nothing # if progress_type == 'none': # def mark_progress_None(count): # return None # mark_progress = mark_progress_None # # simple: one dot per progress. no flush. # if progress_type == 'simple': # def mark_progress_simple(count): # write_fn('.') # mark_progress = mark_progress_simple # # dots: spaced dots # if progress_type == 'dots': # indent_ = ' ' # write_fn(indent_) # if spacing > 0: # # With spacing # newline_len = spacing * line_len // spacing # def mark_progress_sdot(count): # write_fn('.') # count_ = count + 1 # if (count_) % newline_len == 0: # write_fn('\n' + indent_) # PROGRESS_FLUSH() # elif (count_) % spacing == 0: # write_fn(' ') # PROGRESS_FLUSH() # elif (count_) % flush_after == 0: # PROGRESS_FLUSH() # mark_progress = mark_progress_sdot # else: # # No spacing # newline_len = line_len # def mark_progress_dot(count): # write_fn('.') # count_ = count + 1 # if (count_) % newline_len == 0: # write_fn('\n' + indent_) # PROGRESS_FLUSH() # elif (count_) % flush_after == 0: # PROGRESS_FLUSH() # mark_progress = mark_progress_dot # # fmtstr: formated string progress # if progress_type == 'fmtstr': # fmt_str = progress_str(max_val, lbl=lbl, repl=repl, approx=approx) # def mark_progress_fmtstr(count): # count_ = count + 1 # write_fn(fmt_str % (count_)) # if (count_) % flush_after == 0: # PROGRESS_FLUSH() # mark_progress = mark_progress_fmtstr # # FIXME idk why argparse2.ARGS_ is none here. # if '--aggroflush' in sys.argv: # def mark_progress_agressive(count): # mark_progress(count) # PROGRESS_FLUSH() # return mark_progress_agressive # def end_progress(): # write_fn('\n') # PROGRESS_FLUSH() # #mark_progress(0) # if mark_start: # mark_progress(-1) # return mark_progress, end_progress # raise Exception('unkown progress type = %r' % progress_type)
if __name__ == '__main__': """ CommandLine: python -c "import utool, utool.util_progress; utool.doctest_funcs(utool.util_progress, allexamples=True)" python -c "import utool, utool.util_progress; utool.doctest_funcs(utool.util_progress)" python -m utool.util_progress python -m utool.util_progress --allexamples """ import multiprocessing multiprocessing.freeze_support() # for win32 import utool as ut # NOQA ut.doctest_funcs()