Source code for utool.util_resources

# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from utool import util_inject
from utool import util_str
print, rrr, profile = util_inject.inject2(__name__, '[resource]')

try:
    # Resource does not exist in win32
    import resource

    def time_in_usermode():
        stime = resource.getrusage(resource.RUSAGE_SELF).ru_stime
        return stime

    def time_in_systemmode():
        utime = resource.getrusage(resource.RUSAGE_SELF).ru_utime
        return utime

    def peak_memory():
        """Returns the resident set size (the portion of
        a process's memory that is held in RAM.)
        """
        # MAXRSS is expressed in kilobytes. Convert to bytes
        # FIXME: MAXRSS is NOT expressed in kilobytes. use resource.getpagesize to Convert to bytes
        # References: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
        #resource.getpagesize
        maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
        return maxrss

    def get_resource_limits():
        #rlimit_keys = [key for key in six.iterkeys(resource.__dict__) if key.startswith('RLIMIT_')]
        #print('\n'.join(['(\'%s\', resource.%s),' % (key.replace('RLIMIT_', ''), key) for key in rlimit_keys]))
        rlim_keytups = [
            ('MEMLOCK', resource.RLIMIT_MEMLOCK),
            ('NOFILE', resource.RLIMIT_NOFILE),
            ('CPU', resource.RLIMIT_CPU),
            ('DATA', resource.RLIMIT_DATA),
            ('OFILE', resource.RLIMIT_OFILE),
            ('STACK', resource.RLIMIT_STACK),
            ('FSIZE', resource.RLIMIT_FSIZE),
            ('CORE', resource.RLIMIT_CORE),
            ('NPROC', resource.RLIMIT_NPROC),
            ('AS', resource.RLIMIT_AS),
            ('RSS', resource.RLIMIT_RSS),
        ]
        rlim_valtups = [(lbl, resource.getrlimit(rlim_key)) for (lbl, rlim_key) in rlim_keytups]
        def rlimval_str(rlim_val):
            soft, hard = rlim_val
            softstr = util_str.byte_str2(soft) if soft != -1 else 'None'
            hardstr = util_str.byte_str2(hard) if hard != -1 else 'None'
            return '%12s, %12s' % (softstr, hardstr)
        rlim_strs = ['%8s: %s' % (lbl, rlimval_str(rlim_val)) for (lbl, rlim_val) in rlim_valtups]
        print('Resource Limits: ')
        print('%8s  %12s  %12s' % ('id', 'soft', 'hard'))
        print('\n'.join(rlim_strs))
        return rlim_strs

    #def rusage_flags():
        #0	ru_utime	time in user mode (float)
        #1	ru_stime	time in system mode (float)
        #2	ru_maxrss	maximum resident set size
        #3	ru_ixrss	shared memory size
        #4	ru_idrss	unshared memory size
        #5	ru_isrss	unshared stack size
        #6	ru_minflt	page faults not requiring I/O
        #7	ru_majflt	page faults requiring I/O
        #8	ru_nswap	number of swap outs
        #9	ru_inblock	block input operations
        #10	ru_oublock	block output operations
        #11	ru_msgsnd	messages sent
        #12	ru_msgrcv	messages received
        #13	ru_nsignals	signals received
        #14	ru_nvcsw	voluntary context switches
        #15	ru_nivcsw	involuntary context switches
except ImportError:
[docs] def time_in_usermode(): raise NotImplementedError('unavailable in win32')
[docs] def time_in_systemmode(): raise NotImplementedError('unavailable in win32')
[docs] def peak_memory(): """Returns the resident set size (the portion of a process's memory that is held in RAM.) """ raise NotImplementedError('unavailable in win32')
[docs] def get_resource_limits(): raise NotImplementedError('unavailable in win32')
[docs]def time_str2(seconds): return '%.2f sec' % (seconds,)
[docs]def get_resource_usage_str(): usage_str_list = [ ('+______________________'), ('| RESOURCE_USAGE process.get_memory_info()[0] / float(2 ** 20) '), ('| * current_memory = %s' % util_str.byte_str2(current_memory_usage())), ] try: usage_str_list.extend([ ('| * peak_memory = %s' % util_str.byte_str2(peak_memory())), ('| * user_time = %s' % time_str2(time_in_usermode())), ('| * system_time = %s' % time_str2(time_in_systemmode())), ]) except Exception: pass usage_str_list.append('L______________________') usage_str = '\n'.join(usage_str_list) return usage_str
[docs]def current_memory_usage(): """ Returns this programs current memory usage in bytes """ import psutil proc = psutil.Process(os.getpid()) #meminfo = proc.get_memory_info() meminfo = proc.memory_info() rss = meminfo[0] # Resident Set Size / Mem Usage vms = meminfo[1] # Virtual Memory Size / VM Size # NOQA return rss
[docs]def get_matching_process_ids(cmd_pattern, user_pattern): """ CommandLine: export PID=30196 export PID=$(python -c "import utool as ut; print(ut.get_matching_process_ids('jonc', 'python2.7'))") export PID=$(python -c "import utool as ut; print(ut.get_matching_process_ids('jonc', 'matlab'))") sudo -H echo $PID ps -o pid,comm,nice -p $PID renice 10 -p $PID sudo renice -4 -p $PID user_pattern = 'jonc' cmd_pattern = 'main.py' user_pattern = None cmd_pattern = 'matlab' get_matching_process_ids(cmd_pattern, user_pattern) """ import psutil import re process_list = list(psutil.process_iter()) def matches_pattern(proc, user_pattern, cmd_pattern): matches_user = True if user_pattern is None else re.match(user_pattern, proc.username()) cmdline_str = ' '.join(proc.cmdline()) matches_name = True if cmd_pattern is None else re.search(cmd_pattern, cmdline_str) return matches_user and matches_name filtered_proc_list = [proc for proc in process_list if matches_pattern(proc, user_pattern, cmd_pattern)] for proc in filtered_proc_list: print(' | '.join([str(proc.username()), str(proc.nice()), str(proc), ' '.join(proc.cmdline())])) #print(proc.cmdline()) #print(proc.pid) #print('---') important_process_list = [proc for proc in process_list if proc.nice() < -4] for proc in important_process_list: print(' -- '.join([str(proc.username()), str(proc.nice()), str(proc), ' '.join(proc.cmdline())])) #for proc in filtered_proc_list: # print('---') # print(proc) # print(proc.cmdline()) # print(proc.nice()) # print(proc.pid) filtered_pid_list = [proc.pid for proc in filtered_proc_list] return filtered_pid_list
[docs]def num_cpus(): import psutil return psutil.NUM_CPUS
[docs]def available_memory(): """ Returns total system wide available memory in bytes """ import psutil return psutil.virtual_memory().available
[docs]def total_memory(): """ Returns total system wide memory in bytes """ import psutil return psutil.virtual_memory().total
[docs]def used_memory(): """ Returns total system wide used memory in bytes """ return total_memory() - available_memory()
[docs]def memstats(): print(get_memstats_str())
[docs]def get_memstats_str(): return '\n'.join([ ('[psutil] total = %s' % util_str.byte_str2(total_memory())), ('[psutil] available = %s' % util_str.byte_str2(available_memory())), ('[psutil] used = %s' % util_str.byte_str2(used_memory())), ('[psutil] current = %s' % util_str.byte_str2(current_memory_usage())), ])
[docs]def get_python_datastructure_sizes(): """ References: http://stackoverflow.com/questions/1331471/in-memory-size-of-python-stucture CommandLine: python -m utool.util_resources --test-get_python_datastructure_sizes Example: >>> # ENABLE_DOCTEST >>> from utool.util_resources import * # NOQA >>> import utool as ut # NOQA >>> type_sizes = get_python_datastructure_sizes() >>> result = ut.dict_str(type_sizes, sorted_=True) >>> print(result) """ import sys import decimal import six empty_types = { 'int' : 0, 'float' : 0.0, 'dict' : dict(), 'set' : set(), 'tuple' : tuple(), 'list' : list(), 'str' : '', 'unicode' : u'', 'decimal' : decimal.Decimal(0), 'object' : object(), } type_sizes = {key: sys.getsizeof(val) for key, val in six.iteritems(empty_types)} return type_sizes #psutil.virtual_memory() #psutil.swap_memory() #psutil.disk_partitions() #psutil.disk_usage("/") #psutil.disk_io_counters() #psutil.net_io_counters(pernic=True) #psutil.get_users() #psutil.get_boot_time() #psutil.get_pid_list()
if __name__ == '__main__': """ CommandLine: python -m utool.util_resources python -m utool.util_resources --allexamples python -m utool.util_resources --allexamples --noface --nosrc """ import multiprocessing multiprocessing.freeze_support() # for win32 import utool as ut # NOQA ut.doctest_funcs()