# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import operator
import six
import itertools
import functools
from six.moves import zip, map, zip_longest, range, filter, reduce
from utool import util_iter
from utool import util_inject
from utool import util_str
from utool import util_type
from utool._internal.meta_util_six import get_funcname, set_funcname
print, rrr, profile = util_inject.inject2(__name__, '[list]')
if util_type.HAVE_NUMPY:
import numpy as np
# --- List Allocations ---
[docs]def lmap(func, iter_, **kwargs):
"""
list map - eagerly evaulates map like in python2
(but you aren't using that right?)
"""
return [func(arg, **kwargs) for arg in iter_]
# return list(map(func, iter_))
[docs]def maplen(iter_):
return list(map(len, iter_))
[docs]def replace_nones(list_, repl=-1):
r"""
Recursively removes Nones in all lists and sublists and replaces them with
the repl variable
Args:
list_ (list):
repl (obj): replacement value
Returns:
list
CommandLine:
python -m utool.util_list --test-replace_nones
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> # build test data
>>> list_ = [None, 0, 1, 2]
>>> repl = -1
>>> # execute function
>>> repl_list = replace_nones(list_, repl)
>>> # verify results
>>> result = str(repl_list)
>>> print(result)
[-1, 0, 1, 2]
"""
repl_list = [
repl if item is None else (
replace_nones(item, repl) if isinstance(item, list) else item
)
for item in list_
]
return repl_list
[docs]def recursive_replace(list_, target, repl=-1):
r"""
Recursively removes target in all lists and sublists and replaces them with
the repl variable
"""
repl_list = [
recursive_replace(item, target, repl) if isinstance(item, (list, np.ndarray)) else
(repl if item == target else item)
for item in list_
]
return repl_list
[docs]def list_replace(list_, target, repl):
r"""
alias
Recursively removes target in all lists and sublists and replaces them with
the repl variable
"""
return recursive_replace(list_, target, repl)
[docs]def alloc_lists(num_alloc):
""" allocates space for a ``list`` of lists """
return [[] for _ in range(num_alloc)]
[docs]def alloc_nones(num_alloc):
""" allocates space for a ``list`` of Nones """
return [None] * num_alloc
#return [None for _ in range(num_alloc)]
[docs]def ensure_list_size(list_, size_):
""" Allocates more space if needbe.
Ensures len(``list_``) == ``size_``.
Args:
list_ (list): ``list`` to extend
size_ (int): amount to exent by
"""
lendiff = (size_) - len(list_)
if lendiff > 0:
extension = [None for _ in range(lendiff)]
list_.extend(extension)
# --- List Searching --- #
[docs]def get_list_column_slice(list_, start=None, stop=None, stride=None):
return list(util_iter.iget_list_column_slice(list_, start, stop, stride))
[docs]def get_list_column(list_, colx):
r"""
accepts a list of (indexables) and returns a list of indexables
can also return a list of list of indexables if colx is a list
Args:
list_ (list): list of lists
colx (int or list): index or key in each sublist get item
Returns:
list: list of selected items
CommandLine:
python -m utool.util_list --test-get_list_column
Example0:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [['a', 'b'], ['c', 'd']]
>>> colx = 0
>>> result = get_list_column(list_, colx)
>>> import utool as ut
>>> result = ut.list_str(result, nl=False)
>>> print(result)
['a', 'c']
Example1:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [['a', 'b'], ['c', 'd']]
>>> colx = [1, 0]
>>> result = get_list_column(list_, colx)
>>> import utool as ut
>>> result = ut.list_str(result, nl=False)
>>> print(result)
[['b', 'a'], ['d', 'c']]
Example2:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [{'spam': 'EGGS', 'ham': 'SPAM'}, {'spam': 'JAM', 'ham': 'PRAM'},]
>>> # colx can be a key or list of keys as well
>>> colx = ['spam']
>>> result = get_list_column(list_, colx)
>>> import utool as ut
>>> result = ut.list_str(result, nl=False)
>>> print(result)
[['EGGS'], ['JAM']]
"""
return list(util_iter.iget_list_column(list_, colx))
#if isinstance(colx, list):
# # multi select
# return [[row[colx_] for colx_ in colx] for row in list_]
#else:
# return [row[colx] for row in list_]
take_column = get_list_column
#def get_list_row(list_, rowx):
# return list_[rowx]
[docs]def safe_listget(list_, index, default='?'):
""" depricate """
if index >= len(list_):
return default
ret = list_[index]
if ret is None:
return default
return ret
[docs]def listclip(list_, num, fromback=False):
r"""
Args:
list_ (list):
num (int):
Returns:
sublist:
CommandLine:
python -m utool.util_list --test-listclip
Example1:
>>> # ENABLE_DOCTEST
>>> import utool as ut
>>> # build test data
>>> list_ = [1, 2, 3, 4, 5]
>>> result_list = []
>>> # execute function
>>> num = 3
>>> result_list += [ut.listclip(list_, num)]
>>> num = 9
>>> result_list += [ut.listclip(list_, num)]
>>> # verify results
>>> result = ut.list_str(result_list)
>>> print(result)
[
[1, 2, 3],
[1, 2, 3, 4, 5],
]
Example2:
>>> # ENABLE_DOCTEST
>>> import utool as ut
>>> # build test data
>>> list_ = [1, 2, 3, 4, 5]
>>> result_list = []
>>> # execute function
>>> num = 3
>>> result = ut.listclip(list_, num, fromback=True)
>>> print(result)
[3, 4, 5]
"""
num_ = min(len(list_), num)
if fromback:
sublist = list_[-num_:]
else:
sublist = list_[:num_]
return sublist
[docs]def find_list_indexes(list_, item_list):
"""
Args:
list_ (list): list of items to be searched
item_list (list): list of items to find
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = ['a', 'b', 'c']
>>> item_list = ['d', 'c', 'b', 'f']
>>> index_list = find_list_indexes(list_, item_list)
>>> result = ('index_list = %r' % (index_list,))
>>> print(result)
index_list = [None, 2, 1, None]
"""
index_list = [listfind(list_, item) for item in item_list]
return index_list
[docs]def listfind(list_, tofind):
r"""
get the position of item ``tofind`` in ``list_`` if it exists
otherwise returns None
Args:
list_ (?):
tofind (?):
Returns:
int: index of ``tofind`` in ``list_``
Example1:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = ['a', 'b', 'c']
>>> tofind = 'b'
>>> result = listfind(list_, tofind)
>>> print(result)
1
"""
try:
return list_.index(tofind)
except ValueError:
return None
[docs]def search_list(text_list, pattern, flags=0):
"""
CommandLine:
python -m utool.util_list --test-search_list
Example:
>>> # ENABLE_DOCTEST
>>> import utool as ut
>>> text_list = ['ham', 'jam', 'eggs', 'spam']
>>> pattern = '.am'
>>> flags = 0
>>> (valid_index_list, valid_match_list) = ut.search_list(text_list, pattern, flags)
>>> result = str(valid_index_list)
>>> print(result)
[0, 1, 3]
"""
import re
import utool as ut
match_list = [re.search(pattern, text, flags=flags) for text in text_list]
valid_index_list = [index for index, match in enumerate(match_list) if match is not None]
valid_match_list = ut.take(match_list, valid_index_list)
return valid_index_list, valid_match_list
# --- List Modification --- #
[docs]def multi_replace(instr, search_list=[], repl_list=None):
"""
Does a string replace with a list of search and replacements
TODO: rename
"""
repl_list = [''] * len(search_list) if repl_list is None else repl_list
for ser, repl in zip(search_list, repl_list):
instr = instr.replace(ser, repl)
return instr
[docs]def flatten(list_):
r"""
Args:
list_ (list): list of lists
Returns:
list: flat list
CommandLine:
python -m utool.util_list --test-flatten
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> list_ = [['a', 'b'], ['c', 'd']]
>>> unflat_list2 = flatten(list_)
>>> result = ut.list_str(unflat_list2, nl=False)
>>> print(result)
['a', 'b', 'c', 'd']
"""
return list(util_iter.iflatten(list_))
[docs]def invertible_flatten(unflat_list):
r"""
Flattens ``list`` but remember how to reconstruct the unflat ``list``
Returns flat ``list`` and the unflat ``list`` with indexes into the flat
``list``
Args:
unflat_list (list): list of nested lists that we will flatten.
Returns:
tuple : (flat_list, reverse_list)
CommandLine:
python -m utool.util_list --exec-invertible_flatten --show
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> unflat_list = [[1, 2, 3], [4, 5], [6, 6]]
>>> flat_list, reverse_list = invertible_flatten(unflat_list)
>>> result = ('flat_list = %s\n' % (ut.repr2(flat_list),))
>>> result += ('reverse_list = %s' % (ut.repr2(reverse_list),))
>>> print(result)
flat_list = [1, 2, 3, 4, 5, 6, 6]
reverse_list = [[0, 1, 2], [3, 4], [5, 6]]
"""
# def nextnum(trick_=[0]):
# num = trick_[0]
# trick_[0] += 1
# return num
nextnum = functools.partial(six.next, itertools.count(0))
# Build an unflat list of flat indexes
reverse_list = [[nextnum() for _ in tup] for tup in unflat_list]
flat_list = flatten(unflat_list)
return flat_list, reverse_list
@profile
[docs]def unflatten(flat_list, reverse_list):
""" Rebuilds unflat list from invertible_flatten
Args:
flat_list (list): the flattened list
reverse_list (list): the list which undoes flattenting
Returns:
unflat_list2: original nested list
SeeAlso:
invertible_flatten
invertible_flatten2
unflatten2
"""
unflat_list2 = [[flat_list[index] for index in tup] for tup in reverse_list]
return unflat_list2
@profile
[docs]def accumulate(iterator):
"""
Notice:
use itertools.accumulate in python > 3.2
"""
total = 0
for item in iterator:
total += item
yield total
@profile
[docs]def invertible_flatten2(unflat_list):
"""
An alternative to invertible_flatten which uses cumsum
Flattens ``list`` but remember how to reconstruct the unflat ``list``
Returns flat ``list`` and the unflat ``list`` with indexes into the flat
``list``
Args:
unflat_list (list):
Returns:
tuple: flat_list, cumlen_list
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool
>>> utool.util_list
>>> unflat_list = [[5], [2, 3, 12, 3, 3], [9], [13, 3], [5]]
>>> flat_list, cumlen_list = invertible_flatten2(unflat_list)
>>> unflat_list2 = unflatten2(flat_list, cumlen_list)
>>> assert unflat_list2 == unflat_list
>>> result = ((flat_list, cumlen_list))
>>> print(result)
([5, 2, 3, 12, 3, 3, 9, 13, 3, 5], [1, 6, 7, 9, 10])
TODO: This flatten is faster fix it to be used everywhere
Timeit:
unflat_list = [[random.random() for _ in range(int(random.random() * 1000))] for __ in range(200)]
unflat_arrs = list(map(np.array, unflat_list))
%timeit invertible_flatten2(unflat_list)
%timeit invertible_flatten2_numpy(unflat_list)
%timeit invertible_flatten2_numpy(unflat_arrs)
SeeAlso:
invertible_flatten
unflatten
unflatten2
Timeits:
import utool
unflat_list = aids_list1
flat_aids1, reverse_list = utool.invertible_flatten(unflat_list)
flat_aids2, cumlen_list = utool.invertible_flatten2(unflat_list)
unflat_list1 = utool.unflatten(flat_aids1, reverse_list)
unflat_list2 = utool.unflatten2(flat_aids2, cumlen_list)
assert list(map(list, unflat_list1)) == unflat_list2
print(utool.get_object_size_str(unflat_list, 'unflat_list '))
print(utool.get_object_size_str(flat_aids1, 'flat_aids1 '))
print(utool.get_object_size_str(flat_aids2, 'flat_aids2 '))
print(utool.get_object_size_str(reverse_list, 'reverse_list '))
print(utool.get_object_size_str(cumlen_list, 'cumlen_list '))
print(utool.get_object_size_str(unflat_list1, 'unflat_list1 '))
print(utool.get_object_size_str(unflat_list2, 'unflat_list2 '))
print('Timings 1:)
%timeit utool.invertible_flatten(unflat_list)
%timeit utool.unflatten(flat_aids1, reverse_list)
print('Timings 2:)
%timeit utool.invertible_flatten2(unflat_list)
%timeit utool.unflatten2(flat_aids2, cumlen_list)
"""
sublen_list = list(map(len, unflat_list))
if not util_type.HAVE_NUMPY:
cumlen_list = np.cumsum(sublen_list)
# Build an unflat list of flat indexes
else:
cumlen_list = list(accumulate(sublen_list))
flat_list = flatten(unflat_list)
return flat_list, cumlen_list
@profile
[docs]def invertible_flatten2_numpy(unflat_arrs, axis=0):
""" more numpy version
TODO: move to vtool
Args:
unflat_arrs (list): list of ndarrays
Returns:
tuple: (flat_list, cumlen_list)
CommandLine:
python -m utool.util_list --test-invertible_flatten2_numpy
Example:
>>> # ENABLE_DOCTET
>>> from utool.util_list import * # NOQA
>>> unflat_arrs = [np.array([1, 2, 1]), np.array([5, 9]), np.array([4])]
>>> (flat_list, cumlen_list) = invertible_flatten2_numpy(unflat_arrs)
>>> result = str((flat_list, cumlen_list))
>>> print(result)
(array([1, 2, 1, 5, 9, 4]), array([3, 5, 6]))
"""
cumlen_list = np.cumsum([arr.shape[axis] for arr in unflat_arrs])
flat_list = np.concatenate(unflat_arrs, axis=axis)
return flat_list, cumlen_list
[docs]def unflatten2(flat_list, cumlen_list):
""" Rebuilds unflat list from invertible_flatten
Args:
flat_list (list): the flattened list
cumlen_list (list): the list which undoes flattenting
Returns:
unflat_list2: original nested list
SeeAlso:
invertible_flatten
invertible_flatten2
unflatten2
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool
>>> utool.util_list
>>> flat_list = [5, 2, 3, 12, 3, 3, 9, 13, 3, 5]
>>> cumlen_list = [ 1, 6, 7, 9, 10]
>>> unflat_list2 = unflatten2(flat_list, cumlen_list)
>>> result = (unflat_list2)
>>> print(result)
[[5], [2, 3, 12, 3, 3], [9], [13, 3], [5]]
"""
unflat_list2 = [flat_list[low:high] for low, high in
zip(itertools.chain([0], cumlen_list), cumlen_list)]
return unflat_list2
@profile
[docs]def unflat_unique_rowid_map(func, unflat_rowids, **kwargs):
"""
performs only one call to the underlying func with unique rowids the func
must be some lookup function
TODO: move this to a better place.
CommandLine:
python -m utool.util_list --test-unflat_unique_rowid_map:0
python -m utool.util_list --test-unflat_unique_rowid_map:1
Example0:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> kwargs = {}
>>> unflat_rowids = [[1, 2, 3], [2, 5], [1], []]
>>> num_calls0 = [0]
>>> num_input0 = [0]
>>> def func0(rowids, num_calls0=num_calls0, num_input0=num_input0):
... num_calls0[0] += 1
... num_input0[0] += len(rowids)
... return [rowid + 10 for rowid in rowids]
>>> func = func0
>>> unflat_vals = unflat_unique_rowid_map(func, unflat_rowids, **kwargs)
>>> result = [arr.tolist() for arr in unflat_vals]
>>> print(result)
>>> ut.assert_eq(num_calls0[0], 1)
>>> ut.assert_eq(num_input0[0], 4)
[[11, 12, 13], [12, 15], [11], []]
Example1:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> import numpy as np
>>> kwargs = {}
>>> unflat_rowids = [[1, 2, 3], [2, 5], [1], []]
>>> num_calls1 = [0]
>>> num_input1 = [0]
>>> def func1(rowids, num_calls1=num_calls1, num_input1=num_input1, np=np):
... num_calls1[0] += 1
... num_input1[0] += len(rowids)
... return [np.array([rowid + 10, rowid, 3]) for rowid in rowids]
>>> func = func1
>>> unflat_vals = unflat_unique_rowid_map(func, unflat_rowids, **kwargs)
>>> result = [arr.tolist() for arr in unflat_vals]
>>> print(result)
>>> ut.assert_eq(num_calls1[0], 1)
>>> ut.assert_eq(num_input1[0], 4)
[[[11, 1, 3], [12, 2, 3], [13, 3, 3]], [[12, 2, 3], [15, 5, 3]], [[11, 1, 3]], []]
"""
import utool as ut
# First flatten the list, and remember the original dimensions
flat_rowids, reverse_list = ut.invertible_flatten2(unflat_rowids)
# Then make the input unique
flat_rowids_arr = np.array(flat_rowids)
unique_flat_rowids, inverse_unique = np.unique(flat_rowids_arr, return_inverse=True)
# Then preform the lookup / implicit mapping
unique_flat_vals = func(unique_flat_rowids, **kwargs)
# Then broadcast unique values back to original flat positions
flat_vals_ = np.array(unique_flat_vals)[inverse_unique]
#flat_vals_ = np.array(unique_flat_vals).take(inverse_unique, axis=0)
output_shape = tuple(list(flat_rowids_arr.shape) + list(flat_vals_.shape[1:]))
flat_vals = np.array(flat_vals_).reshape(output_shape)
# Then _unflatten the results to the original input dimensions
unflat_vals = ut.unflatten2(flat_vals, reverse_list)
return unflat_vals
[docs]def tuplize(list_):
""" Converts each scalar item in a list to a dimension-1 tuple
"""
tup_list = [item if util_iter.isiterable(item) else (item,) for item in list_]
return tup_list
[docs]def flattenize(list_):
""" maps flatten to a tuplized list
Weird function. DEPRICATE
Example:
>>> list_ = [[1, 2, 3], [2, 3, [4, 2, 1]], [3, 2], [[1, 2], [3, 4]]]
>>> import utool
>>> from itertools import zip
>>> val_list1 = [(1, 2), (2, 4), (5, 3)]
>>> id_list1 = [(1,), (2,), (3,)]
>>> out_list1 = utool.flattenize(zip(val_list1, id_list1))
>>> val_list2 = [1, 4, 5]
>>> id_list2 = [(1,), (2,), (3,)]
>>> out_list2 = utool.flattenize(zip(val_list2, id_list2))
>>> val_list3 = [1, 4, 5]
>>> id_list3 = [1, 2, 3]
>>> out_list3 = utool.flattenize(zip(val_list3, id_list3))
out_list4 = list(zip(val_list3, id_list3))
%timeit utool.flattenize(zip(val_list1, id_list1))
%timeit utool.flattenize(zip(val_list2, id_list2))
%timeit utool.flattenize(zip(val_list3, id_list3))
%timeit list(zip(val_list3, id_list3))
100000 loops, best of 3: 14 us per loop
100000 loops, best of 3: 16.5 us per loop
100000 loops, best of 3: 18 us per loop
1000000 loops, best of 3: 1.18 us per loop
"""
#return map(iflatten, list_)
#if not isiterable(list_):
# list2_ = (list_,)
#else:
# list2_ = list_
tuplized_iter = list(map(tuplize, list_))
flatenized_list = list(map(flatten, tuplized_iter))
return flatenized_list
[docs]def safe_slice(list_, *args):
""" safe_slice(list_, [start], stop, [end], [step])
Slices list and truncates if out of bounds
"""
if len(args) == 3:
start = args[0]
stop = args[1]
step = args[2]
else:
step = 1
if len(args) == 2:
start = args[0]
stop = args[1]
else:
start = 0
stop = args[0]
len_ = len(list_)
if stop > len_:
stop = len_
return list_[slice(start, stop, step)]
# --- List Queries --- #
[docs]def allsame(list_, strict=True):
"""
checks to see if list is equal everywhere
Args:
list_ (list):
Returns:
True if all items in the list are equal
"""
if len(list_) == 0:
return True
first_item = list_[0]
return list_all_eq_to(list_, first_item, strict)
[docs]def list_all_eq_to(list_, val, strict=True):
"""
checks to see if list is equal everywhere to a value
Args:
list_ (list):
val : value to check against
Returns:
True if all items in the list are equal to val
"""
if util_type.HAVE_NUMPY and isinstance(val, np.ndarray):
return all([np.all(item == val) for item in list_])
try:
return all([item == val for item in list_])
except ValueError:
if not strict:
return all([repr(item) == repr(val) for item in list_])
else:
raise
[docs]def get_dirty_items(item_list, flag_list):
"""
Returns each item in item_list where not flag in flag_list
Args:
item_list (list):
flag_list (list):
Returns:
dirty_items
"""
assert len(item_list) == len(flag_list)
dirty_items = [item for (item, flag) in
zip(item_list, flag_list)
if not flag]
#print('num_dirty_items = %r' % len(dirty_items))
#print('item_list = %r' % (item_list,))
#print('flag_list = %r' % (flag_list,))
return dirty_items
[docs]def compress(item_list, flag_list):
"""
like np.compress but for lists
Returns items in item list where the corresponding item in flag list is
True
Args:
item_list (list): list of items to mask
flag_list (list): list of booleans used as a mask
Returns:
list : filtered_items - masked items
"""
assert len(item_list) == len(flag_list), (
'lists should correspond. len(item_list)=%r len(flag_list)=%r' %
(len(item_list), len(flag_list)))
filtered_items = list(util_iter.iter_compress(item_list, flag_list))
return filtered_items
[docs]def ziptake(items_list, indexes_list):
"""
SeeAlso:
vt.ziptake
"""
return [take(list_, index_list)
for list_, index_list in zip(items_list, indexes_list)]
[docs]def zipcompress(items_list, flags_list):
"""
SeeAlso:
vt.zipcompress
"""
return [compress(list_, flags)
for list_, flags in zip(items_list, flags_list)]
[docs]def list_zipflatten(*items_lists):
return [flatten(items) for items in zip(*items_lists)]
[docs]def list_compresstake(items_list, flags_list):
return [compress(list_, flags) for list_, flags in zip(items_list, flags_list)]
[docs]def filter_items(item_list, flag_list):
"""
Returns items in item list where the corresponding item in flag list is True
Args:
item_list (list):
flag_list (list):
Returns:
filtered_items
SeeAlso:
util_iter.iter_compress
"""
return compress(item_list, flag_list)
[docs]def filterfalse_items(item_list, flag_list):
"""
Returns items in item list where the corresponding item in flag list is true
Args:
item_list (list): list of items
flag_list (list): list of truthy values
Returns:
filtered_items : items where the corresponding flag was truthy
SeeAlso:
util_iter.ifilterfalse_items
"""
assert len(item_list) == len(flag_list)
filtered_items = list(util_iter.ifilterfalse_items(item_list, flag_list))
return filtered_items
[docs]def filter_Nones(item_list):
"""
Removes any nones from the list
Args:
item_list (list):
Returns:
sublist which does not contain Nones
"""
return list(util_iter.ifilter_Nones(item_list))
# --- List combinations --- #
[docs]def intersect_ordered(list1, list2):
"""
returns list1 elements that are also in list2. preserves order of list1
intersect_ordered
Args:
list1 (list):
list2 (list):
Returns:
list: new_list
Example:
>>> from utool.util_list import * # NOQA
>>> list1 = ['featweight_rowid', 'feature_rowid', 'config_rowid', 'featweight_forground_weight']
>>> list2 = [u'featweight_rowid']
>>> result = intersect_ordered(list1, list2)
>>> print(result)
['featweight_rowid']
"""
return [item for item in list1 if item in set(list2)]
[docs]def list_intersection(list1, list2):
return set(list1).intersection(set(list2))
[docs]def list_issubset(list1, list2):
return set(list1).issubset(set(list2))
[docs]def list_issuperset(list1, list2):
return set(list1).issuperset(set(list2))
issubset = list_issubset
is_subset = list_issubset
is_superset = list_issuperset
issuperset = list_issuperset
[docs]def list_set_equal(list1, list2):
return set(list1) == set(list2)
[docs]def is_subset_of_any(set_, other_sets):
"""
returns True if set_ is a subset of any set in other_sets
Args:
set_ (set):
other_sets (list of sets):
Returns:
bool: flag
CommandLine:
python -m utool.util_list --test-is_subset_of_any
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> # build test data
>>> set_ = {1, 2}
>>> other_sets = [{1, 4}, {3, 2, 1}]
>>> # execute function
>>> result = is_subset_of_any(set_, other_sets)
>>> # verify results
>>> print(result)
True
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> # build test data
>>> set_ = {1, 2}
>>> other_sets = [{1, 4}, {3, 2}]
>>> # execute function
>>> result = is_subset_of_any(set_, other_sets)
>>> # verify results
>>> print(result)
False
"""
set_ = set(set_)
other_sets = map(set, other_sets)
return any([set_.issubset(other_set) for other_set in other_sets])
[docs]def priority_sort(list_, priority):
r"""
Args:
list_ (list):
priority (list): desired order of items
Returns:
list: reordered_list
CommandLine:
python -m utool.util_list --test-priority_argsort
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [2, 4, 6, 8, 10]
>>> priority = [8, 2, 6, 9]
>>> reordered_list = priority_sort(list_, priority)
>>> result = str(reordered_list)
>>> print(result)
[8, 2, 6, 4, 10]
"""
# remove requested priority items not in the list
priority_ = setintersect_ordered(priority, list_)
reordered_list = unique_ordered(priority_ + list_)
return reordered_list
[docs]def priority_argsort(list_, priority):
r"""
Args:
list_ (list):
priority (list): desired order of items
Returns:
list: reordered_index_list
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> list_ = [2, 4, 6, 8, 10]
>>> priority = [8, 2, 6, 9]
>>> sortx = priority_argsort(list_, priority)
>>> reordered_list = priority_sort(list_, priority)
>>> assert ut.take(list_, sortx) == reordered_list
>>> result = str(sortx)
>>> print(result)
[3, 0, 2, 1, 4]
"""
reordered_list = priority_sort(list_, priority)
# FIXME: inefficient
sortx = [list_.index(item) for item in reordered_list]
return sortx
[docs]def flag_unique_items(list_):
"""
Returns a list of flags corresponding to the first time an item is seen
Args:
list_ (list): list of items
Returns:
flag_list
"""
seen = set()
def unseen(item):
if item in seen:
return False
seen.add(item)
return True
flag_list = [unseen(item) for item in list_]
return flag_list
[docs]def iflag_unique_items(list_):
"""
Returns a list of flags corresponding to the first time an item is seen
Args:
list_ (list): list of items
Returns:
flag_iter
"""
seen = set()
def unseen(item):
if item in seen:
return False
seen.add(item)
return True
flag_iter = (unseen(item) for item in list_)
return flag_iter
[docs]def unique_ordered(list_):
"""
Returns unique items in ``list_`` in the order they were seen.
Args:
list_ (list):
Returns:
list: unique_list - unique list which maintains order
CommandLine:
python -m utool.util_list --exec-unique_ordered
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [4, 6, 6, 0, 6, 1, 0, 2, 2, 1]
>>> unique_list = unique_ordered(list_)
>>> result = ('unique_list = %s' % (str(unique_list),))
>>> print(result)
unique_list = [4, 6, 0, 1, 2]
"""
flag_list = flag_unique_items(list_)
unique_list = compress(list_, flag_list)
return unique_list
[docs]def unique_unordered(list_):
"""
wrapper around list(set(list_))
"""
return list(set(list_))
[docs]def unique(list_, ordered=True):
"""
Returns unique items in ``list_``.
Generally, unordered (*should be) faster.
"""
if ordered:
return unique_ordered(list_)
else:
return unique_unordered(list_)
[docs]def flat_unique(*lists_, **kwargs):
""" returns items unique across all lists """
return unique(flatten(lists_), **kwargs)
[docs]def setdiff(list1, list2):
"""
returns list1 elements that are not in list2. preserves order of list1
Args:
list1 (list):
list2 (list):
Returns:
list: new_list
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> list1 = ['featweight_rowid', 'feature_rowid', 'config_rowid', 'featweight_forground_weight']
>>> list2 = [u'featweight_rowid']
>>> new_list = setdiff_ordered(list1, list2)
>>> result = ut.list_str(new_list, nl=False)
>>> print(result)
['feature_rowid', 'config_rowid', 'featweight_forground_weight']
"""
set2 = set(list2)
return [item for item in list1 if item not in set2]
[docs]def setdiff_flags(list1, list2):
return list(isetdiff_flags(list1, list2))
[docs]def isetdiff_flags(list1, list2):
"""
move to util_iter
"""
set2 = set(list2)
return (item not in set2 for item in list1)
setdiff_ordered = setdiff
[docs]def setintersect_ordered(list1, list2):
"""
returns list1 elements that are in list2. preserves order of list1
setintersect_ordered
Args:
list1 (list):
list2 (list):
Returns:
list: new_list
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list1 = [1, 2, 3, 5, 8, 13, 21]
>>> list2 = [6, 4, 2, 21, 8]
>>> new_list = setintersect_ordered(list1, list2)
>>> result = new_list
>>> print(result)
[2, 8, 21]
"""
return [item for item in list1 if item in set(list2)]
setintersect = setintersect_ordered
[docs]def sortedby(item_list, key_list, reverse=False):
""" sorts ``item_list`` using key_list
Args:
list_ (list): list to sort
key_list (list): list to sort by
reverse (bool): sort order is descending (largest first)
if reverse is True else acscending (smallest first)
Returns:
list : ``list_`` sorted by the values of another ``list``. defaults to
ascending order
SeeAlso:
sortedby2
Examples:
>>> # ENABLE_DOCTEST
>>> import utool
>>> list_ = [1, 2, 3, 4, 5]
>>> key_list = [2, 5, 3, 1, 5]
>>> result = utool.sortedby(list_, key_list, reverse=True)
[5, 2, 3, 1, 4]
"""
assert len(item_list) == len(key_list), (
'Expected same len. Got: %r != %r' % (len(item_list), len(key_list)))
sorted_list = [item for (key, item) in
sorted(list(zip(key_list, item_list)), reverse=reverse)]
return sorted_list
[docs]def sortedby2(item_list, *args, **kwargs):
""" sorts ``item_list`` using key_list
Args:
item_list (list): list to sort
Varargs:
*args (list): multiple lists to sort by
Kwargs:
reverse (bool): sort order is descending if True else acscending
Returns:
list : ``list_`` sorted by the values of another ``list``. defaults to
ascending order
CommandLine:
python -m utool.util_list --exec-sortedby2 --show
Examples:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> item_list = [1, 2, 3, 4, 5]
>>> key_list1 = [1, 1, 2, 3, 4]
>>> key_list2 = [2, 1, 4, 1, 1]
>>> args = (key_list1, key_list2)
>>> kwargs = dict(reverse=False)
>>> result = ut.sortedby2(item_list, *args, **kwargs)
>>> print(result)
[2, 1, 3, 4, 5]
Examples:
>>> # ENABLE_DOCTEST
>>> # Python 3 Compatibility Test
>>> import utool as ut
>>> item_list = [1, 2, 3, 4, 5]
>>> key_list1 = ['a', 'a', 2, 3, 4]
>>> key_list2 = ['b', 'a', 4, 1, 1]
>>> args = (key_list1, key_list2)
>>> kwargs = dict(reverse=False)
>>> result = ut.sortedby2(item_list, *args, **kwargs)
>>> print(result)
[3, 4, 5, 2, 1]
"""
import operator
assert all([len(item_list) == len_ for len_ in map(len, args)])
reverse = kwargs.get('reverse', False)
key = operator.itemgetter(*range(1, len(args) + 1))
tup_list = list(zip(item_list, *args))
#print(tup_list)
try:
sorted_tups = sorted(tup_list, key=key, reverse=reverse)
except TypeError:
# Python 3 does not allow sorting mixed types
def keyfunc(tup):
return tuple(map(str, tup[1:]))
sorted_tups = sorted(tup_list, key=keyfunc, reverse=reverse)
sorted_list = [tup[0] for tup in sorted_tups]
return sorted_list
[docs]def list_unflat_take(items_list, unflat_index_list):
r"""
Args:
items_list (list):
unflat_index_list (list):
CommandLine:
python -m utool.util_list --exec-list_unflat_take
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> items_list = [1, 2, 3, 4, 5]
>>> unflat_index_list = [[0, 1], [2, 3], [0, 4]]
>>> result = list_unflat_take(items_list, unflat_index_list)
>>> print(result)
[[1, 2], [3, 4], [1, 5]]
"""
return [list_unflat_take(items_list, xs)
if isinstance(xs, list) else
take(items_list, xs)
for xs in unflat_index_list]
[docs]def argsort(*args, **kwargs):
""" like np.argsort but for lists
Varargs:
*args (list): multiple lists to sort by
Kwargs:
reverse (bool): sort order is descending if True else acscending
"""
index_list = list(range(len(args[0])))
return sortedby2(index_list, *args, **kwargs)
[docs]def index_complement(index_list, len_=None):
"""
Returns the other indicies in a list of length ``len_``
"""
mask1 = index_to_boolmask(index_list, len_)
mask2 = not_list(mask1)
index_list_bar = list_where(mask2)
return index_list_bar
[docs]def take_complement(list_, index_list):
""" Returns items in ``list_`` not indexed by index_list """
mask = not_list(index_to_boolmask(index_list, len(list_)))
return compress(list_, mask)
[docs]def take(list_, index_list):
""" like np.take but for lists
Args:
list_ (list): some indexable object
index_list (list, slice, int): some indexing object
Returns:
list or scalar: subset of the list
CommandLine:
python -m utool.util_list --test-take
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [0, 1, 2, 3]
>>> index_list = [2, 0]
>>> result = take(list_, index_list)
>>> print(result)
[2, 0]
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [0, 1, 2, 3]
>>> index = 2
>>> result = take(list_, index)
>>> print(result)
2
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [0, 1, 2, 3]
>>> index = slice(1, None, 2)
>>> result = take(list_, index)
>>> print(result)
[1, 3]
"""
try:
return [list_[index] for index in index_list]
except TypeError:
return list_[index_list]
#if util_iter.isiterable(index_list):
#else:
# def take
# def take2(item_list, indicies, axis):
# def _get_axes(list_, axis);
# pass
[docs]def take_percentile(arr, percent):
""" take the top `percent` items in a list rounding up """
size = len(arr)
stop = min(int(size * percent), len(arr))
return arr[0:stop]
[docs]def list_inverse_take(list_, index_list):
r"""
Args:
list_ (list): list in sorted domain
index_list (list): index list of the unsorted domain
Note:
Seems to be logically equivalent to
ut.take(list_, ut.argsort(index_list)), but faster
Returns:
list: output_list_ - the input list in the unsorted domain
CommandLine:
python -m utool.util_list --test-list_inverse_take
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> # build test data
>>> rank_list = [3, 2, 4, 1, 9, 2]
>>> prop_list = [0, 1, 2, 3, 4, 5]
>>> index_list = ut.argsort(rank_list)
>>> sorted_prop_list = ut.take(prop_list, index_list)
>>> # execute function
>>> list_ = sorted_prop_list
>>> output_list_ = list_inverse_take(list_, index_list)
>>> output_list2_ = ut.take(list_, ut.argsort(index_list))
>>> assert output_list_ == prop_list
>>> assert output_list2_ == prop_list
>>> # verify results
>>> result = str(output_list_)
>>> print(result)
Timeit::
%timeit list_inverse_take(list_, index_list)
%timeit ut.take(list_, ut.argsort(index_list))
"""
output_list_ = [None] * len(index_list)
for item, index in zip(list_, index_list):
output_list_[index] = item
return output_list_
[docs]def where(flag_list):
""" takes flags returns indexes of True values """
return [index for index, flag in enumerate(flag_list) if flag]
[docs]def where_not_None(item_list):
""" returns list of indexes of non None values
SeeAlso:
flag_None_items
"""
return [index for index, item in enumerate(item_list) if item is not None]
[docs]def flag_None_items(list_):
return [item is None for item in list_]
[docs]def flag_not_None_items(list_):
return [item is not None for item in list_]
[docs]def partial_imap_1to1(func, si_func):
""" a bit messy """
@functools.wraps(si_func)
def wrapper(input_):
if not util_iter.isiterable(input_):
return func(si_func(input_))
else:
return list(map(func, si_func(input_)))
set_funcname(wrapper, util_str.get_callable_name(func) + '_mapper_' + get_funcname(si_func))
return wrapper
[docs]def sample_zip(items_list, num_samples, allow_overflow=False, per_bin=1):
""" Helper for sampling
Given a list of lists, samples one item for each list and bins them into
num_samples bins. If all sublists are of equal size this is equivilent to a
zip, but otherewise consecutive bins will have monotonically less
elemements
# Doctest doesn't work with assertionerror
#util_list.sample_zip(items_list, 2)
#...
#AssertionError: Overflow occured
Args:
items_list (list):
num_samples (?):
allow_overflow (bool):
per_bin (int):
Returns:
tuple : (samples_list, overflow_samples)
Examples:
>>> from utool import util_list
>>> items_list = [[1, 2, 3, 4, 0], [5, 6, 7], [], [8, 9], [10]]
>>> util_list.sample_zip(items_list, 5)
...
[[1, 5, 8, 10], [2, 6, 9], [3, 7], [4], [0]]
>>> util_list.sample_zip(items_list, 2, allow_overflow=True)
...
([[1, 5, 8, 10], [2, 6, 9]], [3, 7, 4])
>>> util_list.sample_zip(items_list, 4, allow_overflow=True, per_bin=2)
...
([[1, 5, 8, 10, 2, 6, 9], [3, 7, 4], [], []], [0])
"""
# Prealloc a list of lists
samples_list = [[] for _ in range(num_samples)]
# Sample the ix-th value from every list
samples_iter = zip_longest(*items_list)
sx = 0
for ix, samples_ in zip(range(num_samples), samples_iter):
samples = filter_Nones(samples_)
samples_list[sx].extend(samples)
# Put per_bin from each sublist into a sample
if (ix + 1) % per_bin == 0:
sx += 1
# Check for overflow
if allow_overflow:
overflow_samples = flatten([filter_Nones(samples_) for samples_ in samples_iter])
return samples_list, overflow_samples
else:
try:
samples_iter.next()
except StopIteration:
pass
else:
raise AssertionError('Overflow occured')
return samples_list
[docs]def sample_lists(items_list, num=1, seed=None):
r"""
Args:
items_list (list):
num (int): (default = 1)
seed (None): (default = None)
Returns:
list: samples_list
CommandLine:
python -m utool.util_list --exec-sample_lists
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> items_list = [[], [1, 2, 3], [4], [5, 6], [7, 8, 9, 10]]
>>> num = 2
>>> seed = 0
>>> samples_list = sample_lists(items_list, num, seed)
>>> result = ('samples_list = %s' % (str(samples_list),))
>>> print(result)
samples_list = [[], [3, 2], [4], [5, 6], [10, 9]]
"""
if seed is not None:
rng = np.random.RandomState(seed)
else:
rng = np.random
def random_choice(items, num):
size = min(len(items), num)
return rng.choice(items, size, replace=False).tolist()
samples_list = [random_choice(items, num)
if len(items) > 0 else []
for items in items_list]
return samples_list
[docs]def strided_sample(items, num, offset=0):
r"""
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> # build test data
>>> items = [1, 2, 3, 4, 5]
>>> num = 3
>>> offset = 0
>>> # execute function
>>> sample_items = strided_sample(items, num, offset)
>>> # verify results
>>> result = str(sample_items)
>>> print(result)
"""
import math
stride = max(int(math.ceil(len(items) / num)), 1)
sample_items = items[offset::stride]
return sample_items
[docs]def issorted(list_, op=operator.le):
"""
Args:
list_ (list):
op (builtin_function_or_method):
Returns:
bool : True if the list is sorted
"""
return all(op(list_[ix], list_[ix + 1]) for ix in range(len(list_) - 1))
[docs]def find_nonconsec_indices(unique_vals, consec_vals):
"""
# TODO: rectify with above function
Args:
unique_vals (list):
consec_vals (list):
Returns:
missing_ixs
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import numpy as np
>>> unique_vals = np.array([-2, -1, 1, 2, 10])
>>> max_ = unique_vals.max()
>>> min_ = unique_vals.min()
>>> range_ = max_ - min_
>>> consec_vals = np.linspace(min_, max_ + 1, range_ + 2)
>>> missing_ixs = find_nonconsec_indices(unique_vals, consec_vals)
>>> result = (consec_vals[missing_ixs])
[ 0. 3. 4. 5. 6. 7. 8. 9.]
"""
missing_ixs = []
valx = 0
consecx = 0
while valx < len(unique_vals) and consecx < len(consec_vals):
if unique_vals[valx] != consec_vals[consecx]:
missing_ixs.append(consecx)
else:
valx += 1
consecx += 1
return missing_ixs
[docs]def group_consecutives(data, stepsize=1):
"""
Return list of consecutive lists of numbers from data (number list).
References:
http://stackoverflow.com/questions/7352684/how-to-find-the-groups-of-consecutive-elements-from-an-array-in-numpy
"""
run = []
result = [run]
expect = None
for item in data:
if (item == expect) or (expect is None):
run.append(item)
else:
run = [item]
result.append(run)
expect = item + stepsize
return result
[docs]def group_consecutives_numpy(data, stepsize=1):
"""
Args:
data (?):
stepsize (int):
Returns:
list: list of ndarrays
References:
http://stackoverflow.com/questions/7352684/how-to-find-the-groups-of-consecutive-elements-from-an-array-in-numpy
CommandLine:
python -m utool.util_list --test-group_consecutives
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> # build test data
>>> data = np.array([ 0, 1, 2, 3, 4, 320, 636, 637, 638, 639])
>>> stepsize = 1
>>> # execute function
>>> result = group_consecutives(data, stepsize)
>>> # verify results
>>> print(result)
[array([0, 1, 2, 3, 4]), array([320]), array([636, 637, 638, 639])]
Timeit::
%timeit group_consecutives_numpy(data, stepsize) # 14.8 µs per loop
%timeit group_consecutives(data, stepsize) # 4.47 µs per loop
"""
return np.split(data, np.where(np.diff(data) != stepsize)[0] + 1)
[docs]def debug_consec_list(list_):
"""
Returns:
tuple of (missing_items, missing_indices, duplicate_items)
"""
if not issorted(list_):
print('warning list is not sorted. indices will not match')
sortedlist = sorted(list_)
start = sortedlist[0]
last = start - 1
missing_vals = []
missing_indices = []
duplicate_items = []
for count, item in enumerate(sortedlist):
diff = item - last
if diff > 1:
missing_indices.append(count)
for miss in range(last + 1, last + diff):
missing_vals.append(miss)
elif diff == 0:
duplicate_items.append(item)
elif diff == 1:
# Expected case
pass
else:
raise AssertionError('We sorted the list. diff can not be negative')
last = item
return missing_vals, missing_indices, duplicate_items
[docs]def find_duplicate_items(items):
r"""
Args:
items (list):
Returns:
dict: duplicate_map of indexes
CommandLine:
python -m utool.util_list --test-find_duplicate_items
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> items = [0, 1, 2, 3, 3, 0, 12, 2, 9]
>>> duplicate_map = find_duplicate_items(items)
>>> result = str(duplicate_map)
>>> print(result)
"""
import utool as ut
# Build item histogram
duplicate_map = ut.ddict(list)
for count, item in enumerate(items):
duplicate_map[item].append(count)
# remove singleton items
singleton_keys = []
for key in six.iterkeys(duplicate_map):
if len(duplicate_map[key]) == 1:
singleton_keys.append(key)
for key in singleton_keys:
del duplicate_map[key]
return duplicate_map
#get_non_consecutive_positions = debug_consec_list
[docs]def duplicates_exist(items):
""" returns if list has duplicates """
return len(items) - len(set(items)) != 0
[docs]def isunique(items):
return not duplicates_exist(items)
[docs]def print_duplicate_map(duplicate_map, *args, **kwargs):
# args are corresponding lists
import utool as ut
printfn = kwargs.get('printfn', print)
printfn('There are %d duplicates' % (len(duplicate_map)))
for key, index_list in six.iteritems(duplicate_map):
printfn('item=%s appears %d times at indices: %r' % (key, len(index_list), index_list))
for argx, arg in enumerate(args):
#argname = 'arg%d' % (argx)
argname = ut.get_varname_from_stack(arg, N=2)
for index in index_list:
printfn(' * %s[%d] = %r' % (argname, index, arg[index]))
return duplicate_map
[docs]def debug_duplicate_items(items, *args, **kwargs):
import utool as ut
pad_stdout = kwargs.get('pad_stdout', True)
if pad_stdout:
print('')
print('[util_list] +--- DEBUG DUPLICATE ITEMS %r ---' % ut.get_varname_from_locals(items, ut.get_caller_locals()))
def printfn(msg):
print('[util_list] |' + msg)
#with ut.Indenter('[util_list] | '):
duplicate_map = ut.find_duplicate_items(items)
printkw = {'printfn': printfn}
ut.print_duplicate_map(duplicate_map, *args, **printkw)
print('[util_list] L--- FINISH DEBUG DUPLICATE ITEMS ---')
if pad_stdout:
print('')
return duplicate_map
[docs]def list_depth(list_, func=max, _depth=0):
"""
Returns the deepest level of nesting within a list of lists
Args:
list_ : a nested listlike object
func : depth aggregation strategy (defaults to max)
_depth : internal var
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [[[[[1]]], [3]], [[1], [3]], [[1], [3]]]
>>> result = (list_depth(list_, _depth=0))
>>> print(result)
"""
depth_list = [list_depth(item, func=func, _depth=_depth + 1)
for item in list_ if util_type.is_listlike(item)]
if len(depth_list) > 0:
return func(depth_list)
else:
return _depth
[docs]def list_deep_types(list_):
"""
Returns all types in a deep list
"""
type_list = []
for item in list_:
if util_type.is_listlike(item):
type_list.extend(list_deep_types(item))
else:
type_list.append(type(item))
return type_list
[docs]def depth_profile(list_, max_depth=None, compress_homogenous=True, compress_consecutive=False, new_depth=False):
r"""
Returns a nested list corresponding the shape of the nested structures
lists represent depth, tuples represent shape. The values of the items do
not matter. only the lengths.
Args:
list_ (list):
max_depth (None):
compress_homogenous (bool):
compress_consecutive (bool): experimental
CommandLine:
python -m utool.util_list --test-depth_profile
Setup:
>>> from utool.util_list import * # NOQA
Example0:
>>> # ENABLE_DOCTEST
>>> list_ = [[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]], [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
>>> result = depth_profile(list_)
>>> print(result)
(2, 3, 4)
Example1:
>>> # ENABLE_DOCTEST
>>> list_ = [[[[[1]]], [3, 4, 33]], [[1], [2, 3], [4, [5, 5]]], [1, 3]]
>>> result = depth_profile(list_)
>>> print(result)
[[(1, 1, 1), 3], [1, 2, [1, 2]], 2]
Example2:
>>> # ENABLE_DOCTEST
>>> list_ = [[[[[1]]], [3, 4, 33]], [[1], [2, 3], [4, [5, 5]]], [1, 3]]
>>> result = depth_profile(list_, max_depth=1)
>>> print(result)
[[(1, '1'), 3], [1, 2, [1, '2']], 2]
Example3:
>>> # ENABLE_DOCTEST
>>> list_ = [[[1, 2], [1, 2, 3]], None]
>>> result = depth_profile(list_, compress_homogenous=True)
>>> print(result)
[[2, 3], 1]
Example4:
>>> # ENABLE_DOCTEST
>>> list_ = [[3, 2], [3, 2], [3, 2], [3, 2], [3, 2], [3, 2], [9, 5, 3], [2, 2]]
>>> result = depth_profile(list_, compress_homogenous=True, compress_consecutive=True)
>>> print(result)
[2] * 6 + [3, 2]
Example5:
>>> # ENABLE_DOCTEST
>>> list_ = [[[3, 9], 2], [[3, 9], 2], [[3, 9], 2], [[3, 9], 2]] #, [3, 2], [3, 2]]
>>> result = depth_profile(list_, compress_homogenous=True, compress_consecutive=True)
>>> print(result)
(4, [2, 1])
Example6:
>>> # ENABLE_DOCTEST
>>> list_ = [[[[1, 2]], [1, 2]], [[[1, 2]], [1, 2]], [[[0, 2]], [1]]]
>>> result1 = depth_profile(list_, compress_homogenous=True, compress_consecutive=False)
>>> result2 = depth_profile(list_, compress_homogenous=True, compress_consecutive=True)
>>> result = str(result1) + '\n' + str(result2)
>>> print(result)
[[(1, 2), 2], [(1, 2), 2], [(1, 2), 1]]
[[(1, 2), 2]] * 2 + [[(1, 2), 1]]
Example7:
>>> # ENABLE_DOCTEST
>>> list_ = [[{'a': [1, 2], 'b': [3, 4, 5]}, [1, 2, 3]], None]
>>> result = depth_profile(list_, compress_homogenous=True)
>>> print(result)
Example8:
>>> # ENABLE_DOCTEST
>>> list_ = [[[1]], [[[1, 1], [1, 1]]], [[[[1, 3], 1], [[1, 3, 3], 1, 1]]]]
>>> result = depth_profile(list_, compress_homogenous=True)
>>> print(result)
Example9:
>>> # ENABLE_DOCTEST
>>> list_ = []
>>> result = depth_profile(list_)
>>> print(result)
# THIS IS AN ERROR???
SHOULD BE
#[1, 1], [1, 2, 2], (1, ([1, 2]), (
Example10:
>>> # ENABLE_DOCTEST
>>> fm1 = [[0, 0], [0, 0]]
>>> fm2 = [[0, 0], [0, 0], [0, 0]]
>>> fm3 = [[0, 0], [0, 0], [0, 0], [0, 0]]
>>> list_ = [0, 0, 0]
>>> list_ = [fm1, fm2, fm3]
>>> max_depth = 0
>>> new_depth = True
>>> result = depth_profile(list_, max_depth=max_depth, new_depth=new_depth)
>>> print(result)
"""
if isinstance(list_, dict):
list_ = list(list_.values()) # handle dict
level_shape_list = []
# For a pure bottom level list return the length
if not any(map(util_type.is_listlike, list_)):
return len(list_)
if False and new_depth:
pass
# max_depth_ = None if max_depth is None else max_depth - 1
# if max_depth_ is None or max_depth_ > 0:
# pass
# else:
# for item in list_:
# if isinstance(item, dict):
# item = list(item.values()) # handle dict
# if util_type.is_listlike(item):
# if max_depth is None:
# level_shape_list.append(depth_profile(item, None))
# else:
# if max_depth >= 0:
# level_shape_list.append(depth_profile(item, max_depth - 1))
# else:
# level_shape_list.append(str(len(item)))
# else:
# level_shape_list.append(1)
else:
for item in list_:
if isinstance(item, dict):
item = list(item.values()) # handle dict
if util_type.is_listlike(item):
if max_depth is None:
level_shape_list.append(depth_profile(item, None))
else:
if max_depth >= 0:
level_shape_list.append(depth_profile(item, max_depth - 1))
else:
level_shape_list.append(str(len(item)))
else:
level_shape_list.append(1)
if compress_homogenous:
# removes redudant information by returning a shape duple
if allsame(level_shape_list):
dim_ = level_shape_list[0]
len_ = len(level_shape_list)
if isinstance(dim_, tuple):
level_shape_list = tuple([len_] + list(dim_))
else:
level_shape_list = tuple([len_, dim_])
if compress_consecutive:
hash_list = list(map(hash, map(str, level_shape_list)))
consec_list = group_consecutives(hash_list, 0)
if len(consec_list) != len(level_shape_list):
len_list = list(map(len, consec_list))
cumsum_list = np.cumsum(len_list)
consec_str = '['
thresh = 1
for len_, cumsum in zip(len_list, cumsum_list):
value = level_shape_list[cumsum - 1]
if len_ > thresh:
consec_str += str(value) + '] * ' + str(len_)
consec_str += ' + ['
else:
consec_str += str(value) + ', '
if consec_str.endswith(', '):
consec_str = consec_str[:-2]
#consec_str += ']'
#consec_str = consec_str.rstrip(', ').rstrip(']')
#consec_str = consec_str.rstrip(', ')
#if consec_str.endswith(']'):
# consec_str = consec_str[:-1]
consec_str += ']'
level_shape_list = consec_str
return level_shape_list
[docs]def list_type(list_):
types = unique_ordered(list(map(type, list_)))
if len(types) == 1:
return types[0]
else:
return types
[docs]def list_type_profile(sequence, compress_homogenous=True, with_dtype=True):
"""
similar to depth_profile but reports types
Args:
sequence (?):
compress_homogenous (bool): (default = True)
Kwargs:
compress_homogenous
Returns:
str: level_type_str
CommandLine:
python -m utool.util_list --exec-list_type_profile
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> sequence = []
>>> compress_homogenous = True
>>> level_type_str = list_type_profile(sequence, compress_homogenous)
>>> result = ('level_type_str = %s' % (str(level_type_str),))
>>> print(result)
"""
# For a pure bottom level list return the length
#if not any(map(util_type.is_listlike, sequence)) or (isinstance(sequence, np.ndarray) and sequence.dtype != object):
if not util_type.is_listlike(sequence) or (isinstance(sequence, np.ndarray) and sequence.dtype != object):
typename = str(type(sequence)).replace('<type \'', '').replace('\'>', '')
if with_dtype and typename == 'numpy.ndarray':
typename = typename.replace('numpy.', '')
typename += '[%s]' % (sequence.dtype,)
level_type_str = typename
return level_type_str
if len(sequence) == 0:
return ''
level_type_list = []
for item in sequence:
#if util_type.is_listlike(item):
level_type_list.append(list_type_profile(item, with_dtype=with_dtype))
if compress_homogenous:
# removes redudant information by returning a type and number
if allsame(level_type_list):
type_ = level_type_list[0]
level_type_str = str(type_) + '*' + str(len(level_type_list))
else:
level_type_str = ', '.join(level_type_list)
typename = str(type(sequence)).replace('<type \'', '').replace('\'>', '')
level_type_str = typename + '(' + str(level_type_str) + ')'
return level_type_str
type_profile = list_type_profile
[docs]def list_cover(list1, list2):
r"""
returns boolean for each position in list1 if it is in list2
Args:
list1 (list):
list2 (list):
Returns:
list: incover_list - true where list1 intersects list2
CommandLine:
python -m utool.util_list --test-list_cover
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> # build test data
>>> list1 = [1, 2, 3, 4, 5, 6]
>>> list2 = [2, 3, 6]
>>> # execute function
>>> incover_list = list_cover(list1, list2)
>>> # verify results
>>> result = str(incover_list)
>>> print(result)
[False, True, True, False, False, True]
"""
set2 = set(list2)
incover_list = [item1 in set2 for item1 in list1]
return incover_list
[docs]def and_lists(*args):
#[all(tup) for tup in zip(*args)]
return list(util_iter.and_iters(*args))
[docs]def xor_lists(*args):
r"""
Returns:
list:
CommandLine:
python -m utool.util_list --test-xor_lists
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> args = ([True, False, False, True], [True, True, False, False])
>>> result = xor_lists(*args)
>>> print(result)
[False, True, False, True]
"""
return [reduce(operator.xor, tup) for tup in zip(*args)]
[docs]def not_list(flag_list):
return [not flag for flag in flag_list]
[docs]def or_lists(*args):
return [any(tup) for tup in zip(*args)]
[docs]def make_sortby_func(item_list, reverse=False):
sortxs_ = argsort(item_list)
sortxs = sortxs_[::-1] if reverse else sortxs_
def sortby_func(list_):
return take(list_, sortxs)
return sortby_func
[docs]def filter_startswith(list_, str_):
def item_startswith(item):
return item.startswith(str_)
return list(filter(item_startswith, list_))
[docs]def list_roll(list_, n):
"""
Like numpy.roll for python lists
Args:
list_ (list):
n (int):
Returns:
list:
References:
http://stackoverflow.com/questions/9457832/python-list-rotation
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [1, 2, 3, 4, 5]
>>> n = 2
>>> result = list_roll(list_, n)
>>> print(result)
[4, 5, 1, 2, 3]
Ignore:
np.roll(list_, n)
"""
return list_[-n:] + list_[:-n]
[docs]def list_argmax(list_):
return np.argmax(np.array(list_))
[docs]def list_argmaxima(list_):
r"""
Args:
list_ (list):
Returns:
list: argmaxima
CommandLine:
python -m utool.util_list --exec-list_argmaxima
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = np.array([1, 2, 3, 3, 3, 2, 1])
>>> argmaxima = list_argmaxima(list_)
>>> result = ('argmaxima = %s' % (str(argmaxima),))
>>> print(result)
argmaxima = [2 3 4]
"""
argmax = list_argmax(list_)
maxval = list_[argmax]
argmaxima = np.where((np.isclose(maxval, list_)))[0]
return argmaxima
[docs]def make_index_lookup(list_):
r"""
Args:
list_ (list): assumed to have unique items
Returns:
dict: mapping from item to index
CommandLine:
python -m utool.util_list --exec-make_index_lookup
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> list_ = [5, 3, 8, 2]
>>> idx2_item = ut.make_index_lookup(list_)
>>> result = ut.dict_str(idx2_item, nl=False)
>>> assert ut.dict_take(idx2_item, list_) == list(range(len(list_)))
>>> print(result)
{2: 3, 3: 1, 5: 0, 8: 2}
"""
return dict(zip(list_, range(len(list_))))
[docs]def list_transpose(list_):
r"""
Args:
list_ (list):
Returns:
list:
CommandLine:
python -m utool.util_list --test-list_transpose
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [[1, 2], [3, 4]]
>>> result = list_transpose(list_)
>>> print(result)
[(1, 3), (2, 4)]
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [[1, 2, 3], [3, 4]]
>>> try:
>>> result = list_transpose(list_)
>>> except AssertionError:
>>> pass
>>> else:
>>> assert False, 'should error'
"""
#import utool as ut
num_cols_set = list(set(list(map(len, list_))))
assert len(num_cols_set) == 1, 'inconsistent num_cols_set=%r' % (num_cols_set,)
return list(zip(*list_))
[docs]def delete_items_by_index(list_, index_list):
"""
Args:
list_ (list):
index_list (list):
CommandLine:
python -m utool.util_list --exec-delete_items_by_index
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> list_ = [8, 1, 8, 1, 6, 6, 3, 4, 4, 5, 6]
>>> index_list = [2, -1]
>>> result = delete_items_by_index(list_, index_list)
>>> print(result)
[8, 1, 1, 6, 6, 3, 4, 4, 5]
"""
# Rectify negative indicies
index_list_ = [len(list_) + index if index < 0 else index for index in index_list]
# Remove largest indicies first
index_list_ = sorted(index_list_, reverse=True)
for index in index_list_:
del list_[index]
return list_
[docs]def unflat_map(func, unflat_items, vectorized=False, **kwargs):
r"""
Uses an ibeis lookup function with a non-flat rowid list.
In essence this is equivilent to [list(map(func, _items)) for _items in unflat_items].
The utility of this function is that it only calls method once.
This is more efficient for calls that can take a list of inputs
Args:
func (func): function
unflat_items (list): list of rowid lists
Returns:
list of values: unflat_vals
CommandLine:
python -m utool.util_list --test-unflat_map
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> vectorized = False
>>> kwargs = {}
>>> func = lambda x: x + 1
>>> unflat_items = [[], [1, 2, 3], [4, 5], [6, 7, 8, 9], [], []]
>>> unflat_vals = unflat_map(func, unflat_items)
>>> result = str(unflat_vals)
>>> print(result)
[[], [2, 3, 4], [5, 6], [7, 8, 9, 10], [], []]
"""
import utool as ut
# First flatten the list, and remember the original dimensions
flat_items, reverse_list = ut.invertible_flatten2(unflat_items)
# Then preform the lookup / implicit mapping
if vectorized:
# func is vectorized
flat_vals = func(flat_items, **kwargs)
else:
flat_vals = [func(item, **kwargs) for item in flat_items]
if True:
assert len(flat_vals) == len(flat_items), (
'flat lens not the same, len(flat_vals)=%d len(flat_items)=%d' %
(len(flat_vals), len(flat_items),))
# Then ut.unflatten2 the results to the original input dimensions
unflat_vals = ut.unflatten2(flat_vals, reverse_list)
if True:
assert len(unflat_vals) == len(unflat_items), (
'unflat lens not the same, len(unflat_vals)=%d len(unflat_rowids)=%d' %
(len(unflat_vals), len(unflat_items),))
return unflat_vals
[docs]def unflat_vecmap(func, unflat_items, vectorized=False, **kwargs):
""" unflat map for vectorized functions """
import utool as ut
# First flatten the list, and remember the original dimensions
flat_items, reverse_list = ut.invertible_flatten2(unflat_items)
# Then preform the lookup / implicit mapping
flat_vals = func(flat_items, **kwargs)
if True:
assert len(flat_vals) == len(flat_items), (
'flat lens not the same, len(flat_vals)=%d len(flat_items)=%d' %
(len(flat_vals), len(flat_items),))
# Then ut.unflatten2 the results to the original input dimensions
unflat_vals = ut.unflatten2(flat_vals, reverse_list)
if True:
assert len(unflat_vals) == len(unflat_items), (
'unflat lens not the same, len(unflat_vals)=%d len(unflat_rowids)=%d' %
(len(unflat_vals), len(unflat_items),))
return unflat_vals
[docs]def list_getattr(list_, attrname):
return list(map(operator.attrgetter(attrname), list_))
[docs]def list_reshape(list_, new_shape, trail=False):
r"""
reshapes leaving trailing dimnsions in front if prod(new_shape) != len(list_)
Args:
list_ (list):
new_shape (tuple):
Returns:
list: list_
CommandLine:
python -m utool.util_list --exec-list_reshape --show
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> import numpy as np
>>> list_ = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
>>> new_shape = (2, 2, 3)
>>> newlist = list_reshape(list_, new_shape)
>>> depth = ut.depth_profile(newlist)
>>> result = ('list_ = %s' % (ut.repr2(newlist, nl=1),))
>>> print('depth = %r' % (depth,))
>>> print(result)
>>> newlist2 = np.reshape(list_, depth).tolist()
>>> ut.assert_eq(newlist, newlist2)
"""
if not trail:
total = reduce(operator.mul, new_shape)
assert total == len(list_)
newlist = list_
for dim in reversed(new_shape):
slice_ = (newlist[i::dim] for i in range(dim))
newlist = list(map(list, zip(*slice_)))
if not trail:
newlist = newlist[0]
return newlist
[docs]def index_to_boolmask(index_list, maxval=None):
r"""
Args:
index_list (list):
maxval (None): (default = None)
Kwargs:
maxval
Returns:
list: mask
SeeAlso:
vt.index_to_boolmask numpy version
CommandLine:
python -m vtool.other --exec-index_to_boolmask
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_list import * # NOQA
>>> import utool as ut
>>> index_list = [0, 1, 4]
>>> maxval = 5
>>> mask = ut.index_to_boolmask(index_list, maxval)
>>> result = ('mask = %s' % (ut.repr2(mask, nl=0)))
>>> print(result)
mask = [True, True, False, False, True]
"""
if maxval is None:
maxval = max(index_list) + 1
mask = [False] * maxval
for index in index_list:
mask[index] = True
return mask
# Aliases
list_compress = compress
list_ziptake = ziptake
list_zipcompress = zipcompress
list_where = where
list_take = take
list_argsort = argsort
#def partition2(list_, idxs1, idxs2):
# list1_ = ut.take(list_, idxs1)
# list2_ = list(zip(ut.take(list_, idxs2)))
# partitioned_items = [list1_, list2_]
# return partitioned_items
if __name__ == '__main__':
"""
CommandLine:
python -c "import utool, utool.util_list; utool.doctest_funcs(utool.util_list, allexamples=True)"
python -c "import utool, utool.util_list; utool.doctest_funcs(utool.util_list)"
python -m utool.util_list
python -m utool.util_list --allexamples
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()