Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/celery/utils/functional.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""Functional-style utilties."""
3from __future__ import absolute_import, print_function, unicode_literals
5import inspect
6import sys
7from functools import partial
8from itertools import chain, islice
10from kombu.utils.functional import (LRUCache, dictfilter, is_list, lazy,
11 maybe_evaluate, maybe_list, memoize)
12from vine import promise
14from celery.five import UserList, getfullargspec, range
16__all__ = (
17 'LRUCache', 'is_list', 'maybe_list', 'memoize', 'mlazy', 'noop',
18 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq',
19 'regen', 'dictfilter', 'lazy', 'maybe_evaluate', 'head_from_fun',
20 'maybe', 'fun_accepts_kwargs',
21)
23FUNHEAD_TEMPLATE = """
24def {fun_name}({fun_args}):
25 return {fun_value}
26"""
29class DummyContext(object):
31 def __enter__(self):
32 return self
34 def __exit__(self, *exc_info):
35 pass
38class mlazy(lazy):
39 """Memoized lazy evaluation.
41 The function is only evaluated once, every subsequent access
42 will return the same value.
43 """
45 #: Set to :const:`True` after the object has been evaluated.
46 evaluated = False
47 _value = None
49 def evaluate(self):
50 if not self.evaluated:
51 self._value = super(mlazy, self).evaluate()
52 self.evaluated = True
53 return self._value
56def noop(*args, **kwargs):
57 """No operation.
59 Takes any arguments/keyword arguments and does nothing.
60 """
63def pass1(arg, *args, **kwargs):
64 """Return the first positional argument."""
65 return arg
68def evaluate_promises(it):
69 for value in it:
70 if isinstance(value, promise):
71 value = value()
72 yield value
75def first(predicate, it):
76 """Return the first element in ``it`` that ``predicate`` accepts.
78 If ``predicate`` is None it will return the first item that's not
79 :const:`None`.
80 """
81 return next(
82 (v for v in evaluate_promises(it) if (
83 predicate(v) if predicate is not None else v is not None)),
84 None,
85 )
88def firstmethod(method, on_call=None):
89 """Multiple dispatch.
91 Return a function that with a list of instances,
92 finds the first instance that gives a value for the given method.
94 The list can also contain lazy instances
95 (:class:`~kombu.utils.functional.lazy`.)
96 """
97 def _matcher(it, *args, **kwargs):
98 for obj in it:
99 try:
100 meth = getattr(maybe_evaluate(obj), method)
101 reply = (on_call(meth, *args, **kwargs) if on_call
102 else meth(*args, **kwargs))
103 except AttributeError:
104 pass
105 else:
106 if reply is not None:
107 return reply
108 return _matcher
111def chunks(it, n):
112 """Split an iterator into chunks with `n` elements each.
114 Warning:
115 ``it`` must be an actual iterator, if you pass this a
116 concrete sequence will get you repeating elements.
118 So ``chunks(iter(range(1000)), 10)`` is fine, but
119 ``chunks(range(1000), 10)`` is not.
121 Example:
122 # n == 2
123 >>> x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 2)
124 >>> list(x)
125 [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10]]
127 # n == 3
128 >>> x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 3)
129 >>> list(x)
130 [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10]]
131 """
132 for item in it:
133 yield [item] + list(islice(it, n - 1))
136def padlist(container, size, default=None):
137 """Pad list with default elements.
139 Example:
140 >>> first, last, city = padlist(['George', 'Costanza', 'NYC'], 3)
141 ('George', 'Costanza', 'NYC')
142 >>> first, last, city = padlist(['George', 'Costanza'], 3)
143 ('George', 'Costanza', None)
144 >>> first, last, city, planet = padlist(
145 ... ['George', 'Costanza', 'NYC'], 4, default='Earth',
146 ... )
147 ('George', 'Costanza', 'NYC', 'Earth')
148 """
149 return list(container)[:size] + [default] * (size - len(container))
152def mattrgetter(*attrs):
153 """Get attributes, ignoring attribute errors.
155 Like :func:`operator.itemgetter` but return :const:`None` on missing
156 attributes instead of raising :exc:`AttributeError`.
157 """
158 return lambda obj: {attr: getattr(obj, attr, None) for attr in attrs}
161def uniq(it):
162 """Return all unique elements in ``it``, preserving order."""
163 seen = set()
164 return (seen.add(obj) or obj for obj in it if obj not in seen)
167def regen(it):
168 """Convert iterator to an object that can be consumed multiple times.
170 ``Regen`` takes any iterable, and if the object is an
171 generator it will cache the evaluated list on first access,
172 so that the generator can be "consumed" multiple times.
173 """
174 if isinstance(it, (list, tuple)):
175 return it
176 return _regen(it)
179class _regen(UserList, list):
180 # must be subclass of list so that json can encode.
182 def __init__(self, it):
183 # pylint: disable=super-init-not-called
184 # UserList creates a new list and sets .data, so we don't
185 # want to call init here.
186 self.__it = it
187 self.__index = 0
188 self.__consumed = []
190 def __reduce__(self):
191 return list, (self.data,)
193 def __length_hint__(self):
194 return self.__it.__length_hint__()
196 def __iter__(self):
197 return chain(self.__consumed, self.__it)
199 def __getitem__(self, index):
200 if index < 0:
201 return self.data[index]
202 try:
203 return self.__consumed[index]
204 except IndexError:
205 try:
206 for _ in range(self.__index, index + 1):
207 self.__consumed.append(next(self.__it))
208 except StopIteration:
209 raise IndexError(index)
210 else:
211 return self.__consumed[index]
213 @property
214 def data(self):
215 try:
216 self.__consumed.extend(list(self.__it))
217 except StopIteration:
218 pass
219 return self.__consumed
222def _argsfromspec(spec, replace_defaults=True):
223 if spec.defaults:
224 split = len(spec.defaults)
225 defaults = (list(range(len(spec.defaults))) if replace_defaults
226 else spec.defaults)
227 positional = spec.args[:-split]
228 optional = list(zip(spec.args[-split:], defaults))
229 else:
230 positional, optional = spec.args, []
232 varargs = spec.varargs
233 varkw = spec.varkw
234 if spec.kwonlydefaults:
235 split = len(spec.kwonlydefaults)
236 kwonlyargs = spec.kwonlyargs[:-split]
237 if replace_defaults:
238 kwonlyargs_optional = [
239 (kw, i) for i, kw in enumerate(spec.kwonlyargs[-split:])]
240 else:
241 kwonlyargs_optional = list(spec.kwonlydefaults.items())
242 else:
243 kwonlyargs, kwonlyargs_optional = spec.kwonlyargs, []
245 return ', '.join(filter(None, [
246 ', '.join(positional),
247 ', '.join('{0}={1}'.format(k, v) for k, v in optional),
248 '*{0}'.format(varargs) if varargs else None,
249 '*' if (kwonlyargs or kwonlyargs_optional) and not varargs else None,
250 ', '.join(kwonlyargs) if kwonlyargs else None,
251 ', '.join('{0}="{1}"'.format(k, v) for k, v in kwonlyargs_optional),
252 '**{0}'.format(varkw) if varkw else None,
253 ]))
256def head_from_fun(fun, bound=False, debug=False):
257 """Generate signature function from actual function."""
258 # we could use inspect.Signature here, but that implementation
259 # is very slow since it implements the argument checking
260 # in pure-Python. Instead we use exec to create a new function
261 # with an empty body, meaning it has the same performance as
262 # as just calling a function.
263 is_function = inspect.isfunction(fun)
264 is_callable = hasattr(fun, '__call__')
265 is_cython = fun.__class__.__name__ == 'cython_function_or_method'
266 is_method = inspect.ismethod(fun)
268 if not is_function and is_callable and not is_method and not is_cython:
269 name, fun = fun.__class__.__name__, fun.__call__
270 else:
271 name = fun.__name__
272 definition = FUNHEAD_TEMPLATE.format(
273 fun_name=name,
274 fun_args=_argsfromspec(getfullargspec(fun)),
275 fun_value=1,
276 )
277 if debug: # pragma: no cover
278 print(definition, file=sys.stderr)
279 namespace = {'__name__': fun.__module__}
280 # pylint: disable=exec-used
281 # Tasks are rarely, if ever, created at runtime - exec here is fine.
282 exec(definition, namespace)
283 result = namespace[name]
284 result._source = definition
285 if bound:
286 return partial(result, object())
287 return result
290def arity_greater(fun, n):
291 argspec = getfullargspec(fun)
292 return argspec.varargs or len(argspec.args) > n
295def fun_takes_argument(name, fun, position=None):
296 spec = getfullargspec(fun)
297 return (
298 spec.varkw or spec.varargs or
299 (len(spec.args) >= position if position else name in spec.args)
300 )
303if hasattr(inspect, 'signature'):
304 def fun_accepts_kwargs(fun):
305 """Return true if function accepts arbitrary keyword arguments."""
306 return any(
307 p for p in inspect.signature(fun).parameters.values()
308 if p.kind == p.VAR_KEYWORD
309 )
310else:
311 def fun_accepts_kwargs(fun): # noqa
312 """Return true if function accepts arbitrary keyword arguments."""
313 try:
314 argspec = inspect.getargspec(fun)
315 except TypeError:
316 try:
317 argspec = inspect.getargspec(fun.__call__)
318 except (TypeError, AttributeError):
319 return
320 return not argspec or argspec[2] is not None
323def maybe(typ, val):
324 """Call typ on value if val is defined."""
325 return typ(val) if val is not None else val
328def seq_concat_item(seq, item):
329 """Return copy of sequence seq with item added.
331 Returns:
332 Sequence: if seq is a tuple, the result will be a tuple,
333 otherwise it depends on the implementation of ``__add__``.
334 """
335 return seq + (item,) if isinstance(seq, tuple) else seq + [item]
338def seq_concat_seq(a, b):
339 """Concatenate two sequences: ``a + b``.
341 Returns:
342 Sequence: The return value will depend on the largest sequence
343 - if b is larger and is a tuple, the return value will be a tuple.
344 - if a is larger and is a list, the return value will be a list,
345 """
346 # find the type of the largest sequence
347 prefer = type(max([a, b], key=len))
348 # convert the smallest list to the type of the largest sequence.
349 if not isinstance(a, prefer):
350 a = prefer(a)
351 if not isinstance(b, prefer):
352 b = prefer(b)
353 return a + b