Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2"""Functional-style utilties.""" 

3from __future__ import absolute_import, print_function, unicode_literals 

4 

5import inspect 

6import sys 

7from functools import partial 

8from itertools import chain, islice 

9 

10from kombu.utils.functional import (LRUCache, dictfilter, is_list, lazy, 

11 maybe_evaluate, maybe_list, memoize) 

12from vine import promise 

13 

14from celery.five import UserList, getfullargspec, range 

15 

16__all__ = ( 

17 'LRUCache', 'is_list', 'maybe_list', 'memoize', 'mlazy', 'noop', 

18 'first', 'firstmethod', 'chunks', 'padlist', 'mattrgetter', 'uniq', 

19 'regen', 'dictfilter', 'lazy', 'maybe_evaluate', 'head_from_fun', 

20 'maybe', 'fun_accepts_kwargs', 

21) 

22 

23FUNHEAD_TEMPLATE = """ 

24def {fun_name}({fun_args}): 

25 return {fun_value} 

26""" 

27 

28 

29class DummyContext(object): 

30 

31 def __enter__(self): 

32 return self 

33 

34 def __exit__(self, *exc_info): 

35 pass 

36 

37 

38class mlazy(lazy): 

39 """Memoized lazy evaluation. 

40 

41 The function is only evaluated once, every subsequent access 

42 will return the same value. 

43 """ 

44 

45 #: Set to :const:`True` after the object has been evaluated. 

46 evaluated = False 

47 _value = None 

48 

49 def evaluate(self): 

50 if not self.evaluated: 

51 self._value = super(mlazy, self).evaluate() 

52 self.evaluated = True 

53 return self._value 

54 

55 

56def noop(*args, **kwargs): 

57 """No operation. 

58 

59 Takes any arguments/keyword arguments and does nothing. 

60 """ 

61 

62 

63def pass1(arg, *args, **kwargs): 

64 """Return the first positional argument.""" 

65 return arg 

66 

67 

68def evaluate_promises(it): 

69 for value in it: 

70 if isinstance(value, promise): 

71 value = value() 

72 yield value 

73 

74 

75def first(predicate, it): 

76 """Return the first element in ``it`` that ``predicate`` accepts. 

77 

78 If ``predicate`` is None it will return the first item that's not 

79 :const:`None`. 

80 """ 

81 return next( 

82 (v for v in evaluate_promises(it) if ( 

83 predicate(v) if predicate is not None else v is not None)), 

84 None, 

85 ) 

86 

87 

88def firstmethod(method, on_call=None): 

89 """Multiple dispatch. 

90 

91 Return a function that with a list of instances, 

92 finds the first instance that gives a value for the given method. 

93 

94 The list can also contain lazy instances 

95 (:class:`~kombu.utils.functional.lazy`.) 

96 """ 

97 def _matcher(it, *args, **kwargs): 

98 for obj in it: 

99 try: 

100 meth = getattr(maybe_evaluate(obj), method) 

101 reply = (on_call(meth, *args, **kwargs) if on_call 

102 else meth(*args, **kwargs)) 

103 except AttributeError: 

104 pass 

105 else: 

106 if reply is not None: 

107 return reply 

108 return _matcher 

109 

110 

111def chunks(it, n): 

112 """Split an iterator into chunks with `n` elements each. 

113 

114 Warning: 

115 ``it`` must be an actual iterator, if you pass this a 

116 concrete sequence will get you repeating elements. 

117 

118 So ``chunks(iter(range(1000)), 10)`` is fine, but 

119 ``chunks(range(1000), 10)`` is not. 

120 

121 Example: 

122 # n == 2 

123 >>> x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 2) 

124 >>> list(x) 

125 [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10]] 

126 

127 # n == 3 

128 >>> x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 3) 

129 >>> list(x) 

130 [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10]] 

131 """ 

132 for item in it: 

133 yield [item] + list(islice(it, n - 1)) 

134 

135 

136def padlist(container, size, default=None): 

137 """Pad list with default elements. 

138 

139 Example: 

140 >>> first, last, city = padlist(['George', 'Costanza', 'NYC'], 3) 

141 ('George', 'Costanza', 'NYC') 

142 >>> first, last, city = padlist(['George', 'Costanza'], 3) 

143 ('George', 'Costanza', None) 

144 >>> first, last, city, planet = padlist( 

145 ... ['George', 'Costanza', 'NYC'], 4, default='Earth', 

146 ... ) 

147 ('George', 'Costanza', 'NYC', 'Earth') 

148 """ 

149 return list(container)[:size] + [default] * (size - len(container)) 

150 

151 

152def mattrgetter(*attrs): 

153 """Get attributes, ignoring attribute errors. 

154 

155 Like :func:`operator.itemgetter` but return :const:`None` on missing 

156 attributes instead of raising :exc:`AttributeError`. 

157 """ 

158 return lambda obj: {attr: getattr(obj, attr, None) for attr in attrs} 

159 

160 

161def uniq(it): 

162 """Return all unique elements in ``it``, preserving order.""" 

163 seen = set() 

164 return (seen.add(obj) or obj for obj in it if obj not in seen) 

165 

166 

167def regen(it): 

168 """Convert iterator to an object that can be consumed multiple times. 

169 

170 ``Regen`` takes any iterable, and if the object is an 

171 generator it will cache the evaluated list on first access, 

172 so that the generator can be "consumed" multiple times. 

173 """ 

174 if isinstance(it, (list, tuple)): 

175 return it 

176 return _regen(it) 

177 

178 

179class _regen(UserList, list): 

180 # must be subclass of list so that json can encode. 

181 

182 def __init__(self, it): 

183 # pylint: disable=super-init-not-called 

184 # UserList creates a new list and sets .data, so we don't 

185 # want to call init here. 

186 self.__it = it 

187 self.__index = 0 

188 self.__consumed = [] 

189 

190 def __reduce__(self): 

191 return list, (self.data,) 

192 

193 def __length_hint__(self): 

194 return self.__it.__length_hint__() 

195 

196 def __iter__(self): 

197 return chain(self.__consumed, self.__it) 

198 

199 def __getitem__(self, index): 

200 if index < 0: 

201 return self.data[index] 

202 try: 

203 return self.__consumed[index] 

204 except IndexError: 

205 try: 

206 for _ in range(self.__index, index + 1): 

207 self.__consumed.append(next(self.__it)) 

208 except StopIteration: 

209 raise IndexError(index) 

210 else: 

211 return self.__consumed[index] 

212 

213 @property 

214 def data(self): 

215 try: 

216 self.__consumed.extend(list(self.__it)) 

217 except StopIteration: 

218 pass 

219 return self.__consumed 

220 

221 

222def _argsfromspec(spec, replace_defaults=True): 

223 if spec.defaults: 

224 split = len(spec.defaults) 

225 defaults = (list(range(len(spec.defaults))) if replace_defaults 

226 else spec.defaults) 

227 positional = spec.args[:-split] 

228 optional = list(zip(spec.args[-split:], defaults)) 

229 else: 

230 positional, optional = spec.args, [] 

231 

232 varargs = spec.varargs 

233 varkw = spec.varkw 

234 if spec.kwonlydefaults: 

235 split = len(spec.kwonlydefaults) 

236 kwonlyargs = spec.kwonlyargs[:-split] 

237 if replace_defaults: 

238 kwonlyargs_optional = [ 

239 (kw, i) for i, kw in enumerate(spec.kwonlyargs[-split:])] 

240 else: 

241 kwonlyargs_optional = list(spec.kwonlydefaults.items()) 

242 else: 

243 kwonlyargs, kwonlyargs_optional = spec.kwonlyargs, [] 

244 

245 return ', '.join(filter(None, [ 

246 ', '.join(positional), 

247 ', '.join('{0}={1}'.format(k, v) for k, v in optional), 

248 '*{0}'.format(varargs) if varargs else None, 

249 '*' if (kwonlyargs or kwonlyargs_optional) and not varargs else None, 

250 ', '.join(kwonlyargs) if kwonlyargs else None, 

251 ', '.join('{0}="{1}"'.format(k, v) for k, v in kwonlyargs_optional), 

252 '**{0}'.format(varkw) if varkw else None, 

253 ])) 

254 

255 

256def head_from_fun(fun, bound=False, debug=False): 

257 """Generate signature function from actual function.""" 

258 # we could use inspect.Signature here, but that implementation 

259 # is very slow since it implements the argument checking 

260 # in pure-Python. Instead we use exec to create a new function 

261 # with an empty body, meaning it has the same performance as 

262 # as just calling a function. 

263 is_function = inspect.isfunction(fun) 

264 is_callable = hasattr(fun, '__call__') 

265 is_cython = fun.__class__.__name__ == 'cython_function_or_method' 

266 is_method = inspect.ismethod(fun) 

267 

268 if not is_function and is_callable and not is_method and not is_cython: 

269 name, fun = fun.__class__.__name__, fun.__call__ 

270 else: 

271 name = fun.__name__ 

272 definition = FUNHEAD_TEMPLATE.format( 

273 fun_name=name, 

274 fun_args=_argsfromspec(getfullargspec(fun)), 

275 fun_value=1, 

276 ) 

277 if debug: # pragma: no cover 

278 print(definition, file=sys.stderr) 

279 namespace = {'__name__': fun.__module__} 

280 # pylint: disable=exec-used 

281 # Tasks are rarely, if ever, created at runtime - exec here is fine. 

282 exec(definition, namespace) 

283 result = namespace[name] 

284 result._source = definition 

285 if bound: 

286 return partial(result, object()) 

287 return result 

288 

289 

290def arity_greater(fun, n): 

291 argspec = getfullargspec(fun) 

292 return argspec.varargs or len(argspec.args) > n 

293 

294 

295def fun_takes_argument(name, fun, position=None): 

296 spec = getfullargspec(fun) 

297 return ( 

298 spec.varkw or spec.varargs or 

299 (len(spec.args) >= position if position else name in spec.args) 

300 ) 

301 

302 

303if hasattr(inspect, 'signature'): 

304 def fun_accepts_kwargs(fun): 

305 """Return true if function accepts arbitrary keyword arguments.""" 

306 return any( 

307 p for p in inspect.signature(fun).parameters.values() 

308 if p.kind == p.VAR_KEYWORD 

309 ) 

310else: 

311 def fun_accepts_kwargs(fun): # noqa 

312 """Return true if function accepts arbitrary keyword arguments.""" 

313 try: 

314 argspec = inspect.getargspec(fun) 

315 except TypeError: 

316 try: 

317 argspec = inspect.getargspec(fun.__call__) 

318 except (TypeError, AttributeError): 

319 return 

320 return not argspec or argspec[2] is not None 

321 

322 

323def maybe(typ, val): 

324 """Call typ on value if val is defined.""" 

325 return typ(val) if val is not None else val 

326 

327 

328def seq_concat_item(seq, item): 

329 """Return copy of sequence seq with item added. 

330 

331 Returns: 

332 Sequence: if seq is a tuple, the result will be a tuple, 

333 otherwise it depends on the implementation of ``__add__``. 

334 """ 

335 return seq + (item,) if isinstance(seq, tuple) else seq + [item] 

336 

337 

338def seq_concat_seq(a, b): 

339 """Concatenate two sequences: ``a + b``. 

340 

341 Returns: 

342 Sequence: The return value will depend on the largest sequence 

343 - if b is larger and is a tuple, the return value will be a tuple. 

344 - if a is larger and is a list, the return value will be a list, 

345 """ 

346 # find the type of the largest sequence 

347 prefer = type(max([a, b], key=len)) 

348 # convert the smallest list to the type of the largest sequence. 

349 if not isinstance(a, prefer): 

350 a = prefer(a) 

351 if not isinstance(b, prefer): 

352 b = prefer(b) 

353 return a + b