Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/celery/canvas.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""Composing task work-flows.
4.. seealso:
6 You should import these from :mod:`celery` and not this module.
7"""
8from __future__ import absolute_import, unicode_literals
10import itertools
11import operator
12from collections import deque
13from copy import deepcopy
14from functools import partial as _partial
15from functools import reduce
16from operator import itemgetter
18from kombu.utils.functional import fxrange, reprcall
19from kombu.utils.objects import cached_property
20from kombu.utils.uuid import uuid
21from vine import barrier
23from celery._state import current_app
24from celery.five import PY3, python_2_unicode_compatible
25from celery.local import try_import
26from celery.result import GroupResult, allow_join_result
27from celery.utils import abstract
28from celery.utils.collections import ChainMap
29from celery.utils.functional import _regen
30from celery.utils.functional import chunks as _chunks
31from celery.utils.functional import (is_list, maybe_list, regen,
32 seq_concat_item, seq_concat_seq)
33from celery.utils.objects import getitem_property
34from celery.utils.text import remove_repeating_from_task, truncate
36try:
37 from collections.abc import MutableSequence
38except ImportError:
39 # TODO: Remove this when we drop Python 2.7 support
40 from collections import MutableSequence
42__all__ = (
43 'Signature', 'chain', 'xmap', 'xstarmap', 'chunks',
44 'group', 'chord', 'signature', 'maybe_signature',
45)
47# json in Python 2.7 borks if dict contains byte keys.
48JSON_NEEDS_UNICODE_KEYS = PY3 and not try_import('simplejson')
51def maybe_unroll_group(group):
52 """Unroll group with only one member."""
53 # Issue #1656
54 try:
55 size = len(group.tasks)
56 except TypeError:
57 try:
58 size = group.tasks.__length_hint__()
59 except (AttributeError, TypeError):
60 return group
61 else:
62 return list(group.tasks)[0] if size == 1 else group
63 else:
64 return group.tasks[0] if size == 1 else group
67def task_name_from(task):
68 return getattr(task, 'name', task)
71def _upgrade(fields, sig):
72 """Used by custom signatures in .from_dict, to keep common fields."""
73 sig.update(chord_size=fields.get('chord_size'))
74 return sig
77@abstract.CallableSignature.register
78@python_2_unicode_compatible
79class Signature(dict):
80 """Task Signature.
82 Class that wraps the arguments and execution options
83 for a single task invocation.
85 Used as the parts in a :class:`group` and other constructs,
86 or to pass tasks around as callbacks while being compatible
87 with serializers with a strict type subset.
89 Signatures can also be created from tasks:
91 - Using the ``.signature()`` method that has the same signature
92 as ``Task.apply_async``:
94 .. code-block:: pycon
96 >>> add.signature(args=(1,), kwargs={'kw': 2}, options={})
98 - or the ``.s()`` shortcut that works for star arguments:
100 .. code-block:: pycon
102 >>> add.s(1, kw=2)
104 - the ``.s()`` shortcut does not allow you to specify execution options
105 but there's a chaning `.set` method that returns the signature:
107 .. code-block:: pycon
109 >>> add.s(2, 2).set(countdown=10).set(expires=30).delay()
111 Note:
112 You should use :func:`~celery.signature` to create new signatures.
113 The ``Signature`` class is the type returned by that function and
114 should be used for ``isinstance`` checks for signatures.
116 See Also:
117 :ref:`guide-canvas` for the complete guide.
119 Arguments:
120 task (Union[Type[celery.app.task.Task], str]): Either a task
121 class/instance, or the name of a task.
122 args (Tuple): Positional arguments to apply.
123 kwargs (Dict): Keyword arguments to apply.
124 options (Dict): Additional options to :meth:`Task.apply_async`.
126 Note:
127 If the first argument is a :class:`dict`, the other
128 arguments will be ignored and the values in the dict will be used
129 instead::
131 >>> s = signature('tasks.add', args=(2, 2))
132 >>> signature(s)
133 {'task': 'tasks.add', args=(2, 2), kwargs={}, options={}}
134 """
136 TYPES = {}
137 _app = _type = None
139 @classmethod
140 def register_type(cls, name=None):
141 def _inner(subclass):
142 cls.TYPES[name or subclass.__name__] = subclass
143 return subclass
145 return _inner
147 @classmethod
148 def from_dict(cls, d, app=None):
149 typ = d.get('subtask_type')
150 if typ:
151 target_cls = cls.TYPES[typ]
152 if target_cls is not cls:
153 return target_cls.from_dict(d, app=app)
154 return Signature(d, app=app)
156 def __init__(self, task=None, args=None, kwargs=None, options=None,
157 type=None, subtask_type=None, immutable=False,
158 app=None, **ex):
159 self._app = app
161 if isinstance(task, dict):
162 super(Signature, self).__init__(task) # works like dict(d)
163 else:
164 # Also supports using task class/instance instead of string name.
165 try:
166 task_name = task.name
167 except AttributeError:
168 task_name = task
169 else:
170 self._type = task
172 super(Signature, self).__init__(
173 task=task_name, args=tuple(args or ()),
174 kwargs=kwargs or {},
175 options=dict(options or {}, **ex),
176 subtask_type=subtask_type,
177 immutable=immutable,
178 chord_size=None,
179 )
181 def __call__(self, *partial_args, **partial_kwargs):
182 """Call the task directly (in the current process)."""
183 args, kwargs, _ = self._merge(partial_args, partial_kwargs, None)
184 return self.type(*args, **kwargs)
186 def delay(self, *partial_args, **partial_kwargs):
187 """Shortcut to :meth:`apply_async` using star arguments."""
188 return self.apply_async(partial_args, partial_kwargs)
190 def apply(self, args=None, kwargs=None, **options):
191 """Call task locally.
193 Same as :meth:`apply_async` but executed the task inline instead
194 of sending a task message.
195 """
196 args = args if args else ()
197 kwargs = kwargs if kwargs else {}
198 # Extra options set to None are dismissed
199 options = {k: v for k, v in options.items() if v is not None}
200 # For callbacks: extra args are prepended to the stored args.
201 args, kwargs, options = self._merge(args, kwargs, options)
202 return self.type.apply(args, kwargs, **options)
204 def apply_async(self, args=None, kwargs=None, route_name=None, **options):
205 """Apply this task asynchronously.
207 Arguments:
208 args (Tuple): Partial args to be prepended to the existing args.
209 kwargs (Dict): Partial kwargs to be merged with existing kwargs.
210 options (Dict): Partial options to be merged
211 with existing options.
213 Returns:
214 ~@AsyncResult: promise of future evaluation.
216 See also:
217 :meth:`~@Task.apply_async` and the :ref:`guide-calling` guide.
218 """
219 args = args if args else ()
220 kwargs = kwargs if kwargs else {}
221 # Extra options set to None are dismissed
222 options = {k: v for k, v in options.items() if v is not None}
223 try:
224 _apply = self._apply_async
225 except IndexError: # pragma: no cover
226 # no tasks for chain, etc to find type
227 return
228 # For callbacks: extra args are prepended to the stored args.
229 if args or kwargs or options:
230 args, kwargs, options = self._merge(args, kwargs, options)
231 else:
232 args, kwargs, options = self.args, self.kwargs, self.options
233 # pylint: disable=too-many-function-args
234 # Borks on this, as it's a property
235 return _apply(args, kwargs, **options)
237 def _merge(self, args=None, kwargs=None, options=None, force=False):
238 args = args if args else ()
239 kwargs = kwargs if kwargs else {}
240 options = options if options else {}
241 if self.immutable and not force:
242 return (self.args, self.kwargs,
243 dict(self.options,
244 **options) if options else self.options)
245 return (tuple(args) + tuple(self.args) if args else self.args,
246 dict(self.kwargs, **kwargs) if kwargs else self.kwargs,
247 dict(self.options, **options) if options else self.options)
249 def clone(self, args=None, kwargs=None, **opts):
250 """Create a copy of this signature.
252 Arguments:
253 args (Tuple): Partial args to be prepended to the existing args.
254 kwargs (Dict): Partial kwargs to be merged with existing kwargs.
255 options (Dict): Partial options to be merged with
256 existing options.
257 """
258 args = args if args else ()
259 kwargs = kwargs if kwargs else {}
260 # need to deepcopy options so origins links etc. is not modified.
261 if args or kwargs or opts:
262 args, kwargs, opts = self._merge(args, kwargs, opts)
263 else:
264 args, kwargs, opts = self.args, self.kwargs, self.options
265 signature = Signature.from_dict({'task': self.task,
266 'args': tuple(args),
267 'kwargs': kwargs,
268 'options': deepcopy(opts),
269 'subtask_type': self.subtask_type,
270 'chord_size': self.chord_size,
271 'immutable': self.immutable},
272 app=self._app)
273 signature._type = self._type
274 return signature
276 partial = clone
278 def freeze(self, _id=None, group_id=None, chord=None,
279 root_id=None, parent_id=None):
280 """Finalize the signature by adding a concrete task id.
282 The task won't be called and you shouldn't call the signature
283 twice after freezing it as that'll result in two task messages
284 using the same task id.
286 Returns:
287 ~@AsyncResult: promise of future evaluation.
288 """
289 # pylint: disable=redefined-outer-name
290 # XXX chord is also a class in outer scope.
291 opts = self.options
292 try:
293 tid = opts['task_id']
294 except KeyError:
295 tid = opts['task_id'] = _id or uuid()
296 if root_id:
297 opts['root_id'] = root_id
298 if parent_id:
299 opts['parent_id'] = parent_id
300 if 'reply_to' not in opts:
301 opts['reply_to'] = self.app.oid
302 if group_id:
303 opts['group_id'] = group_id
304 if chord:
305 opts['chord'] = chord
306 # pylint: disable=too-many-function-args
307 # Borks on this, as it's a property.
308 return self.AsyncResult(tid)
310 _freeze = freeze
312 def replace(self, args=None, kwargs=None, options=None):
313 """Replace the args, kwargs or options set for this signature.
315 These are only replaced if the argument for the section is
316 not :const:`None`.
317 """
318 signature = self.clone()
319 if args is not None:
320 signature.args = args
321 if kwargs is not None:
322 signature.kwargs = kwargs
323 if options is not None:
324 signature.options = options
325 return signature
327 def set(self, immutable=None, **options):
328 """Set arbitrary execution options (same as ``.options.update(…)``).
330 Returns:
331 Signature: This is a chaining method call
332 (i.e., it will return ``self``).
333 """
334 if immutable is not None:
335 self.set_immutable(immutable)
336 self.options.update(options)
337 return self
339 def set_immutable(self, immutable):
340 self.immutable = immutable
342 def _with_list_option(self, key):
343 items = self.options.setdefault(key, [])
344 if not isinstance(items, MutableSequence):
345 items = self.options[key] = [items]
346 return items
348 def append_to_list_option(self, key, value):
349 items = self._with_list_option(key)
350 if value not in items:
351 items.append(value)
352 return value
354 def extend_list_option(self, key, value):
355 items = self._with_list_option(key)
356 items.extend(maybe_list(value))
358 def link(self, callback):
359 """Add callback task to be applied if this task succeeds.
361 Returns:
362 Signature: the argument passed, for chaining
363 or use with :func:`~functools.reduce`.
364 """
365 return self.append_to_list_option('link', callback)
367 def link_error(self, errback):
368 """Add callback task to be applied on error in task execution.
370 Returns:
371 Signature: the argument passed, for chaining
372 or use with :func:`~functools.reduce`.
373 """
374 return self.append_to_list_option('link_error', errback)
376 def on_error(self, errback):
377 """Version of :meth:`link_error` that supports chaining.
379 on_error chains the original signature, not the errback so::
381 >>> add.s(2, 2).on_error(errback.s()).delay()
383 calls the ``add`` task, not the ``errback`` task, but the
384 reverse is true for :meth:`link_error`.
385 """
386 self.link_error(errback)
387 return self
389 def flatten_links(self):
390 """Return a recursive list of dependencies.
392 "unchain" if you will, but with links intact.
393 """
394 return list(itertools.chain.from_iterable(itertools.chain(
395 [[self]],
396 (link.flatten_links()
397 for link in maybe_list(self.options.get('link')) or [])
398 )))
400 def __or__(self, other):
401 # These could be implemented in each individual class,
402 # I'm sure, but for now we have this.
403 if isinstance(self, group):
404 # group() | task -> chord
405 return chord(self, body=other, app=self._app)
406 elif isinstance(other, group):
407 # unroll group with one member
408 other = maybe_unroll_group(other)
409 if isinstance(self, _chain):
410 # chain | group() -> chain
411 tasks = self.unchain_tasks()
412 if not tasks:
413 # If the chain is empty, return the group
414 return other
415 return _chain(seq_concat_item(
416 tasks, other), app=self._app)
417 # task | group() -> chain
418 return _chain(self, other, app=self.app)
420 if not isinstance(self, _chain) and isinstance(other, _chain):
421 # task | chain -> chain
422 return _chain(seq_concat_seq(
423 (self,), other.unchain_tasks()), app=self._app)
424 elif isinstance(other, _chain):
425 # chain | chain -> chain
426 return _chain(seq_concat_seq(
427 self.unchain_tasks(), other.unchain_tasks()), app=self._app)
428 elif isinstance(self, chord):
429 # chord | task -> attach to body
430 sig = self.clone()
431 sig.body = sig.body | other
432 return sig
433 elif isinstance(other, Signature):
434 if isinstance(self, _chain):
435 if self.tasks and isinstance(self.tasks[-1], group):
436 # CHAIN [last item is group] | TASK -> chord
437 sig = self.clone()
438 sig.tasks[-1] = chord(
439 sig.tasks[-1], other, app=self._app)
440 return sig
441 elif self.tasks and isinstance(self.tasks[-1], chord):
442 # CHAIN [last item is chord] -> chain with chord body.
443 sig = self.clone()
444 sig.tasks[-1].body = sig.tasks[-1].body | other
445 return sig
446 else:
447 # chain | task -> chain
448 return _chain(seq_concat_item(
449 self.unchain_tasks(), other), app=self._app)
450 # task | task -> chain
451 return _chain(self, other, app=self._app)
452 return NotImplemented
454 def election(self):
455 type = self.type
456 app = type.app
457 tid = self.options.get('task_id') or uuid()
459 with app.producer_or_acquire(None) as producer:
460 props = type.backend.on_task_call(producer, tid)
461 app.control.election(tid, 'task',
462 self.clone(task_id=tid, **props),
463 connection=producer.connection)
464 return type.AsyncResult(tid)
466 def reprcall(self, *args, **kwargs):
467 args, kwargs, _ = self._merge(args, kwargs, {}, force=True)
468 return reprcall(self['task'], args, kwargs)
470 def __deepcopy__(self, memo):
471 memo[id(self)] = self
472 return dict(self)
474 def __invert__(self):
475 return self.apply_async().get()
477 def __reduce__(self):
478 # for serialization, the task type is lazily loaded,
479 # and not stored in the dict itself.
480 return signature, (dict(self),)
482 def __json__(self):
483 return dict(self)
485 def __repr__(self):
486 return self.reprcall()
488 if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
489 def items(self):
490 for k, v in dict.items(self):
491 yield k.decode() if isinstance(k, bytes) else k, v
493 @property
494 def name(self):
495 # for duck typing compatibility with Task.name
496 return self.task
498 @cached_property
499 def type(self):
500 return self._type or self.app.tasks[self['task']]
502 @cached_property
503 def app(self):
504 return self._app or current_app
506 @cached_property
507 def AsyncResult(self):
508 try:
509 return self.type.AsyncResult
510 except KeyError: # task not registered
511 return self.app.AsyncResult
513 @cached_property
514 def _apply_async(self):
515 try:
516 return self.type.apply_async
517 except KeyError:
518 return _partial(self.app.send_task, self['task'])
520 id = getitem_property('options.task_id', 'Task UUID')
521 parent_id = getitem_property('options.parent_id', 'Task parent UUID.')
522 root_id = getitem_property('options.root_id', 'Task root UUID.')
523 task = getitem_property('task', 'Name of task.')
524 args = getitem_property('args', 'Positional arguments to task.')
525 kwargs = getitem_property('kwargs', 'Keyword arguments to task.')
526 options = getitem_property('options', 'Task execution options.')
527 subtask_type = getitem_property('subtask_type', 'Type of signature')
528 chord_size = getitem_property(
529 'chord_size', 'Size of chord (if applicable)')
530 immutable = getitem_property(
531 'immutable', 'Flag set if no longer accepts new arguments')
534def _prepare_chain_from_options(options, tasks, use_link):
535 # When we publish groups we reuse the same options dictionary for all of
536 # the tasks in the group. See:
537 # https://github.com/celery/celery/blob/fb37cb0b8/celery/canvas.py#L1022.
538 # Issue #5354 reported that the following type of canvases
539 # causes a Celery worker to hang:
540 # group(
541 # add.s(1, 1),
542 # add.s(1, 1)
543 # ) | tsum.s() | add.s(1) | group(add.s(1), add.s(1))
544 # The resolution of #5354 in PR #5681 was to only set the `chain` key
545 # in the options dictionary if it is not present.
546 # Otherwise we extend the existing list of tasks in the chain with the new
547 # tasks: options['chain'].extend(chain_).
548 # Before PR #5681 we overrode the `chain` key in each iteration
549 # of the loop which applies all the tasks in the group:
550 # options['chain'] = tasks if not use_link else None
551 # This caused Celery to execute chains correctly in most cases since
552 # in each iteration the `chain` key would reset itself to a new value
553 # and the side effect of mutating the key did not propagate
554 # to the next task in the group.
555 # Since we now mutated the `chain` key, a *list* which is passed
556 # by *reference*, the next task in the group will extend the list
557 # of tasks in the chain instead of setting a new one from the chain_
558 # variable above.
559 # This causes Celery to execute a chain, even though there might not be
560 # one to begin with. Alternatively, it causes Celery to execute more tasks
561 # that were previously present in the previous task in the group.
562 # The solution is to be careful and never mutate the options dictionary
563 # to begin with.
564 # Here is an example of a canvas which triggers this issue:
565 # add.s(5, 6) | group((add.s(1) | add.s(2), add.s(3))).
566 # The expected result is [14, 14]. However, when we extend the `chain`
567 # key the `add.s(3)` task erroneously has `add.s(2)` in its chain since
568 # it was previously applied to `add.s(1)`.
569 # Without being careful not to mutate the options dictionary, the result
570 # in this case is [16, 14].
571 # To avoid deep-copying the entire options dictionary every single time we
572 # run a chain we use a ChainMap and ensure that we never mutate
573 # the original `chain` key, hence we use list_a + list_b to create a new
574 # list.
575 if use_link:
576 return ChainMap({'chain': None}, options)
577 elif 'chain' not in options:
578 return ChainMap({'chain': tasks}, options)
579 elif tasks is not None:
580 # chain option may already be set, resulting in
581 # "multiple values for keyword argument 'chain'" error.
582 # Issue #3379.
583 # If a chain already exists, we need to extend it with the next
584 # tasks in the chain.
585 # Issue #5354.
586 # WARNING: Be careful not to mutate `options['chain']`.
587 return ChainMap({'chain': options['chain'] + tasks},
588 options)
591@Signature.register_type(name='chain')
592@python_2_unicode_compatible
593class _chain(Signature):
594 tasks = getitem_property('kwargs.tasks', 'Tasks in chain.')
596 @classmethod
597 def from_dict(cls, d, app=None):
598 tasks = d['kwargs']['tasks']
599 if tasks:
600 if isinstance(tasks, tuple): # aaaargh
601 tasks = d['kwargs']['tasks'] = list(tasks)
602 tasks = [maybe_signature(task, app=app) for task in tasks]
603 return _upgrade(d, _chain(tasks, app=app, **d['options']))
605 def __init__(self, *tasks, **options):
606 tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
607 else tasks)
608 Signature.__init__(
609 self, 'celery.chain', (), {'tasks': tasks}, **options
610 )
611 self._use_link = options.pop('use_link', None)
612 self.subtask_type = 'chain'
613 self._frozen = None
615 def __call__(self, *args, **kwargs):
616 if self.tasks:
617 return self.apply_async(args, kwargs)
619 def clone(self, *args, **kwargs):
620 to_signature = maybe_signature
621 signature = Signature.clone(self, *args, **kwargs)
622 signature.kwargs['tasks'] = [
623 to_signature(sig, app=self._app, clone=True)
624 for sig in signature.kwargs['tasks']
625 ]
626 return signature
628 def unchain_tasks(self):
629 # Clone chain's tasks assigning signatures from link_error
630 # to each task
631 tasks = [t.clone() for t in self.tasks]
632 for sig in self.options.get('link_error', []):
633 for task in tasks:
634 task.link_error(sig)
635 return tasks
637 def apply_async(self, args=None, kwargs=None, **options):
638 # python is best at unpacking kwargs, so .run is here to do that.
639 args = args if args else ()
640 kwargs = kwargs if kwargs else []
641 app = self.app
642 if app.conf.task_always_eager:
643 with allow_join_result():
644 return self.apply(args, kwargs, **options)
645 return self.run(args, kwargs, app=app, **(
646 dict(self.options, **options) if options else self.options))
648 def run(self, args=None, kwargs=None, group_id=None, chord=None,
649 task_id=None, link=None, link_error=None, publisher=None,
650 producer=None, root_id=None, parent_id=None, app=None, **options):
651 # pylint: disable=redefined-outer-name
652 # XXX chord is also a class in outer scope.
653 args = args if args else ()
654 kwargs = kwargs if kwargs else []
655 app = app or self.app
656 use_link = self._use_link
657 if use_link is None and app.conf.task_protocol == 1:
658 use_link = True
659 args = (tuple(args) + tuple(self.args)
660 if args and not self.immutable else self.args)
662 tasks, results = self.prepare_steps(
663 args, kwargs, self.tasks, root_id, parent_id, link_error, app,
664 task_id, group_id, chord,
665 )
667 if results:
668 if link:
669 tasks[0].extend_list_option('link', link)
670 first_task = tasks.pop()
671 options = _prepare_chain_from_options(options, tasks, use_link)
673 first_task.apply_async(**options)
674 return results[0]
676 def freeze(self, _id=None, group_id=None, chord=None,
677 root_id=None, parent_id=None):
678 # pylint: disable=redefined-outer-name
679 # XXX chord is also a class in outer scope.
680 _, results = self._frozen = self.prepare_steps(
681 self.args, self.kwargs, self.tasks, root_id, parent_id, None,
682 self.app, _id, group_id, chord, clone=False,
683 )
684 return results[0]
686 def prepare_steps(self, args, kwargs, tasks,
687 root_id=None, parent_id=None, link_error=None, app=None,
688 last_task_id=None, group_id=None, chord_body=None,
689 clone=True, from_dict=Signature.from_dict):
690 app = app or self.app
691 # use chain message field for protocol 2 and later.
692 # this avoids pickle blowing the stack on the recursion
693 # required by linking task together in a tree structure.
694 # (why is pickle using recursion? or better yet why cannot python
695 # do tail call optimization making recursion actually useful?)
696 use_link = self._use_link
697 if use_link is None and app.conf.task_protocol == 1:
698 use_link = True
699 steps = deque(tasks)
701 steps_pop = steps.pop
702 steps_extend = steps.extend
704 prev_task = None
705 prev_res = None
706 tasks, results = [], []
707 i = 0
708 # NOTE: We are doing this in reverse order.
709 # The result is a list of tasks in reverse order, that is
710 # passed as the ``chain`` message field.
711 # As it's reversed the worker can just do ``chain.pop()`` to
712 # get the next task in the chain.
713 while steps:
714 task = steps_pop()
715 is_first_task, is_last_task = not steps, not i
717 if not isinstance(task, abstract.CallableSignature):
718 task = from_dict(task, app=app)
719 if isinstance(task, group):
720 task = maybe_unroll_group(task)
722 # first task gets partial args from chain
723 if clone:
724 if is_first_task:
725 task = task.clone(args, kwargs)
726 else:
727 task = task.clone()
728 elif is_first_task:
729 task.args = tuple(args) + tuple(task.args)
731 if isinstance(task, _chain):
732 # splice the chain
733 steps_extend(task.tasks)
734 continue
736 if isinstance(task, group) and prev_task:
737 # automatically upgrade group(...) | s to chord(group, s)
738 # for chords we freeze by pretending it's a normal
739 # signature instead of a group.
740 tasks.pop()
741 results.pop()
742 try:
743 task = chord(
744 task, body=prev_task,
745 task_id=prev_res.task_id, root_id=root_id, app=app,
746 )
747 except AttributeError:
748 # A GroupResult does not have a task_id since it consists
749 # of multiple tasks.
750 # We therefore, have to construct the chord without it.
751 # Issues #5467, #3585.
752 task = chord(
753 task, body=prev_task,
754 root_id=root_id, app=app,
755 )
757 if is_last_task:
758 # chain(task_id=id) means task id is set for the last task
759 # in the chain. If the chord is part of a chord/group
760 # then that chord/group must synchronize based on the
761 # last task in the chain, so we only set the group_id and
762 # chord callback for the last task.
763 res = task.freeze(
764 last_task_id,
765 root_id=root_id, group_id=group_id, chord=chord_body,
766 )
767 else:
768 res = task.freeze(root_id=root_id)
770 i += 1
772 if prev_task:
773 if use_link:
774 # link previous task to this task.
775 task.link(prev_task)
777 if prev_res and not prev_res.parent:
778 prev_res.parent = res
780 if link_error:
781 for errback in maybe_list(link_error):
782 task.link_error(errback)
784 tasks.append(task)
785 results.append(res)
787 prev_task, prev_res = task, res
788 if isinstance(task, chord):
789 app.backend.ensure_chords_allowed()
790 # If the task is a chord, and the body is a chain
791 # the chain has already been prepared, and res is
792 # set to the last task in the callback chain.
794 # We need to change that so that it points to the
795 # group result object.
796 node = res
797 while node.parent:
798 node = node.parent
799 prev_res = node
800 return tasks, results
802 def apply(self, args=None, kwargs=None, **options):
803 args = args if args else ()
804 kwargs = kwargs if kwargs else {}
805 last, (fargs, fkwargs) = None, (args, kwargs)
806 for task in self.tasks:
807 res = task.clone(fargs, fkwargs).apply(
808 last and (last.get(),), **dict(self.options, **options))
809 res.parent, last, (fargs, fkwargs) = last, res, (None, None)
810 return last
812 @property
813 def app(self):
814 app = self._app
815 if app is None:
816 try:
817 app = self.tasks[0]._app
818 except LookupError:
819 pass
820 return app or current_app
822 def __repr__(self):
823 if not self.tasks:
824 return '<{0}@{1:#x}: empty>'.format(
825 type(self).__name__, id(self))
826 return remove_repeating_from_task(
827 self.tasks[0]['task'],
828 ' | '.join(repr(t) for t in self.tasks))
831class chain(_chain):
832 """Chain tasks together.
834 Each tasks follows one another,
835 by being applied as a callback of the previous task.
837 Note:
838 If called with only one argument, then that argument must
839 be an iterable of tasks to chain: this allows us
840 to use generator expressions.
842 Example:
843 This is effectively :math:`((2 + 2) + 4)`:
845 .. code-block:: pycon
847 >>> res = chain(add.s(2, 2), add.s(4))()
848 >>> res.get()
849 8
851 Calling a chain will return the result of the last task in the chain.
852 You can get to the other tasks by following the ``result.parent``'s:
854 .. code-block:: pycon
856 >>> res.parent.get()
857 4
859 Using a generator expression:
861 .. code-block:: pycon
863 >>> lazy_chain = chain(add.s(i) for i in range(10))
864 >>> res = lazy_chain(3)
866 Arguments:
867 *tasks (Signature): List of task signatures to chain.
868 If only one argument is passed and that argument is
869 an iterable, then that'll be used as the list of signatures
870 to chain instead. This means that you can use a generator
871 expression.
873 Returns:
874 ~celery.chain: A lazy signature that can be called to apply the first
875 task in the chain. When that task succeeds the next task in the
876 chain is applied, and so on.
877 """
879 # could be function, but must be able to reference as :class:`chain`.
880 def __new__(cls, *tasks, **kwargs):
881 # This forces `chain(X, Y, Z)` to work the same way as `X | Y | Z`
882 if not kwargs and tasks:
883 if len(tasks) != 1 or is_list(tasks[0]):
884 tasks = tasks[0] if len(tasks) == 1 else tasks
885 # if is_list(tasks) and len(tasks) == 1:
886 # return super(chain, cls).__new__(cls, tasks, **kwargs)
887 return reduce(operator.or_, tasks, chain())
888 return super(chain, cls).__new__(cls, *tasks, **kwargs)
891class _basemap(Signature):
892 _task_name = None
893 _unpack_args = itemgetter('task', 'it')
895 @classmethod
896 def from_dict(cls, d, app=None):
897 return _upgrade(
898 d, cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']),
899 )
901 def __init__(self, task, it, **options):
902 Signature.__init__(
903 self, self._task_name, (),
904 {'task': task, 'it': regen(it)}, immutable=True, **options
905 )
907 def apply_async(self, args=None, kwargs=None, **opts):
908 # need to evaluate generators
909 args = args if args else ()
910 kwargs = kwargs if kwargs else {}
911 task, it = self._unpack_args(self.kwargs)
912 return self.type.apply_async(
913 (), {'task': task, 'it': list(it)},
914 route_name=task_name_from(self.kwargs.get('task')), **opts
915 )
918@Signature.register_type()
919@python_2_unicode_compatible
920class xmap(_basemap):
921 """Map operation for tasks.
923 Note:
924 Tasks executed sequentially in process, this is not a
925 parallel operation like :class:`group`.
926 """
928 _task_name = 'celery.map'
930 def __repr__(self):
931 task, it = self._unpack_args(self.kwargs)
932 return '[{0}(x) for x in {1}]'.format(
933 task.task, truncate(repr(it), 100))
936@Signature.register_type()
937@python_2_unicode_compatible
938class xstarmap(_basemap):
939 """Map operation for tasks, using star arguments."""
941 _task_name = 'celery.starmap'
943 def __repr__(self):
944 task, it = self._unpack_args(self.kwargs)
945 return '[{0}(*x) for x in {1}]'.format(
946 task.task, truncate(repr(it), 100))
949@Signature.register_type()
950class chunks(Signature):
951 """Partition of tasks into chunks of size n."""
953 _unpack_args = itemgetter('task', 'it', 'n')
955 @classmethod
956 def from_dict(cls, d, app=None):
957 return _upgrade(
958 d, chunks(*cls._unpack_args(
959 d['kwargs']), app=app, **d['options']),
960 )
962 def __init__(self, task, it, n, **options):
963 Signature.__init__(
964 self, 'celery.chunks', (),
965 {'task': task, 'it': regen(it), 'n': n},
966 immutable=True, **options
967 )
969 def __call__(self, **options):
970 return self.apply_async(**options)
972 def apply_async(self, args=None, kwargs=None, **opts):
973 args = args if args else ()
974 kwargs = kwargs if kwargs else {}
975 return self.group().apply_async(
976 args, kwargs,
977 route_name=task_name_from(self.kwargs.get('task')), **opts
978 )
980 def group(self):
981 # need to evaluate generators
982 task, it, n = self._unpack_args(self.kwargs)
983 return group((xstarmap(task, part, app=self._app)
984 for part in _chunks(iter(it), n)),
985 app=self._app)
987 @classmethod
988 def apply_chunks(cls, task, it, n, app=None):
989 return cls(task, it, n, app=app)()
992def _maybe_group(tasks, app):
993 if isinstance(tasks, dict):
994 tasks = signature(tasks, app=app)
996 if isinstance(tasks, (group, _chain)):
997 tasks = tasks.tasks
998 elif isinstance(tasks, abstract.CallableSignature):
999 tasks = [tasks]
1000 else:
1001 tasks = [signature(t, app=app) for t in tasks]
1002 return tasks
1005@Signature.register_type()
1006@python_2_unicode_compatible
1007class group(Signature):
1008 """Creates a group of tasks to be executed in parallel.
1010 A group is lazy so you must call it to take action and evaluate
1011 the group.
1013 Note:
1014 If only one argument is passed, and that argument is an iterable
1015 then that'll be used as the list of tasks instead: this
1016 allows us to use ``group`` with generator expressions.
1018 Example:
1019 >>> lazy_group = group([add.s(2, 2), add.s(4, 4)])
1020 >>> promise = lazy_group() # <-- evaluate: returns lazy result.
1021 >>> promise.get() # <-- will wait for the task to return
1022 [4, 8]
1024 Arguments:
1025 *tasks (List[Signature]): A list of signatures that this group will
1026 call. If there's only one argument, and that argument is an
1027 iterable, then that'll define the list of signatures instead.
1028 **options (Any): Execution options applied to all tasks
1029 in the group.
1031 Returns:
1032 ~celery.group: signature that when called will then call all of the
1033 tasks in the group (and return a :class:`GroupResult` instance
1034 that can be used to inspect the state of the group).
1035 """
1037 tasks = getitem_property('kwargs.tasks', 'Tasks in group.')
1039 @classmethod
1040 def from_dict(cls, d, app=None):
1041 return _upgrade(
1042 d, group(d['kwargs']['tasks'], app=app, **d['options']),
1043 )
1045 def __init__(self, *tasks, **options):
1046 if len(tasks) == 1:
1047 tasks = tasks[0]
1048 if isinstance(tasks, group):
1049 tasks = tasks.tasks
1050 if isinstance(tasks, abstract.CallableSignature):
1051 tasks = [tasks.clone()]
1052 if not isinstance(tasks, _regen):
1053 tasks = regen(tasks)
1054 Signature.__init__(
1055 self, 'celery.group', (), {'tasks': tasks}, **options
1056 )
1057 self.subtask_type = 'group'
1059 def __call__(self, *partial_args, **options):
1060 return self.apply_async(partial_args, **options)
1062 def skew(self, start=1.0, stop=None, step=1.0):
1063 it = fxrange(start, stop, step, repeatlast=True)
1064 for task in self.tasks:
1065 task.set(countdown=next(it))
1066 return self
1068 def apply_async(self, args=None, kwargs=None, add_to_parent=True,
1069 producer=None, link=None, link_error=None, **options):
1070 args = args if args else ()
1071 if link is not None:
1072 raise TypeError('Cannot add link to group: use a chord')
1073 if link_error is not None:
1074 raise TypeError(
1075 'Cannot add link to group: do that on individual tasks')
1076 app = self.app
1077 if app.conf.task_always_eager:
1078 return self.apply(args, kwargs, **options)
1079 if not self.tasks:
1080 return self.freeze()
1082 options, group_id, root_id = self._freeze_gid(options)
1083 tasks = self._prepared(self.tasks, [], group_id, root_id, app)
1084 p = barrier()
1085 results = list(self._apply_tasks(tasks, producer, app, p,
1086 args=args, kwargs=kwargs, **options))
1087 result = self.app.GroupResult(group_id, results, ready_barrier=p)
1088 p.finalize()
1090 # - Special case of group(A.s() | group(B.s(), C.s()))
1091 # That is, group with single item that's a chain but the
1092 # last task in that chain is a group.
1093 #
1094 # We cannot actually support arbitrary GroupResults in chains,
1095 # but this special case we can.
1096 if len(result) == 1 and isinstance(result[0], GroupResult):
1097 result = result[0]
1099 parent_task = app.current_worker_task
1100 if add_to_parent and parent_task:
1101 parent_task.add_trail(result)
1102 return result
1104 def apply(self, args=None, kwargs=None, **options):
1105 args = args if args else ()
1106 kwargs = kwargs if kwargs else {}
1107 app = self.app
1108 if not self.tasks:
1109 return self.freeze() # empty group returns GroupResult
1110 options, group_id, root_id = self._freeze_gid(options)
1111 tasks = self._prepared(self.tasks, [], group_id, root_id, app)
1112 return app.GroupResult(group_id, [
1113 sig.apply(args=args, kwargs=kwargs, **options) for sig, _ in tasks
1114 ])
1116 def set_immutable(self, immutable):
1117 for task in self.tasks:
1118 task.set_immutable(immutable)
1120 def link(self, sig):
1121 # Simply link to first task
1122 sig = sig.clone().set(immutable=True)
1123 return self.tasks[0].link(sig)
1125 def link_error(self, sig):
1126 try:
1127 sig = sig.clone().set(immutable=True)
1128 except AttributeError:
1129 # See issue #5265. I don't use isinstance because current tests
1130 # pass a Mock object as argument.
1131 sig['immutable'] = True
1132 sig = Signature.from_dict(sig)
1133 return self.tasks[0].link_error(sig)
1135 def _prepared(self, tasks, partial_args, group_id, root_id, app,
1136 CallableSignature=abstract.CallableSignature,
1137 from_dict=Signature.from_dict,
1138 isinstance=isinstance, tuple=tuple):
1139 for task in tasks:
1140 if isinstance(task, CallableSignature):
1141 # local sigs are always of type Signature, and we
1142 # clone them to make sure we don't modify the originals.
1143 task = task.clone()
1144 else:
1145 # serialized sigs must be converted to Signature.
1146 task = from_dict(task, app=app)
1147 if isinstance(task, group):
1148 # needs yield_from :(
1149 unroll = task._prepared(
1150 task.tasks, partial_args, group_id, root_id, app,
1151 )
1152 for taskN, resN in unroll:
1153 yield taskN, resN
1154 else:
1155 if partial_args and not task.immutable:
1156 task.args = tuple(partial_args) + tuple(task.args)
1157 yield task, task.freeze(group_id=group_id, root_id=root_id)
1159 def _apply_tasks(self, tasks, producer=None, app=None, p=None,
1160 add_to_parent=None, chord=None,
1161 args=None, kwargs=None, **options):
1162 # pylint: disable=redefined-outer-name
1163 # XXX chord is also a class in outer scope.
1164 app = app or self.app
1165 with app.producer_or_acquire(producer) as producer:
1166 for sig, res in tasks:
1167 sig.apply_async(producer=producer, add_to_parent=False,
1168 chord=sig.options.get('chord') or chord,
1169 args=args, kwargs=kwargs,
1170 **options)
1172 # adding callback to result, such that it will gradually
1173 # fulfill the barrier.
1174 #
1175 # Using barrier.add would use result.then, but we need
1176 # to add the weak argument here to only create a weak
1177 # reference to the object.
1178 if p and not p.cancelled and not p.ready:
1179 p.size += 1
1180 res.then(p, weak=True)
1181 yield res # <-- r.parent, etc set in the frozen result.
1183 def _freeze_gid(self, options):
1184 # remove task_id and use that as the group_id,
1185 # if we don't remove it then every task will have the same id...
1186 options = dict(self.options, **options)
1187 options['group_id'] = group_id = (
1188 options.pop('task_id', uuid()))
1189 return options, group_id, options.get('root_id')
1191 def freeze(self, _id=None, group_id=None, chord=None,
1192 root_id=None, parent_id=None):
1193 # pylint: disable=redefined-outer-name
1194 # XXX chord is also a class in outer scope.
1195 opts = self.options
1196 try:
1197 gid = opts['task_id']
1198 except KeyError:
1199 gid = opts['task_id'] = group_id or uuid()
1200 if group_id:
1201 opts['group_id'] = group_id
1202 if chord:
1203 opts['chord'] = chord
1204 root_id = opts.setdefault('root_id', root_id)
1205 parent_id = opts.setdefault('parent_id', parent_id)
1206 new_tasks = []
1207 # Need to unroll subgroups early so that chord gets the
1208 # right result instance for chord_unlock etc.
1209 results = list(self._freeze_unroll(
1210 new_tasks, group_id, chord, root_id, parent_id,
1211 ))
1212 if isinstance(self.tasks, MutableSequence):
1213 self.tasks[:] = new_tasks
1214 else:
1215 self.tasks = new_tasks
1216 return self.app.GroupResult(gid, results)
1218 _freeze = freeze
1220 def _freeze_unroll(self, new_tasks, group_id, chord, root_id, parent_id):
1221 # pylint: disable=redefined-outer-name
1222 # XXX chord is also a class in outer scope.
1223 stack = deque(self.tasks)
1224 while stack:
1225 task = maybe_signature(stack.popleft(), app=self._app).clone()
1226 if isinstance(task, group):
1227 stack.extendleft(task.tasks)
1228 else:
1229 new_tasks.append(task)
1230 yield task.freeze(group_id=group_id,
1231 chord=chord, root_id=root_id,
1232 parent_id=parent_id)
1234 def __repr__(self):
1235 if self.tasks:
1236 return remove_repeating_from_task(
1237 self.tasks[0]['task'],
1238 'group({0.tasks!r})'.format(self))
1239 return 'group(<empty>)'
1241 def __len__(self):
1242 return len(self.tasks)
1244 @property
1245 def app(self):
1246 app = self._app
1247 if app is None:
1248 try:
1249 app = self.tasks[0].app
1250 except LookupError:
1251 pass
1252 return app if app is not None else current_app
1255@Signature.register_type()
1256@python_2_unicode_compatible
1257class chord(Signature):
1258 r"""Barrier synchronization primitive.
1260 A chord consists of a header and a body.
1262 The header is a group of tasks that must complete before the callback is
1263 called. A chord is essentially a callback for a group of tasks.
1265 The body is applied with the return values of all the header
1266 tasks as a list.
1268 Example:
1270 The chord:
1272 .. code-block:: pycon
1274 >>> res = chord([add.s(2, 2), add.s(4, 4)])(sum_task.s())
1276 is effectively :math:`\Sigma ((2 + 2) + (4 + 4))`:
1278 .. code-block:: pycon
1280 >>> res.get()
1281 12
1282 """
1284 @classmethod
1285 def from_dict(cls, d, app=None):
1286 options = d.copy()
1287 args, options['kwargs'] = cls._unpack_args(**options['kwargs'])
1288 return _upgrade(d, cls(*args, app=app, **options))
1290 @staticmethod
1291 def _unpack_args(header=None, body=None, **kwargs):
1292 # Python signatures are better at extracting keys from dicts
1293 # than manually popping things off.
1294 return (header, body), kwargs
1296 def __init__(self, header, body=None, task='celery.chord',
1297 args=None, kwargs=None, app=None, **options):
1298 args = args if args else ()
1299 kwargs = kwargs if kwargs else {}
1300 Signature.__init__(
1301 self, task, args,
1302 {'kwargs': kwargs, 'header': _maybe_group(header, app),
1303 'body': maybe_signature(body, app=app)}, app=app, **options
1304 )
1305 self.subtask_type = 'chord'
1307 def __call__(self, body=None, **options):
1308 return self.apply_async((), {'body': body} if body else {}, **options)
1310 def freeze(self, _id=None, group_id=None, chord=None,
1311 root_id=None, parent_id=None):
1312 # pylint: disable=redefined-outer-name
1313 # XXX chord is also a class in outer scope.
1314 if not isinstance(self.tasks, group):
1315 self.tasks = group(self.tasks, app=self.app)
1316 header_result = self.tasks.freeze(
1317 parent_id=parent_id, root_id=root_id, chord=self.body)
1319 body_result = self.body.freeze(
1320 _id, root_id=root_id, chord=chord, group_id=group_id)
1322 # we need to link the body result back to the group result,
1323 # but the body may actually be a chain,
1324 # so find the first result without a parent
1325 node = body_result
1326 seen = set()
1327 while node:
1328 if node.id in seen:
1329 raise RuntimeError('Recursive result parents')
1330 seen.add(node.id)
1331 if node.parent is None:
1332 node.parent = header_result
1333 break
1334 node = node.parent
1335 self.id = self.tasks.id
1336 return body_result
1338 def apply_async(self, args=None, kwargs=None, task_id=None,
1339 producer=None, publisher=None, connection=None,
1340 router=None, result_cls=None, **options):
1341 args = args if args else ()
1342 kwargs = kwargs if kwargs else {}
1343 args = (tuple(args) + tuple(self.args)
1344 if args and not self.immutable else self.args)
1345 body = kwargs.pop('body', None) or self.kwargs['body']
1346 kwargs = dict(self.kwargs['kwargs'], **kwargs)
1347 body = body.clone(**options)
1348 app = self._get_app(body)
1349 tasks = (self.tasks.clone() if isinstance(self.tasks, group)
1350 else group(self.tasks, app=app))
1351 if app.conf.task_always_eager:
1352 with allow_join_result():
1353 return self.apply(args, kwargs,
1354 body=body, task_id=task_id, **options)
1356 merged_options = dict(self.options, **options) if options else self.options
1357 option_task_id = merged_options.pop("task_id", None)
1358 if task_id is None:
1359 task_id = option_task_id
1361 # chord([A, B, ...], C)
1362 return self.run(tasks, body, args, task_id=task_id, **merged_options)
1364 def apply(self, args=None, kwargs=None,
1365 propagate=True, body=None, **options):
1366 args = args if args else ()
1367 kwargs = kwargs if kwargs else {}
1368 body = self.body if body is None else body
1369 tasks = (self.tasks.clone() if isinstance(self.tasks, group)
1370 else group(self.tasks, app=self.app))
1371 return body.apply(
1372 args=(tasks.apply(args, kwargs).get(propagate=propagate),),
1373 )
1375 def _traverse_tasks(self, tasks, value=None):
1376 stack = deque(tasks)
1377 while stack:
1378 task = stack.popleft()
1379 if isinstance(task, group):
1380 stack.extend(task.tasks)
1381 elif isinstance(task, _chain) and isinstance(task.tasks[-1], group):
1382 stack.extend(task.tasks[-1].tasks)
1383 else:
1384 yield task if value is None else value
1386 def __length_hint__(self):
1387 tasks = (self.tasks.tasks if isinstance(self.tasks, group)
1388 else self.tasks)
1389 return sum(self._traverse_tasks(tasks, 1))
1391 def run(self, header, body, partial_args, app=None, interval=None,
1392 countdown=1, max_retries=None, eager=False,
1393 task_id=None, **options):
1394 app = app or self._get_app(body)
1395 group_id = header.options.get('task_id') or uuid()
1396 root_id = body.options.get('root_id')
1397 body.chord_size = self.__length_hint__()
1398 options = dict(self.options, **options) if options else self.options
1399 if options:
1400 options.pop('task_id', None)
1401 body.options.update(options)
1403 bodyres = body.freeze(task_id, root_id=root_id)
1405 # Chains should not be passed to the header tasks. See #3771
1406 options.pop('chain', None)
1407 # Neither should chords, for deeply nested chords to work
1408 options.pop('chord', None)
1409 options.pop('task_id', None)
1411 header_result = header.freeze(group_id=group_id, chord=body, root_id=root_id)
1413 if len(header_result) > 0:
1414 app.backend.apply_chord(
1415 header_result,
1416 body,
1417 interval=interval,
1418 countdown=countdown,
1419 max_retries=max_retries,
1420 )
1421 header_result = header(*partial_args, task_id=group_id, **options)
1422 # The execution of a chord body is normally triggered by its header's
1423 # tasks completing. If the header is empty this will never happen, so
1424 # we execute the body manually here.
1425 else:
1426 body.delay([])
1428 bodyres.parent = header_result
1429 return bodyres
1431 def clone(self, *args, **kwargs):
1432 signature = Signature.clone(self, *args, **kwargs)
1433 # need to make copy of body
1434 try:
1435 signature.kwargs['body'] = maybe_signature(
1436 signature.kwargs['body'], clone=True)
1437 except (AttributeError, KeyError):
1438 pass
1439 return signature
1441 def link(self, callback):
1442 self.body.link(callback)
1443 return callback
1445 def link_error(self, errback):
1446 self.body.link_error(errback)
1447 return errback
1449 def set_immutable(self, immutable):
1450 # changes mutability of header only, not callback.
1451 for task in self.tasks:
1452 task.set_immutable(immutable)
1454 def __repr__(self):
1455 if self.body:
1456 if isinstance(self.body, _chain):
1457 return remove_repeating_from_task(
1458 self.body.tasks[0]['task'],
1459 '%({0} | {1!r})'.format(
1460 self.body.tasks[0].reprcall(self.tasks),
1461 chain(self.body.tasks[1:], app=self._app),
1462 ),
1463 )
1464 return '%' + remove_repeating_from_task(
1465 self.body['task'], self.body.reprcall(self.tasks))
1466 return '<chord without body: {0.tasks!r}>'.format(self)
1468 @cached_property
1469 def app(self):
1470 return self._get_app(self.body)
1472 def _get_app(self, body=None):
1473 app = self._app
1474 if app is None:
1475 try:
1476 tasks = self.tasks.tasks # is a group
1477 except AttributeError:
1478 tasks = self.tasks
1479 if len(tasks):
1480 app = tasks[0]._app
1481 if app is None and body is not None:
1482 app = body._app
1483 return app if app is not None else current_app
1485 tasks = getitem_property('kwargs.header', 'Tasks in chord header.')
1486 body = getitem_property('kwargs.body', 'Body task of chord.')
1489def signature(varies, *args, **kwargs):
1490 """Create new signature.
1492 - if the first argument is a signature already then it's cloned.
1493 - if the first argument is a dict, then a Signature version is returned.
1495 Returns:
1496 Signature: The resulting signature.
1497 """
1498 app = kwargs.get('app')
1499 if isinstance(varies, dict):
1500 if isinstance(varies, abstract.CallableSignature):
1501 return varies.clone()
1502 return Signature.from_dict(varies, app=app)
1503 return Signature(varies, *args, **kwargs)
1506subtask = signature # noqa: E305 XXX compat
1509def maybe_signature(d, app=None, clone=False):
1510 """Ensure obj is a signature, or None.
1512 Arguments:
1513 d (Optional[Union[abstract.CallableSignature, Mapping]]):
1514 Signature or dict-serialized signature.
1515 app (celery.Celery):
1516 App to bind signature to.
1517 clone (bool):
1518 If d' is already a signature, the signature
1519 will be cloned when this flag is enabled.
1521 Returns:
1522 Optional[abstract.CallableSignature]
1523 """
1524 if d is not None:
1525 if isinstance(d, abstract.CallableSignature):
1526 if clone:
1527 d = d.clone()
1528 elif isinstance(d, dict):
1529 d = signature(d)
1531 if app is not None:
1532 d._app = app
1533 return d
1536maybe_subtask = maybe_signature # noqa: E305 XXX compat