Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2"""Composing task work-flows. 

3 

4.. seealso: 

5 

6 You should import these from :mod:`celery` and not this module. 

7""" 

8from __future__ import absolute_import, unicode_literals 

9 

10import itertools 

11import operator 

12from collections import deque 

13from copy import deepcopy 

14from functools import partial as _partial 

15from functools import reduce 

16from operator import itemgetter 

17 

18from kombu.utils.functional import fxrange, reprcall 

19from kombu.utils.objects import cached_property 

20from kombu.utils.uuid import uuid 

21from vine import barrier 

22 

23from celery._state import current_app 

24from celery.five import PY3, python_2_unicode_compatible 

25from celery.local import try_import 

26from celery.result import GroupResult, allow_join_result 

27from celery.utils import abstract 

28from celery.utils.collections import ChainMap 

29from celery.utils.functional import _regen 

30from celery.utils.functional import chunks as _chunks 

31from celery.utils.functional import (is_list, maybe_list, regen, 

32 seq_concat_item, seq_concat_seq) 

33from celery.utils.objects import getitem_property 

34from celery.utils.text import remove_repeating_from_task, truncate 

35 

36try: 

37 from collections.abc import MutableSequence 

38except ImportError: 

39 # TODO: Remove this when we drop Python 2.7 support 

40 from collections import MutableSequence 

41 

42__all__ = ( 

43 'Signature', 'chain', 'xmap', 'xstarmap', 'chunks', 

44 'group', 'chord', 'signature', 'maybe_signature', 

45) 

46 

47# json in Python 2.7 borks if dict contains byte keys. 

48JSON_NEEDS_UNICODE_KEYS = PY3 and not try_import('simplejson') 

49 

50 

51def maybe_unroll_group(group): 

52 """Unroll group with only one member.""" 

53 # Issue #1656 

54 try: 

55 size = len(group.tasks) 

56 except TypeError: 

57 try: 

58 size = group.tasks.__length_hint__() 

59 except (AttributeError, TypeError): 

60 return group 

61 else: 

62 return list(group.tasks)[0] if size == 1 else group 

63 else: 

64 return group.tasks[0] if size == 1 else group 

65 

66 

67def task_name_from(task): 

68 return getattr(task, 'name', task) 

69 

70 

71def _upgrade(fields, sig): 

72 """Used by custom signatures in .from_dict, to keep common fields.""" 

73 sig.update(chord_size=fields.get('chord_size')) 

74 return sig 

75 

76 

77@abstract.CallableSignature.register 

78@python_2_unicode_compatible 

79class Signature(dict): 

80 """Task Signature. 

81 

82 Class that wraps the arguments and execution options 

83 for a single task invocation. 

84 

85 Used as the parts in a :class:`group` and other constructs, 

86 or to pass tasks around as callbacks while being compatible 

87 with serializers with a strict type subset. 

88 

89 Signatures can also be created from tasks: 

90 

91 - Using the ``.signature()`` method that has the same signature 

92 as ``Task.apply_async``: 

93 

94 .. code-block:: pycon 

95 

96 >>> add.signature(args=(1,), kwargs={'kw': 2}, options={}) 

97 

98 - or the ``.s()`` shortcut that works for star arguments: 

99 

100 .. code-block:: pycon 

101 

102 >>> add.s(1, kw=2) 

103 

104 - the ``.s()`` shortcut does not allow you to specify execution options 

105 but there's a chaning `.set` method that returns the signature: 

106 

107 .. code-block:: pycon 

108 

109 >>> add.s(2, 2).set(countdown=10).set(expires=30).delay() 

110 

111 Note: 

112 You should use :func:`~celery.signature` to create new signatures. 

113 The ``Signature`` class is the type returned by that function and 

114 should be used for ``isinstance`` checks for signatures. 

115 

116 See Also: 

117 :ref:`guide-canvas` for the complete guide. 

118 

119 Arguments: 

120 task (Union[Type[celery.app.task.Task], str]): Either a task 

121 class/instance, or the name of a task. 

122 args (Tuple): Positional arguments to apply. 

123 kwargs (Dict): Keyword arguments to apply. 

124 options (Dict): Additional options to :meth:`Task.apply_async`. 

125 

126 Note: 

127 If the first argument is a :class:`dict`, the other 

128 arguments will be ignored and the values in the dict will be used 

129 instead:: 

130 

131 >>> s = signature('tasks.add', args=(2, 2)) 

132 >>> signature(s) 

133 {'task': 'tasks.add', args=(2, 2), kwargs={}, options={}} 

134 """ 

135 

136 TYPES = {} 

137 _app = _type = None 

138 

139 @classmethod 

140 def register_type(cls, name=None): 

141 def _inner(subclass): 

142 cls.TYPES[name or subclass.__name__] = subclass 

143 return subclass 

144 

145 return _inner 

146 

147 @classmethod 

148 def from_dict(cls, d, app=None): 

149 typ = d.get('subtask_type') 

150 if typ: 

151 target_cls = cls.TYPES[typ] 

152 if target_cls is not cls: 

153 return target_cls.from_dict(d, app=app) 

154 return Signature(d, app=app) 

155 

156 def __init__(self, task=None, args=None, kwargs=None, options=None, 

157 type=None, subtask_type=None, immutable=False, 

158 app=None, **ex): 

159 self._app = app 

160 

161 if isinstance(task, dict): 

162 super(Signature, self).__init__(task) # works like dict(d) 

163 else: 

164 # Also supports using task class/instance instead of string name. 

165 try: 

166 task_name = task.name 

167 except AttributeError: 

168 task_name = task 

169 else: 

170 self._type = task 

171 

172 super(Signature, self).__init__( 

173 task=task_name, args=tuple(args or ()), 

174 kwargs=kwargs or {}, 

175 options=dict(options or {}, **ex), 

176 subtask_type=subtask_type, 

177 immutable=immutable, 

178 chord_size=None, 

179 ) 

180 

181 def __call__(self, *partial_args, **partial_kwargs): 

182 """Call the task directly (in the current process).""" 

183 args, kwargs, _ = self._merge(partial_args, partial_kwargs, None) 

184 return self.type(*args, **kwargs) 

185 

186 def delay(self, *partial_args, **partial_kwargs): 

187 """Shortcut to :meth:`apply_async` using star arguments.""" 

188 return self.apply_async(partial_args, partial_kwargs) 

189 

190 def apply(self, args=None, kwargs=None, **options): 

191 """Call task locally. 

192 

193 Same as :meth:`apply_async` but executed the task inline instead 

194 of sending a task message. 

195 """ 

196 args = args if args else () 

197 kwargs = kwargs if kwargs else {} 

198 # Extra options set to None are dismissed 

199 options = {k: v for k, v in options.items() if v is not None} 

200 # For callbacks: extra args are prepended to the stored args. 

201 args, kwargs, options = self._merge(args, kwargs, options) 

202 return self.type.apply(args, kwargs, **options) 

203 

204 def apply_async(self, args=None, kwargs=None, route_name=None, **options): 

205 """Apply this task asynchronously. 

206 

207 Arguments: 

208 args (Tuple): Partial args to be prepended to the existing args. 

209 kwargs (Dict): Partial kwargs to be merged with existing kwargs. 

210 options (Dict): Partial options to be merged 

211 with existing options. 

212 

213 Returns: 

214 ~@AsyncResult: promise of future evaluation. 

215 

216 See also: 

217 :meth:`~@Task.apply_async` and the :ref:`guide-calling` guide. 

218 """ 

219 args = args if args else () 

220 kwargs = kwargs if kwargs else {} 

221 # Extra options set to None are dismissed 

222 options = {k: v for k, v in options.items() if v is not None} 

223 try: 

224 _apply = self._apply_async 

225 except IndexError: # pragma: no cover 

226 # no tasks for chain, etc to find type 

227 return 

228 # For callbacks: extra args are prepended to the stored args. 

229 if args or kwargs or options: 

230 args, kwargs, options = self._merge(args, kwargs, options) 

231 else: 

232 args, kwargs, options = self.args, self.kwargs, self.options 

233 # pylint: disable=too-many-function-args 

234 # Borks on this, as it's a property 

235 return _apply(args, kwargs, **options) 

236 

237 def _merge(self, args=None, kwargs=None, options=None, force=False): 

238 args = args if args else () 

239 kwargs = kwargs if kwargs else {} 

240 options = options if options else {} 

241 if self.immutable and not force: 

242 return (self.args, self.kwargs, 

243 dict(self.options, 

244 **options) if options else self.options) 

245 return (tuple(args) + tuple(self.args) if args else self.args, 

246 dict(self.kwargs, **kwargs) if kwargs else self.kwargs, 

247 dict(self.options, **options) if options else self.options) 

248 

249 def clone(self, args=None, kwargs=None, **opts): 

250 """Create a copy of this signature. 

251 

252 Arguments: 

253 args (Tuple): Partial args to be prepended to the existing args. 

254 kwargs (Dict): Partial kwargs to be merged with existing kwargs. 

255 options (Dict): Partial options to be merged with 

256 existing options. 

257 """ 

258 args = args if args else () 

259 kwargs = kwargs if kwargs else {} 

260 # need to deepcopy options so origins links etc. is not modified. 

261 if args or kwargs or opts: 

262 args, kwargs, opts = self._merge(args, kwargs, opts) 

263 else: 

264 args, kwargs, opts = self.args, self.kwargs, self.options 

265 signature = Signature.from_dict({'task': self.task, 

266 'args': tuple(args), 

267 'kwargs': kwargs, 

268 'options': deepcopy(opts), 

269 'subtask_type': self.subtask_type, 

270 'chord_size': self.chord_size, 

271 'immutable': self.immutable}, 

272 app=self._app) 

273 signature._type = self._type 

274 return signature 

275 

276 partial = clone 

277 

278 def freeze(self, _id=None, group_id=None, chord=None, 

279 root_id=None, parent_id=None): 

280 """Finalize the signature by adding a concrete task id. 

281 

282 The task won't be called and you shouldn't call the signature 

283 twice after freezing it as that'll result in two task messages 

284 using the same task id. 

285 

286 Returns: 

287 ~@AsyncResult: promise of future evaluation. 

288 """ 

289 # pylint: disable=redefined-outer-name 

290 # XXX chord is also a class in outer scope. 

291 opts = self.options 

292 try: 

293 tid = opts['task_id'] 

294 except KeyError: 

295 tid = opts['task_id'] = _id or uuid() 

296 if root_id: 

297 opts['root_id'] = root_id 

298 if parent_id: 

299 opts['parent_id'] = parent_id 

300 if 'reply_to' not in opts: 

301 opts['reply_to'] = self.app.oid 

302 if group_id: 

303 opts['group_id'] = group_id 

304 if chord: 

305 opts['chord'] = chord 

306 # pylint: disable=too-many-function-args 

307 # Borks on this, as it's a property. 

308 return self.AsyncResult(tid) 

309 

310 _freeze = freeze 

311 

312 def replace(self, args=None, kwargs=None, options=None): 

313 """Replace the args, kwargs or options set for this signature. 

314 

315 These are only replaced if the argument for the section is 

316 not :const:`None`. 

317 """ 

318 signature = self.clone() 

319 if args is not None: 

320 signature.args = args 

321 if kwargs is not None: 

322 signature.kwargs = kwargs 

323 if options is not None: 

324 signature.options = options 

325 return signature 

326 

327 def set(self, immutable=None, **options): 

328 """Set arbitrary execution options (same as ``.options.update(…)``). 

329 

330 Returns: 

331 Signature: This is a chaining method call 

332 (i.e., it will return ``self``). 

333 """ 

334 if immutable is not None: 

335 self.set_immutable(immutable) 

336 self.options.update(options) 

337 return self 

338 

339 def set_immutable(self, immutable): 

340 self.immutable = immutable 

341 

342 def _with_list_option(self, key): 

343 items = self.options.setdefault(key, []) 

344 if not isinstance(items, MutableSequence): 

345 items = self.options[key] = [items] 

346 return items 

347 

348 def append_to_list_option(self, key, value): 

349 items = self._with_list_option(key) 

350 if value not in items: 

351 items.append(value) 

352 return value 

353 

354 def extend_list_option(self, key, value): 

355 items = self._with_list_option(key) 

356 items.extend(maybe_list(value)) 

357 

358 def link(self, callback): 

359 """Add callback task to be applied if this task succeeds. 

360 

361 Returns: 

362 Signature: the argument passed, for chaining 

363 or use with :func:`~functools.reduce`. 

364 """ 

365 return self.append_to_list_option('link', callback) 

366 

367 def link_error(self, errback): 

368 """Add callback task to be applied on error in task execution. 

369 

370 Returns: 

371 Signature: the argument passed, for chaining 

372 or use with :func:`~functools.reduce`. 

373 """ 

374 return self.append_to_list_option('link_error', errback) 

375 

376 def on_error(self, errback): 

377 """Version of :meth:`link_error` that supports chaining. 

378 

379 on_error chains the original signature, not the errback so:: 

380 

381 >>> add.s(2, 2).on_error(errback.s()).delay() 

382 

383 calls the ``add`` task, not the ``errback`` task, but the 

384 reverse is true for :meth:`link_error`. 

385 """ 

386 self.link_error(errback) 

387 return self 

388 

389 def flatten_links(self): 

390 """Return a recursive list of dependencies. 

391 

392 "unchain" if you will, but with links intact. 

393 """ 

394 return list(itertools.chain.from_iterable(itertools.chain( 

395 [[self]], 

396 (link.flatten_links() 

397 for link in maybe_list(self.options.get('link')) or []) 

398 ))) 

399 

400 def __or__(self, other): 

401 # These could be implemented in each individual class, 

402 # I'm sure, but for now we have this. 

403 if isinstance(self, group): 

404 # group() | task -> chord 

405 return chord(self, body=other, app=self._app) 

406 elif isinstance(other, group): 

407 # unroll group with one member 

408 other = maybe_unroll_group(other) 

409 if isinstance(self, _chain): 

410 # chain | group() -> chain 

411 tasks = self.unchain_tasks() 

412 if not tasks: 

413 # If the chain is empty, return the group 

414 return other 

415 return _chain(seq_concat_item( 

416 tasks, other), app=self._app) 

417 # task | group() -> chain 

418 return _chain(self, other, app=self.app) 

419 

420 if not isinstance(self, _chain) and isinstance(other, _chain): 

421 # task | chain -> chain 

422 return _chain(seq_concat_seq( 

423 (self,), other.unchain_tasks()), app=self._app) 

424 elif isinstance(other, _chain): 

425 # chain | chain -> chain 

426 return _chain(seq_concat_seq( 

427 self.unchain_tasks(), other.unchain_tasks()), app=self._app) 

428 elif isinstance(self, chord): 

429 # chord | task -> attach to body 

430 sig = self.clone() 

431 sig.body = sig.body | other 

432 return sig 

433 elif isinstance(other, Signature): 

434 if isinstance(self, _chain): 

435 if self.tasks and isinstance(self.tasks[-1], group): 

436 # CHAIN [last item is group] | TASK -> chord 

437 sig = self.clone() 

438 sig.tasks[-1] = chord( 

439 sig.tasks[-1], other, app=self._app) 

440 return sig 

441 elif self.tasks and isinstance(self.tasks[-1], chord): 

442 # CHAIN [last item is chord] -> chain with chord body. 

443 sig = self.clone() 

444 sig.tasks[-1].body = sig.tasks[-1].body | other 

445 return sig 

446 else: 

447 # chain | task -> chain 

448 return _chain(seq_concat_item( 

449 self.unchain_tasks(), other), app=self._app) 

450 # task | task -> chain 

451 return _chain(self, other, app=self._app) 

452 return NotImplemented 

453 

454 def election(self): 

455 type = self.type 

456 app = type.app 

457 tid = self.options.get('task_id') or uuid() 

458 

459 with app.producer_or_acquire(None) as producer: 

460 props = type.backend.on_task_call(producer, tid) 

461 app.control.election(tid, 'task', 

462 self.clone(task_id=tid, **props), 

463 connection=producer.connection) 

464 return type.AsyncResult(tid) 

465 

466 def reprcall(self, *args, **kwargs): 

467 args, kwargs, _ = self._merge(args, kwargs, {}, force=True) 

468 return reprcall(self['task'], args, kwargs) 

469 

470 def __deepcopy__(self, memo): 

471 memo[id(self)] = self 

472 return dict(self) 

473 

474 def __invert__(self): 

475 return self.apply_async().get() 

476 

477 def __reduce__(self): 

478 # for serialization, the task type is lazily loaded, 

479 # and not stored in the dict itself. 

480 return signature, (dict(self),) 

481 

482 def __json__(self): 

483 return dict(self) 

484 

485 def __repr__(self): 

486 return self.reprcall() 

487 

488 if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover 

489 def items(self): 

490 for k, v in dict.items(self): 

491 yield k.decode() if isinstance(k, bytes) else k, v 

492 

493 @property 

494 def name(self): 

495 # for duck typing compatibility with Task.name 

496 return self.task 

497 

498 @cached_property 

499 def type(self): 

500 return self._type or self.app.tasks[self['task']] 

501 

502 @cached_property 

503 def app(self): 

504 return self._app or current_app 

505 

506 @cached_property 

507 def AsyncResult(self): 

508 try: 

509 return self.type.AsyncResult 

510 except KeyError: # task not registered 

511 return self.app.AsyncResult 

512 

513 @cached_property 

514 def _apply_async(self): 

515 try: 

516 return self.type.apply_async 

517 except KeyError: 

518 return _partial(self.app.send_task, self['task']) 

519 

520 id = getitem_property('options.task_id', 'Task UUID') 

521 parent_id = getitem_property('options.parent_id', 'Task parent UUID.') 

522 root_id = getitem_property('options.root_id', 'Task root UUID.') 

523 task = getitem_property('task', 'Name of task.') 

524 args = getitem_property('args', 'Positional arguments to task.') 

525 kwargs = getitem_property('kwargs', 'Keyword arguments to task.') 

526 options = getitem_property('options', 'Task execution options.') 

527 subtask_type = getitem_property('subtask_type', 'Type of signature') 

528 chord_size = getitem_property( 

529 'chord_size', 'Size of chord (if applicable)') 

530 immutable = getitem_property( 

531 'immutable', 'Flag set if no longer accepts new arguments') 

532 

533 

534def _prepare_chain_from_options(options, tasks, use_link): 

535 # When we publish groups we reuse the same options dictionary for all of 

536 # the tasks in the group. See: 

537 # https://github.com/celery/celery/blob/fb37cb0b8/celery/canvas.py#L1022. 

538 # Issue #5354 reported that the following type of canvases 

539 # causes a Celery worker to hang: 

540 # group( 

541 # add.s(1, 1), 

542 # add.s(1, 1) 

543 # ) | tsum.s() | add.s(1) | group(add.s(1), add.s(1)) 

544 # The resolution of #5354 in PR #5681 was to only set the `chain` key 

545 # in the options dictionary if it is not present. 

546 # Otherwise we extend the existing list of tasks in the chain with the new 

547 # tasks: options['chain'].extend(chain_). 

548 # Before PR #5681 we overrode the `chain` key in each iteration 

549 # of the loop which applies all the tasks in the group: 

550 # options['chain'] = tasks if not use_link else None 

551 # This caused Celery to execute chains correctly in most cases since 

552 # in each iteration the `chain` key would reset itself to a new value 

553 # and the side effect of mutating the key did not propagate 

554 # to the next task in the group. 

555 # Since we now mutated the `chain` key, a *list* which is passed 

556 # by *reference*, the next task in the group will extend the list 

557 # of tasks in the chain instead of setting a new one from the chain_ 

558 # variable above. 

559 # This causes Celery to execute a chain, even though there might not be 

560 # one to begin with. Alternatively, it causes Celery to execute more tasks 

561 # that were previously present in the previous task in the group. 

562 # The solution is to be careful and never mutate the options dictionary 

563 # to begin with. 

564 # Here is an example of a canvas which triggers this issue: 

565 # add.s(5, 6) | group((add.s(1) | add.s(2), add.s(3))). 

566 # The expected result is [14, 14]. However, when we extend the `chain` 

567 # key the `add.s(3)` task erroneously has `add.s(2)` in its chain since 

568 # it was previously applied to `add.s(1)`. 

569 # Without being careful not to mutate the options dictionary, the result 

570 # in this case is [16, 14]. 

571 # To avoid deep-copying the entire options dictionary every single time we 

572 # run a chain we use a ChainMap and ensure that we never mutate 

573 # the original `chain` key, hence we use list_a + list_b to create a new 

574 # list. 

575 if use_link: 

576 return ChainMap({'chain': None}, options) 

577 elif 'chain' not in options: 

578 return ChainMap({'chain': tasks}, options) 

579 elif tasks is not None: 

580 # chain option may already be set, resulting in 

581 # "multiple values for keyword argument 'chain'" error. 

582 # Issue #3379. 

583 # If a chain already exists, we need to extend it with the next 

584 # tasks in the chain. 

585 # Issue #5354. 

586 # WARNING: Be careful not to mutate `options['chain']`. 

587 return ChainMap({'chain': options['chain'] + tasks}, 

588 options) 

589 

590 

591@Signature.register_type(name='chain') 

592@python_2_unicode_compatible 

593class _chain(Signature): 

594 tasks = getitem_property('kwargs.tasks', 'Tasks in chain.') 

595 

596 @classmethod 

597 def from_dict(cls, d, app=None): 

598 tasks = d['kwargs']['tasks'] 

599 if tasks: 

600 if isinstance(tasks, tuple): # aaaargh 

601 tasks = d['kwargs']['tasks'] = list(tasks) 

602 tasks = [maybe_signature(task, app=app) for task in tasks] 

603 return _upgrade(d, _chain(tasks, app=app, **d['options'])) 

604 

605 def __init__(self, *tasks, **options): 

606 tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) 

607 else tasks) 

608 Signature.__init__( 

609 self, 'celery.chain', (), {'tasks': tasks}, **options 

610 ) 

611 self._use_link = options.pop('use_link', None) 

612 self.subtask_type = 'chain' 

613 self._frozen = None 

614 

615 def __call__(self, *args, **kwargs): 

616 if self.tasks: 

617 return self.apply_async(args, kwargs) 

618 

619 def clone(self, *args, **kwargs): 

620 to_signature = maybe_signature 

621 signature = Signature.clone(self, *args, **kwargs) 

622 signature.kwargs['tasks'] = [ 

623 to_signature(sig, app=self._app, clone=True) 

624 for sig in signature.kwargs['tasks'] 

625 ] 

626 return signature 

627 

628 def unchain_tasks(self): 

629 # Clone chain's tasks assigning signatures from link_error 

630 # to each task 

631 tasks = [t.clone() for t in self.tasks] 

632 for sig in self.options.get('link_error', []): 

633 for task in tasks: 

634 task.link_error(sig) 

635 return tasks 

636 

637 def apply_async(self, args=None, kwargs=None, **options): 

638 # python is best at unpacking kwargs, so .run is here to do that. 

639 args = args if args else () 

640 kwargs = kwargs if kwargs else [] 

641 app = self.app 

642 if app.conf.task_always_eager: 

643 with allow_join_result(): 

644 return self.apply(args, kwargs, **options) 

645 return self.run(args, kwargs, app=app, **( 

646 dict(self.options, **options) if options else self.options)) 

647 

648 def run(self, args=None, kwargs=None, group_id=None, chord=None, 

649 task_id=None, link=None, link_error=None, publisher=None, 

650 producer=None, root_id=None, parent_id=None, app=None, **options): 

651 # pylint: disable=redefined-outer-name 

652 # XXX chord is also a class in outer scope. 

653 args = args if args else () 

654 kwargs = kwargs if kwargs else [] 

655 app = app or self.app 

656 use_link = self._use_link 

657 if use_link is None and app.conf.task_protocol == 1: 

658 use_link = True 

659 args = (tuple(args) + tuple(self.args) 

660 if args and not self.immutable else self.args) 

661 

662 tasks, results = self.prepare_steps( 

663 args, kwargs, self.tasks, root_id, parent_id, link_error, app, 

664 task_id, group_id, chord, 

665 ) 

666 

667 if results: 

668 if link: 

669 tasks[0].extend_list_option('link', link) 

670 first_task = tasks.pop() 

671 options = _prepare_chain_from_options(options, tasks, use_link) 

672 

673 first_task.apply_async(**options) 

674 return results[0] 

675 

676 def freeze(self, _id=None, group_id=None, chord=None, 

677 root_id=None, parent_id=None): 

678 # pylint: disable=redefined-outer-name 

679 # XXX chord is also a class in outer scope. 

680 _, results = self._frozen = self.prepare_steps( 

681 self.args, self.kwargs, self.tasks, root_id, parent_id, None, 

682 self.app, _id, group_id, chord, clone=False, 

683 ) 

684 return results[0] 

685 

686 def prepare_steps(self, args, kwargs, tasks, 

687 root_id=None, parent_id=None, link_error=None, app=None, 

688 last_task_id=None, group_id=None, chord_body=None, 

689 clone=True, from_dict=Signature.from_dict): 

690 app = app or self.app 

691 # use chain message field for protocol 2 and later. 

692 # this avoids pickle blowing the stack on the recursion 

693 # required by linking task together in a tree structure. 

694 # (why is pickle using recursion? or better yet why cannot python 

695 # do tail call optimization making recursion actually useful?) 

696 use_link = self._use_link 

697 if use_link is None and app.conf.task_protocol == 1: 

698 use_link = True 

699 steps = deque(tasks) 

700 

701 steps_pop = steps.pop 

702 steps_extend = steps.extend 

703 

704 prev_task = None 

705 prev_res = None 

706 tasks, results = [], [] 

707 i = 0 

708 # NOTE: We are doing this in reverse order. 

709 # The result is a list of tasks in reverse order, that is 

710 # passed as the ``chain`` message field. 

711 # As it's reversed the worker can just do ``chain.pop()`` to 

712 # get the next task in the chain. 

713 while steps: 

714 task = steps_pop() 

715 is_first_task, is_last_task = not steps, not i 

716 

717 if not isinstance(task, abstract.CallableSignature): 

718 task = from_dict(task, app=app) 

719 if isinstance(task, group): 

720 task = maybe_unroll_group(task) 

721 

722 # first task gets partial args from chain 

723 if clone: 

724 if is_first_task: 

725 task = task.clone(args, kwargs) 

726 else: 

727 task = task.clone() 

728 elif is_first_task: 

729 task.args = tuple(args) + tuple(task.args) 

730 

731 if isinstance(task, _chain): 

732 # splice the chain 

733 steps_extend(task.tasks) 

734 continue 

735 

736 if isinstance(task, group) and prev_task: 

737 # automatically upgrade group(...) | s to chord(group, s) 

738 # for chords we freeze by pretending it's a normal 

739 # signature instead of a group. 

740 tasks.pop() 

741 results.pop() 

742 try: 

743 task = chord( 

744 task, body=prev_task, 

745 task_id=prev_res.task_id, root_id=root_id, app=app, 

746 ) 

747 except AttributeError: 

748 # A GroupResult does not have a task_id since it consists 

749 # of multiple tasks. 

750 # We therefore, have to construct the chord without it. 

751 # Issues #5467, #3585. 

752 task = chord( 

753 task, body=prev_task, 

754 root_id=root_id, app=app, 

755 ) 

756 

757 if is_last_task: 

758 # chain(task_id=id) means task id is set for the last task 

759 # in the chain. If the chord is part of a chord/group 

760 # then that chord/group must synchronize based on the 

761 # last task in the chain, so we only set the group_id and 

762 # chord callback for the last task. 

763 res = task.freeze( 

764 last_task_id, 

765 root_id=root_id, group_id=group_id, chord=chord_body, 

766 ) 

767 else: 

768 res = task.freeze(root_id=root_id) 

769 

770 i += 1 

771 

772 if prev_task: 

773 if use_link: 

774 # link previous task to this task. 

775 task.link(prev_task) 

776 

777 if prev_res and not prev_res.parent: 

778 prev_res.parent = res 

779 

780 if link_error: 

781 for errback in maybe_list(link_error): 

782 task.link_error(errback) 

783 

784 tasks.append(task) 

785 results.append(res) 

786 

787 prev_task, prev_res = task, res 

788 if isinstance(task, chord): 

789 app.backend.ensure_chords_allowed() 

790 # If the task is a chord, and the body is a chain 

791 # the chain has already been prepared, and res is 

792 # set to the last task in the callback chain. 

793 

794 # We need to change that so that it points to the 

795 # group result object. 

796 node = res 

797 while node.parent: 

798 node = node.parent 

799 prev_res = node 

800 return tasks, results 

801 

802 def apply(self, args=None, kwargs=None, **options): 

803 args = args if args else () 

804 kwargs = kwargs if kwargs else {} 

805 last, (fargs, fkwargs) = None, (args, kwargs) 

806 for task in self.tasks: 

807 res = task.clone(fargs, fkwargs).apply( 

808 last and (last.get(),), **dict(self.options, **options)) 

809 res.parent, last, (fargs, fkwargs) = last, res, (None, None) 

810 return last 

811 

812 @property 

813 def app(self): 

814 app = self._app 

815 if app is None: 

816 try: 

817 app = self.tasks[0]._app 

818 except LookupError: 

819 pass 

820 return app or current_app 

821 

822 def __repr__(self): 

823 if not self.tasks: 

824 return '<{0}@{1:#x}: empty>'.format( 

825 type(self).__name__, id(self)) 

826 return remove_repeating_from_task( 

827 self.tasks[0]['task'], 

828 ' | '.join(repr(t) for t in self.tasks)) 

829 

830 

831class chain(_chain): 

832 """Chain tasks together. 

833 

834 Each tasks follows one another, 

835 by being applied as a callback of the previous task. 

836 

837 Note: 

838 If called with only one argument, then that argument must 

839 be an iterable of tasks to chain: this allows us 

840 to use generator expressions. 

841 

842 Example: 

843 This is effectively :math:`((2 + 2) + 4)`: 

844 

845 .. code-block:: pycon 

846 

847 >>> res = chain(add.s(2, 2), add.s(4))() 

848 >>> res.get() 

849 8 

850 

851 Calling a chain will return the result of the last task in the chain. 

852 You can get to the other tasks by following the ``result.parent``'s: 

853 

854 .. code-block:: pycon 

855 

856 >>> res.parent.get() 

857 4 

858 

859 Using a generator expression: 

860 

861 .. code-block:: pycon 

862 

863 >>> lazy_chain = chain(add.s(i) for i in range(10)) 

864 >>> res = lazy_chain(3) 

865 

866 Arguments: 

867 *tasks (Signature): List of task signatures to chain. 

868 If only one argument is passed and that argument is 

869 an iterable, then that'll be used as the list of signatures 

870 to chain instead. This means that you can use a generator 

871 expression. 

872 

873 Returns: 

874 ~celery.chain: A lazy signature that can be called to apply the first 

875 task in the chain. When that task succeeds the next task in the 

876 chain is applied, and so on. 

877 """ 

878 

879 # could be function, but must be able to reference as :class:`chain`. 

880 def __new__(cls, *tasks, **kwargs): 

881 # This forces `chain(X, Y, Z)` to work the same way as `X | Y | Z` 

882 if not kwargs and tasks: 

883 if len(tasks) != 1 or is_list(tasks[0]): 

884 tasks = tasks[0] if len(tasks) == 1 else tasks 

885 # if is_list(tasks) and len(tasks) == 1: 

886 # return super(chain, cls).__new__(cls, tasks, **kwargs) 

887 return reduce(operator.or_, tasks, chain()) 

888 return super(chain, cls).__new__(cls, *tasks, **kwargs) 

889 

890 

891class _basemap(Signature): 

892 _task_name = None 

893 _unpack_args = itemgetter('task', 'it') 

894 

895 @classmethod 

896 def from_dict(cls, d, app=None): 

897 return _upgrade( 

898 d, cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']), 

899 ) 

900 

901 def __init__(self, task, it, **options): 

902 Signature.__init__( 

903 self, self._task_name, (), 

904 {'task': task, 'it': regen(it)}, immutable=True, **options 

905 ) 

906 

907 def apply_async(self, args=None, kwargs=None, **opts): 

908 # need to evaluate generators 

909 args = args if args else () 

910 kwargs = kwargs if kwargs else {} 

911 task, it = self._unpack_args(self.kwargs) 

912 return self.type.apply_async( 

913 (), {'task': task, 'it': list(it)}, 

914 route_name=task_name_from(self.kwargs.get('task')), **opts 

915 ) 

916 

917 

918@Signature.register_type() 

919@python_2_unicode_compatible 

920class xmap(_basemap): 

921 """Map operation for tasks. 

922 

923 Note: 

924 Tasks executed sequentially in process, this is not a 

925 parallel operation like :class:`group`. 

926 """ 

927 

928 _task_name = 'celery.map' 

929 

930 def __repr__(self): 

931 task, it = self._unpack_args(self.kwargs) 

932 return '[{0}(x) for x in {1}]'.format( 

933 task.task, truncate(repr(it), 100)) 

934 

935 

936@Signature.register_type() 

937@python_2_unicode_compatible 

938class xstarmap(_basemap): 

939 """Map operation for tasks, using star arguments.""" 

940 

941 _task_name = 'celery.starmap' 

942 

943 def __repr__(self): 

944 task, it = self._unpack_args(self.kwargs) 

945 return '[{0}(*x) for x in {1}]'.format( 

946 task.task, truncate(repr(it), 100)) 

947 

948 

949@Signature.register_type() 

950class chunks(Signature): 

951 """Partition of tasks into chunks of size n.""" 

952 

953 _unpack_args = itemgetter('task', 'it', 'n') 

954 

955 @classmethod 

956 def from_dict(cls, d, app=None): 

957 return _upgrade( 

958 d, chunks(*cls._unpack_args( 

959 d['kwargs']), app=app, **d['options']), 

960 ) 

961 

962 def __init__(self, task, it, n, **options): 

963 Signature.__init__( 

964 self, 'celery.chunks', (), 

965 {'task': task, 'it': regen(it), 'n': n}, 

966 immutable=True, **options 

967 ) 

968 

969 def __call__(self, **options): 

970 return self.apply_async(**options) 

971 

972 def apply_async(self, args=None, kwargs=None, **opts): 

973 args = args if args else () 

974 kwargs = kwargs if kwargs else {} 

975 return self.group().apply_async( 

976 args, kwargs, 

977 route_name=task_name_from(self.kwargs.get('task')), **opts 

978 ) 

979 

980 def group(self): 

981 # need to evaluate generators 

982 task, it, n = self._unpack_args(self.kwargs) 

983 return group((xstarmap(task, part, app=self._app) 

984 for part in _chunks(iter(it), n)), 

985 app=self._app) 

986 

987 @classmethod 

988 def apply_chunks(cls, task, it, n, app=None): 

989 return cls(task, it, n, app=app)() 

990 

991 

992def _maybe_group(tasks, app): 

993 if isinstance(tasks, dict): 

994 tasks = signature(tasks, app=app) 

995 

996 if isinstance(tasks, (group, _chain)): 

997 tasks = tasks.tasks 

998 elif isinstance(tasks, abstract.CallableSignature): 

999 tasks = [tasks] 

1000 else: 

1001 tasks = [signature(t, app=app) for t in tasks] 

1002 return tasks 

1003 

1004 

1005@Signature.register_type() 

1006@python_2_unicode_compatible 

1007class group(Signature): 

1008 """Creates a group of tasks to be executed in parallel. 

1009 

1010 A group is lazy so you must call it to take action and evaluate 

1011 the group. 

1012 

1013 Note: 

1014 If only one argument is passed, and that argument is an iterable 

1015 then that'll be used as the list of tasks instead: this 

1016 allows us to use ``group`` with generator expressions. 

1017 

1018 Example: 

1019 >>> lazy_group = group([add.s(2, 2), add.s(4, 4)]) 

1020 >>> promise = lazy_group() # <-- evaluate: returns lazy result. 

1021 >>> promise.get() # <-- will wait for the task to return 

1022 [4, 8] 

1023 

1024 Arguments: 

1025 *tasks (List[Signature]): A list of signatures that this group will 

1026 call. If there's only one argument, and that argument is an 

1027 iterable, then that'll define the list of signatures instead. 

1028 **options (Any): Execution options applied to all tasks 

1029 in the group. 

1030 

1031 Returns: 

1032 ~celery.group: signature that when called will then call all of the 

1033 tasks in the group (and return a :class:`GroupResult` instance 

1034 that can be used to inspect the state of the group). 

1035 """ 

1036 

1037 tasks = getitem_property('kwargs.tasks', 'Tasks in group.') 

1038 

1039 @classmethod 

1040 def from_dict(cls, d, app=None): 

1041 return _upgrade( 

1042 d, group(d['kwargs']['tasks'], app=app, **d['options']), 

1043 ) 

1044 

1045 def __init__(self, *tasks, **options): 

1046 if len(tasks) == 1: 

1047 tasks = tasks[0] 

1048 if isinstance(tasks, group): 

1049 tasks = tasks.tasks 

1050 if isinstance(tasks, abstract.CallableSignature): 

1051 tasks = [tasks.clone()] 

1052 if not isinstance(tasks, _regen): 

1053 tasks = regen(tasks) 

1054 Signature.__init__( 

1055 self, 'celery.group', (), {'tasks': tasks}, **options 

1056 ) 

1057 self.subtask_type = 'group' 

1058 

1059 def __call__(self, *partial_args, **options): 

1060 return self.apply_async(partial_args, **options) 

1061 

1062 def skew(self, start=1.0, stop=None, step=1.0): 

1063 it = fxrange(start, stop, step, repeatlast=True) 

1064 for task in self.tasks: 

1065 task.set(countdown=next(it)) 

1066 return self 

1067 

1068 def apply_async(self, args=None, kwargs=None, add_to_parent=True, 

1069 producer=None, link=None, link_error=None, **options): 

1070 args = args if args else () 

1071 if link is not None: 

1072 raise TypeError('Cannot add link to group: use a chord') 

1073 if link_error is not None: 

1074 raise TypeError( 

1075 'Cannot add link to group: do that on individual tasks') 

1076 app = self.app 

1077 if app.conf.task_always_eager: 

1078 return self.apply(args, kwargs, **options) 

1079 if not self.tasks: 

1080 return self.freeze() 

1081 

1082 options, group_id, root_id = self._freeze_gid(options) 

1083 tasks = self._prepared(self.tasks, [], group_id, root_id, app) 

1084 p = barrier() 

1085 results = list(self._apply_tasks(tasks, producer, app, p, 

1086 args=args, kwargs=kwargs, **options)) 

1087 result = self.app.GroupResult(group_id, results, ready_barrier=p) 

1088 p.finalize() 

1089 

1090 # - Special case of group(A.s() | group(B.s(), C.s())) 

1091 # That is, group with single item that's a chain but the 

1092 # last task in that chain is a group. 

1093 # 

1094 # We cannot actually support arbitrary GroupResults in chains, 

1095 # but this special case we can. 

1096 if len(result) == 1 and isinstance(result[0], GroupResult): 

1097 result = result[0] 

1098 

1099 parent_task = app.current_worker_task 

1100 if add_to_parent and parent_task: 

1101 parent_task.add_trail(result) 

1102 return result 

1103 

1104 def apply(self, args=None, kwargs=None, **options): 

1105 args = args if args else () 

1106 kwargs = kwargs if kwargs else {} 

1107 app = self.app 

1108 if not self.tasks: 

1109 return self.freeze() # empty group returns GroupResult 

1110 options, group_id, root_id = self._freeze_gid(options) 

1111 tasks = self._prepared(self.tasks, [], group_id, root_id, app) 

1112 return app.GroupResult(group_id, [ 

1113 sig.apply(args=args, kwargs=kwargs, **options) for sig, _ in tasks 

1114 ]) 

1115 

1116 def set_immutable(self, immutable): 

1117 for task in self.tasks: 

1118 task.set_immutable(immutable) 

1119 

1120 def link(self, sig): 

1121 # Simply link to first task 

1122 sig = sig.clone().set(immutable=True) 

1123 return self.tasks[0].link(sig) 

1124 

1125 def link_error(self, sig): 

1126 try: 

1127 sig = sig.clone().set(immutable=True) 

1128 except AttributeError: 

1129 # See issue #5265. I don't use isinstance because current tests 

1130 # pass a Mock object as argument. 

1131 sig['immutable'] = True 

1132 sig = Signature.from_dict(sig) 

1133 return self.tasks[0].link_error(sig) 

1134 

1135 def _prepared(self, tasks, partial_args, group_id, root_id, app, 

1136 CallableSignature=abstract.CallableSignature, 

1137 from_dict=Signature.from_dict, 

1138 isinstance=isinstance, tuple=tuple): 

1139 for task in tasks: 

1140 if isinstance(task, CallableSignature): 

1141 # local sigs are always of type Signature, and we 

1142 # clone them to make sure we don't modify the originals. 

1143 task = task.clone() 

1144 else: 

1145 # serialized sigs must be converted to Signature. 

1146 task = from_dict(task, app=app) 

1147 if isinstance(task, group): 

1148 # needs yield_from :( 

1149 unroll = task._prepared( 

1150 task.tasks, partial_args, group_id, root_id, app, 

1151 ) 

1152 for taskN, resN in unroll: 

1153 yield taskN, resN 

1154 else: 

1155 if partial_args and not task.immutable: 

1156 task.args = tuple(partial_args) + tuple(task.args) 

1157 yield task, task.freeze(group_id=group_id, root_id=root_id) 

1158 

1159 def _apply_tasks(self, tasks, producer=None, app=None, p=None, 

1160 add_to_parent=None, chord=None, 

1161 args=None, kwargs=None, **options): 

1162 # pylint: disable=redefined-outer-name 

1163 # XXX chord is also a class in outer scope. 

1164 app = app or self.app 

1165 with app.producer_or_acquire(producer) as producer: 

1166 for sig, res in tasks: 

1167 sig.apply_async(producer=producer, add_to_parent=False, 

1168 chord=sig.options.get('chord') or chord, 

1169 args=args, kwargs=kwargs, 

1170 **options) 

1171 

1172 # adding callback to result, such that it will gradually 

1173 # fulfill the barrier. 

1174 # 

1175 # Using barrier.add would use result.then, but we need 

1176 # to add the weak argument here to only create a weak 

1177 # reference to the object. 

1178 if p and not p.cancelled and not p.ready: 

1179 p.size += 1 

1180 res.then(p, weak=True) 

1181 yield res # <-- r.parent, etc set in the frozen result. 

1182 

1183 def _freeze_gid(self, options): 

1184 # remove task_id and use that as the group_id, 

1185 # if we don't remove it then every task will have the same id... 

1186 options = dict(self.options, **options) 

1187 options['group_id'] = group_id = ( 

1188 options.pop('task_id', uuid())) 

1189 return options, group_id, options.get('root_id') 

1190 

1191 def freeze(self, _id=None, group_id=None, chord=None, 

1192 root_id=None, parent_id=None): 

1193 # pylint: disable=redefined-outer-name 

1194 # XXX chord is also a class in outer scope. 

1195 opts = self.options 

1196 try: 

1197 gid = opts['task_id'] 

1198 except KeyError: 

1199 gid = opts['task_id'] = group_id or uuid() 

1200 if group_id: 

1201 opts['group_id'] = group_id 

1202 if chord: 

1203 opts['chord'] = chord 

1204 root_id = opts.setdefault('root_id', root_id) 

1205 parent_id = opts.setdefault('parent_id', parent_id) 

1206 new_tasks = [] 

1207 # Need to unroll subgroups early so that chord gets the 

1208 # right result instance for chord_unlock etc. 

1209 results = list(self._freeze_unroll( 

1210 new_tasks, group_id, chord, root_id, parent_id, 

1211 )) 

1212 if isinstance(self.tasks, MutableSequence): 

1213 self.tasks[:] = new_tasks 

1214 else: 

1215 self.tasks = new_tasks 

1216 return self.app.GroupResult(gid, results) 

1217 

1218 _freeze = freeze 

1219 

1220 def _freeze_unroll(self, new_tasks, group_id, chord, root_id, parent_id): 

1221 # pylint: disable=redefined-outer-name 

1222 # XXX chord is also a class in outer scope. 

1223 stack = deque(self.tasks) 

1224 while stack: 

1225 task = maybe_signature(stack.popleft(), app=self._app).clone() 

1226 if isinstance(task, group): 

1227 stack.extendleft(task.tasks) 

1228 else: 

1229 new_tasks.append(task) 

1230 yield task.freeze(group_id=group_id, 

1231 chord=chord, root_id=root_id, 

1232 parent_id=parent_id) 

1233 

1234 def __repr__(self): 

1235 if self.tasks: 

1236 return remove_repeating_from_task( 

1237 self.tasks[0]['task'], 

1238 'group({0.tasks!r})'.format(self)) 

1239 return 'group(<empty>)' 

1240 

1241 def __len__(self): 

1242 return len(self.tasks) 

1243 

1244 @property 

1245 def app(self): 

1246 app = self._app 

1247 if app is None: 

1248 try: 

1249 app = self.tasks[0].app 

1250 except LookupError: 

1251 pass 

1252 return app if app is not None else current_app 

1253 

1254 

1255@Signature.register_type() 

1256@python_2_unicode_compatible 

1257class chord(Signature): 

1258 r"""Barrier synchronization primitive. 

1259 

1260 A chord consists of a header and a body. 

1261 

1262 The header is a group of tasks that must complete before the callback is 

1263 called. A chord is essentially a callback for a group of tasks. 

1264 

1265 The body is applied with the return values of all the header 

1266 tasks as a list. 

1267 

1268 Example: 

1269 

1270 The chord: 

1271 

1272 .. code-block:: pycon 

1273 

1274 >>> res = chord([add.s(2, 2), add.s(4, 4)])(sum_task.s()) 

1275 

1276 is effectively :math:`\Sigma ((2 + 2) + (4 + 4))`: 

1277 

1278 .. code-block:: pycon 

1279 

1280 >>> res.get() 

1281 12 

1282 """ 

1283 

1284 @classmethod 

1285 def from_dict(cls, d, app=None): 

1286 options = d.copy() 

1287 args, options['kwargs'] = cls._unpack_args(**options['kwargs']) 

1288 return _upgrade(d, cls(*args, app=app, **options)) 

1289 

1290 @staticmethod 

1291 def _unpack_args(header=None, body=None, **kwargs): 

1292 # Python signatures are better at extracting keys from dicts 

1293 # than manually popping things off. 

1294 return (header, body), kwargs 

1295 

1296 def __init__(self, header, body=None, task='celery.chord', 

1297 args=None, kwargs=None, app=None, **options): 

1298 args = args if args else () 

1299 kwargs = kwargs if kwargs else {} 

1300 Signature.__init__( 

1301 self, task, args, 

1302 {'kwargs': kwargs, 'header': _maybe_group(header, app), 

1303 'body': maybe_signature(body, app=app)}, app=app, **options 

1304 ) 

1305 self.subtask_type = 'chord' 

1306 

1307 def __call__(self, body=None, **options): 

1308 return self.apply_async((), {'body': body} if body else {}, **options) 

1309 

1310 def freeze(self, _id=None, group_id=None, chord=None, 

1311 root_id=None, parent_id=None): 

1312 # pylint: disable=redefined-outer-name 

1313 # XXX chord is also a class in outer scope. 

1314 if not isinstance(self.tasks, group): 

1315 self.tasks = group(self.tasks, app=self.app) 

1316 header_result = self.tasks.freeze( 

1317 parent_id=parent_id, root_id=root_id, chord=self.body) 

1318 

1319 body_result = self.body.freeze( 

1320 _id, root_id=root_id, chord=chord, group_id=group_id) 

1321 

1322 # we need to link the body result back to the group result, 

1323 # but the body may actually be a chain, 

1324 # so find the first result without a parent 

1325 node = body_result 

1326 seen = set() 

1327 while node: 

1328 if node.id in seen: 

1329 raise RuntimeError('Recursive result parents') 

1330 seen.add(node.id) 

1331 if node.parent is None: 

1332 node.parent = header_result 

1333 break 

1334 node = node.parent 

1335 self.id = self.tasks.id 

1336 return body_result 

1337 

1338 def apply_async(self, args=None, kwargs=None, task_id=None, 

1339 producer=None, publisher=None, connection=None, 

1340 router=None, result_cls=None, **options): 

1341 args = args if args else () 

1342 kwargs = kwargs if kwargs else {} 

1343 args = (tuple(args) + tuple(self.args) 

1344 if args and not self.immutable else self.args) 

1345 body = kwargs.pop('body', None) or self.kwargs['body'] 

1346 kwargs = dict(self.kwargs['kwargs'], **kwargs) 

1347 body = body.clone(**options) 

1348 app = self._get_app(body) 

1349 tasks = (self.tasks.clone() if isinstance(self.tasks, group) 

1350 else group(self.tasks, app=app)) 

1351 if app.conf.task_always_eager: 

1352 with allow_join_result(): 

1353 return self.apply(args, kwargs, 

1354 body=body, task_id=task_id, **options) 

1355 

1356 merged_options = dict(self.options, **options) if options else self.options 

1357 option_task_id = merged_options.pop("task_id", None) 

1358 if task_id is None: 

1359 task_id = option_task_id 

1360 

1361 # chord([A, B, ...], C) 

1362 return self.run(tasks, body, args, task_id=task_id, **merged_options) 

1363 

1364 def apply(self, args=None, kwargs=None, 

1365 propagate=True, body=None, **options): 

1366 args = args if args else () 

1367 kwargs = kwargs if kwargs else {} 

1368 body = self.body if body is None else body 

1369 tasks = (self.tasks.clone() if isinstance(self.tasks, group) 

1370 else group(self.tasks, app=self.app)) 

1371 return body.apply( 

1372 args=(tasks.apply(args, kwargs).get(propagate=propagate),), 

1373 ) 

1374 

1375 def _traverse_tasks(self, tasks, value=None): 

1376 stack = deque(tasks) 

1377 while stack: 

1378 task = stack.popleft() 

1379 if isinstance(task, group): 

1380 stack.extend(task.tasks) 

1381 elif isinstance(task, _chain) and isinstance(task.tasks[-1], group): 

1382 stack.extend(task.tasks[-1].tasks) 

1383 else: 

1384 yield task if value is None else value 

1385 

1386 def __length_hint__(self): 

1387 tasks = (self.tasks.tasks if isinstance(self.tasks, group) 

1388 else self.tasks) 

1389 return sum(self._traverse_tasks(tasks, 1)) 

1390 

1391 def run(self, header, body, partial_args, app=None, interval=None, 

1392 countdown=1, max_retries=None, eager=False, 

1393 task_id=None, **options): 

1394 app = app or self._get_app(body) 

1395 group_id = header.options.get('task_id') or uuid() 

1396 root_id = body.options.get('root_id') 

1397 body.chord_size = self.__length_hint__() 

1398 options = dict(self.options, **options) if options else self.options 

1399 if options: 

1400 options.pop('task_id', None) 

1401 body.options.update(options) 

1402 

1403 bodyres = body.freeze(task_id, root_id=root_id) 

1404 

1405 # Chains should not be passed to the header tasks. See #3771 

1406 options.pop('chain', None) 

1407 # Neither should chords, for deeply nested chords to work 

1408 options.pop('chord', None) 

1409 options.pop('task_id', None) 

1410 

1411 header_result = header.freeze(group_id=group_id, chord=body, root_id=root_id) 

1412 

1413 if len(header_result) > 0: 

1414 app.backend.apply_chord( 

1415 header_result, 

1416 body, 

1417 interval=interval, 

1418 countdown=countdown, 

1419 max_retries=max_retries, 

1420 ) 

1421 header_result = header(*partial_args, task_id=group_id, **options) 

1422 # The execution of a chord body is normally triggered by its header's 

1423 # tasks completing. If the header is empty this will never happen, so 

1424 # we execute the body manually here. 

1425 else: 

1426 body.delay([]) 

1427 

1428 bodyres.parent = header_result 

1429 return bodyres 

1430 

1431 def clone(self, *args, **kwargs): 

1432 signature = Signature.clone(self, *args, **kwargs) 

1433 # need to make copy of body 

1434 try: 

1435 signature.kwargs['body'] = maybe_signature( 

1436 signature.kwargs['body'], clone=True) 

1437 except (AttributeError, KeyError): 

1438 pass 

1439 return signature 

1440 

1441 def link(self, callback): 

1442 self.body.link(callback) 

1443 return callback 

1444 

1445 def link_error(self, errback): 

1446 self.body.link_error(errback) 

1447 return errback 

1448 

1449 def set_immutable(self, immutable): 

1450 # changes mutability of header only, not callback. 

1451 for task in self.tasks: 

1452 task.set_immutable(immutable) 

1453 

1454 def __repr__(self): 

1455 if self.body: 

1456 if isinstance(self.body, _chain): 

1457 return remove_repeating_from_task( 

1458 self.body.tasks[0]['task'], 

1459 '%({0} | {1!r})'.format( 

1460 self.body.tasks[0].reprcall(self.tasks), 

1461 chain(self.body.tasks[1:], app=self._app), 

1462 ), 

1463 ) 

1464 return '%' + remove_repeating_from_task( 

1465 self.body['task'], self.body.reprcall(self.tasks)) 

1466 return '<chord without body: {0.tasks!r}>'.format(self) 

1467 

1468 @cached_property 

1469 def app(self): 

1470 return self._get_app(self.body) 

1471 

1472 def _get_app(self, body=None): 

1473 app = self._app 

1474 if app is None: 

1475 try: 

1476 tasks = self.tasks.tasks # is a group 

1477 except AttributeError: 

1478 tasks = self.tasks 

1479 if len(tasks): 

1480 app = tasks[0]._app 

1481 if app is None and body is not None: 

1482 app = body._app 

1483 return app if app is not None else current_app 

1484 

1485 tasks = getitem_property('kwargs.header', 'Tasks in chord header.') 

1486 body = getitem_property('kwargs.body', 'Body task of chord.') 

1487 

1488 

1489def signature(varies, *args, **kwargs): 

1490 """Create new signature. 

1491 

1492 - if the first argument is a signature already then it's cloned. 

1493 - if the first argument is a dict, then a Signature version is returned. 

1494 

1495 Returns: 

1496 Signature: The resulting signature. 

1497 """ 

1498 app = kwargs.get('app') 

1499 if isinstance(varies, dict): 

1500 if isinstance(varies, abstract.CallableSignature): 

1501 return varies.clone() 

1502 return Signature.from_dict(varies, app=app) 

1503 return Signature(varies, *args, **kwargs) 

1504 

1505 

1506subtask = signature # noqa: E305 XXX compat 

1507 

1508 

1509def maybe_signature(d, app=None, clone=False): 

1510 """Ensure obj is a signature, or None. 

1511 

1512 Arguments: 

1513 d (Optional[Union[abstract.CallableSignature, Mapping]]): 

1514 Signature or dict-serialized signature. 

1515 app (celery.Celery): 

1516 App to bind signature to. 

1517 clone (bool): 

1518 If d' is already a signature, the signature 

1519 will be cloned when this flag is enabled. 

1520 

1521 Returns: 

1522 Optional[abstract.CallableSignature] 

1523 """ 

1524 if d is not None: 

1525 if isinstance(d, abstract.CallableSignature): 

1526 if clone: 

1527 d = d.clone() 

1528 elif isinstance(d, dict): 

1529 d = signature(d) 

1530 

1531 if app is not None: 

1532 d._app = app 

1533 return d 

1534 

1535 

1536maybe_subtask = maybe_signature # noqa: E305 XXX compat