Coverage for cc_modules/celery.py: 41%
133 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
1#!/usr/bin/env python
3"""
4camcops_server/cc_modules/celery.py
6===============================================================================
8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry.
9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk).
11 This file is part of CamCOPS.
13 CamCOPS is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
18 CamCOPS is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
26===============================================================================
28**Celery app.**
30Basic steps to set up Celery:
32- Our app will be "camcops_server.cc_modules".
33- Within that, Celery expects "celery.py", in which configuration is set up
34 by defining the ``app`` object.
35- Also, in ``__init__.py``, we should import that app. (No, scratch that; not
36 necessary.)
37- That makes ``@shared_task`` work in all other modules here.
38- Finally, here, we ask Celery to scan ``tasks.py`` to find tasks.
40Modified:
42- The ``@shared_task`` decorator doesn't offer all the options that
43 ``@app.task`` has. Let's skip ``@shared_task`` and the increased faff that
44 entails.
46The difficult part seems to be getting a broker URL in the config.
48- If we load the config here, from ``celery.py``, then if the config uses any
49 SQLAlchemy objects, it'll crash because some aren't imported.
50- A better way is to delay configuring the app.
51- But also, it is very tricky if the config uses SQLAlchemy objects; so it
52 shouldn't.
54Note also re logging:
56- The log here is configured (at times, at least) by Celery, so uses its log
57 settings. At the time of startup, that looks like plain ``print()``
58 statements.
60**In general, prefer delayed imports during actual tasks. Otherwise circular
61imports are very hard to avoid.**
63If using a separate ``celery_tasks.py`` file:
65- Import this only after celery.py, or the decorators will fail.
67- If you see this error from ``camcops_server launch_workers`` when using a
68 separate ``celery_tasks.py`` file:
70 .. code-block:: none
72 [2018-12-26 21:08:01,316: ERROR/MainProcess] Received unregistered task of type 'camcops_server.cc_modules.celery_tasks.export_to_recipient_backend'.
73 The message has been ignored and discarded.
75 Did you remember to import the module containing this task?
76 Or maybe you're using relative imports?
78 Please see
79 https://docs.celeryq.org/en/latest/internals/protocol.html
80 for more information.
82 The full contents of the message body was:
83 '[["recipient_email_rnc"], {}, {"callbacks": null, "errbacks": null, "chain": null, "chord": null}]' (98b)
84 Traceback (most recent call last):
85 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/worker/consumer/consumer.py", line 558, in on_task_received
86 strategy = strategies[type_]
87 KeyError: 'camcops_server.cc_modules.celery_tasks.export_to_recipient_backend'
89 then (1) run with ``--verbose``, which will show you the list of registered
90 tasks; (2) note that everything here is absent; (3) insert a "crash" line at
91 the top of this file and re-run; (4) note what's importing this file too
92 early.
94General advice:
96- https://medium.com/@taylorhughes/three-quick-tips-from-two-years-with-celery-c05ff9d7f9eb
98Task decorator options:
100- https://docs.celeryproject.org/en/latest/reference/celery.app.task.html
101- ``bind``: makes the first argument a ``self`` parameter to manipulate the
102 task itself;
103 https://docs.celeryproject.org/en/latest/userguide/tasks.html#example
104- ``acks_late`` (for the decorator) or ``task_acks_late``: see
106 - https://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_acks_late
107 - https://docs.celeryproject.org/en/latest/faq.html#faq-acks-late-vs-retry
108 - Here I am retrying on failure with exponential backoff, but not using
109 ``acks_late`` in addition.
111""" # noqa
113from contextlib import contextmanager
114import logging
115import os
116from typing import Any, Dict, TYPE_CHECKING
118from cardinal_pythonlib.json.serialize import json_encode, json_decode
119from cardinal_pythonlib.logs import BraceStyleAdapter
120from celery import Celery, current_task
121from kombu.serialization import register
123# TODO: Investigate
124# "from numpy.random import uniform" leads to uniform ending up in the
125# documentation for this file and Sphinx error:
126# celery.py:docstring of camcops_server.cc_modules.celery.uniform:9:
127# undefined label: random-quick-start
128from numpy import random
130# noinspection PyUnresolvedReferences
131import camcops_server.cc_modules.cc_all_models # import side effects (ensure all models registered) # noqa
133if TYPE_CHECKING:
134 from celery.app.task import Task as CeleryTask
135 from camcops_server.cc_modules.cc_export import DownloadOptions
136 from camcops_server.cc_modules.cc_request import CamcopsRequest
137 from camcops_server.cc_modules.cc_taskcollection import TaskCollection
139log = BraceStyleAdapter(logging.getLogger(__name__))
142# =============================================================================
143# Constants
144# =============================================================================
146CELERY_APP_NAME = "camcops_server.cc_modules"
147# CELERY_TASKS_MODULE = "celery_tasks"
148# ... look for "celery_tasks.py" (as opposed to the more common "tasks.py")
150CELERY_TASK_MODULE_NAME = CELERY_APP_NAME + ".celery"
152CELERY_SOFT_TIME_LIMIT_SEC = 300.0
153MAX_RETRIES = 10
154RETRY_MIN_DELAY_S = 5.0
155RETRY_MAX_DELAY_S = 60.0
158# =============================================================================
159# Configuration
160# =============================================================================
162register(
163 "json",
164 json_encode,
165 json_decode,
166 content_type="application/json",
167 content_encoding="utf-8",
168)
171def get_celery_settings_dict() -> Dict[str, Any]:
172 """
173 Returns a dictionary of settings to configure Celery.
174 """
175 log.debug("Configuring Celery")
176 from camcops_server.cc_modules.cc_config import (
177 CrontabEntry,
178 get_default_config_from_os_env,
179 ) # delayed import
181 config = get_default_config_from_os_env()
183 # -------------------------------------------------------------------------
184 # Schedule
185 # -------------------------------------------------------------------------
186 schedule = {} # type: Dict[str, Any]
188 # -------------------------------------------------------------------------
189 # User-defined schedule entries
190 # -------------------------------------------------------------------------
191 for crontab_entry in config.crontab_entries:
192 recipient_name = crontab_entry.content
193 schedule_name = f"export_to_{recipient_name}"
194 log.debug(
195 "Adding regular export job {}: crontab: {}",
196 schedule_name,
197 crontab_entry,
198 )
199 schedule[schedule_name] = {
200 "task": CELERY_TASK_MODULE_NAME + ".export_to_recipient_backend",
201 "schedule": crontab_entry.get_celery_schedule(),
202 "args": (recipient_name,),
203 }
205 # -------------------------------------------------------------------------
206 # Housekeeping once per minute
207 # -------------------------------------------------------------------------
208 housekeeping_crontab = CrontabEntry(minute="*", content="dummy")
209 schedule["housekeeping"] = {
210 "task": CELERY_TASK_MODULE_NAME + ".housekeeping",
211 "schedule": housekeeping_crontab.get_celery_schedule(),
212 }
214 # -------------------------------------------------------------------------
215 # Final Celery settings
216 # -------------------------------------------------------------------------
217 return {
218 "beat_schedule": schedule,
219 "broker_url": config.celery_broker_url,
220 "timezone": config.schedule_timezone,
221 "task_annotations": {
222 "camcops_server.cc_modules.celery.export_task_backend": {
223 "rate_limit": config.celery_export_task_rate_limit
224 }
225 },
226 # "worker_log_color": True, # true by default for consoles anyway
227 }
230# =============================================================================
231# The Celery app
232# =============================================================================
234celery_app = Celery()
235celery_app.add_defaults(get_celery_settings_dict())
236# celery_app.autodiscover_tasks([CELERY_APP_NAME],
237# related_name=CELERY_TASKS_MODULE)
239_ = """
241@celery_app.on_configure.connect
242def _app_on_configure(**kwargs) -> None:
243 log.critical("@celery_app.on_configure: {!r}", kwargs)
246@celery_app.on_after_configure.connect
247def _app_on_after_configure(**kwargs) -> None:
248 log.critical("@celery_app.on_after_configure: {!r}", kwargs)
250"""
253# =============================================================================
254# Test tasks
255# =============================================================================
258@celery_app.task(bind=True)
259def debug_task(self) -> None:
260 """
261 Test as follows:
263 .. code-block:: python
265 from camcops_server.cc_modules.celery import *
266 debug_task.delay()
268 and also launch workers with ``camcops_server launch_workers``.
270 For a bound task, the first (``self``) argument is the task instance; see
271 https://docs.celeryproject.org/en/latest/userguide/tasks.html#bound-tasks
273 """
274 log.info(f"self: {self!r}")
275 log.info(f"Backend: {current_task.backend}")
278@celery_app.task
279def debug_task_add(a: float, b: float) -> float:
280 """
281 Test as follows:
283 .. code-block:: python
285 from camcops_server.cc_modules.celery import *
286 debug_task_add.delay()
287 """
288 result = a + b
289 log.info("a = {}, b = {} => a + b = {}", a, b, result)
290 return result
293# =============================================================================
294# Exponential backoff
295# =============================================================================
298def backoff_delay_s(attempts: int) -> float:
299 """
300 Return a backoff delay, in seconds, given a number of attempts.
302 The delay increases very rapidly with the number of attempts:
303 1, 2, 4, 8, 16, 32, ...
305 As per https://blog.balthazar-rouberol.com/celery-best-practices.
307 """
308 return 2.0**attempts
311def jittered_delay_s() -> float:
312 """
313 Returns a retry delay, in seconds, that is jittered.
314 """
315 return random.uniform(RETRY_MIN_DELAY_S, RETRY_MAX_DELAY_S)
318@contextmanager
319def retry_backoff_if_raises(self: "CeleryTask") -> None:
320 """
321 Context manager to retry a Celery task if an exception is raised, using a
322 "backoff" method.
323 """
324 try:
325 yield
326 except Exception as exc:
327 delay_s = backoff_delay_s(self.request.retries)
328 log.error(
329 "Task failed. Backing off. Will retry after {} s. "
330 "Error was:\n{}",
331 delay_s,
332 exc,
333 )
334 self.retry(countdown=delay_s, exc=exc)
337@contextmanager
338def retry_jitter_if_raises(self: "CeleryTask") -> None:
339 """
340 Context manager to retry a Celery task if an exception is raised, using a
341 "jittered delay" method.
342 """
343 try:
344 yield
345 except Exception as exc:
346 delay_s = jittered_delay_s()
347 log.error(
348 "Task failed. Will retry after jittered delay: {} s. "
349 "Error was:\n{}",
350 delay_s,
351 exc,
352 )
353 self.retry(countdown=delay_s, exc=exc)
356# =============================================================================
357# Controlling tasks
358# =============================================================================
361def purge_jobs() -> None:
362 """
363 Purge all jobs from the Celery queue.
364 """
365 log.info("Purging back-end (Celery) jobs")
366 celery_app.control.purge()
367 log.info("... purged.")
370# =============================================================================
371# Note re request creation and context manager
372# =============================================================================
373# NOTE:
374# - You MUST use some sort of context manager to handle requests here, because
375# the normal Pyramid router [which ordinarily called the "finished" callbacks
376# via request._process_finished_callbacks()] will not be plumbed in.
377# - For debugging, use the MySQL command
378# SELECT * FROM information_schema.innodb_locks;
381# =============================================================================
382# Export tasks
383# =============================================================================
386@celery_app.task(
387 bind=True,
388 ignore_result=True,
389 max_retries=MAX_RETRIES,
390 soft_time_limit=CELERY_SOFT_TIME_LIMIT_SEC,
391)
392def export_task_backend(
393 self: "CeleryTask", recipient_name: str, basetable: str, task_pk: int
394) -> None:
395 """
396 This function exports a single task but does so with only simple (string,
397 integer) information, so it can be called via the Celery task queue.
399 - Calls :func:`camcops_server.cc_modules.cc_export.export_task`.
401 Args:
402 self: the Celery task, :class:`celery.app.task.Task`
403 recipient_name: export recipient name (as per the config file)
404 basetable: name of the task's base table
405 task_pk: server PK of the task
406 """
407 from camcops_server.cc_modules.cc_export import (
408 export_task,
409 ) # delayed import
410 from camcops_server.cc_modules.cc_request import (
411 command_line_request_context,
412 ) # delayed import
413 from camcops_server.cc_modules.cc_taskfactory import (
414 task_factory_no_security_checks,
415 ) # delayed import
417 with retry_backoff_if_raises(self):
418 with command_line_request_context() as req:
419 recipient = req.get_export_recipient(recipient_name)
420 task = task_factory_no_security_checks(
421 req.dbsession, basetable, task_pk
422 )
423 if task is None:
424 log.error(
425 "export_task_backend for recipient {!r}: No task "
426 "found for {} {}",
427 recipient_name,
428 basetable,
429 task_pk,
430 )
431 return
432 export_task(req, recipient, task)
435@celery_app.task(
436 bind=True,
437 ignore_result=True,
438 max_retries=MAX_RETRIES,
439 soft_time_limit=CELERY_SOFT_TIME_LIMIT_SEC,
440)
441def export_to_recipient_backend(
442 self: "CeleryTask", recipient_name: str
443) -> None:
444 """
445 From the backend, exports all pending tasks for a given recipient.
447 - Calls :func:`camcops_server.cc_modules.cc_export.export`.
449 There are two ways of doing this, when we call
450 :func:`camcops_server.cc_modules.cc_export.export`. If we set
451 ``schedule_via_backend=True``, this backend job fires up a whole bunch of
452 other backend jobs, one per task to export. If we set
453 ``schedule_via_backend=False``, our current backend job does all the work.
455 Which is best?
457 - Well, keeping it to one job is a bit simpler, perhaps.
458 - But everything is locked independently so we can do the multi-job
459 version, and we may as well use all the workers available. So my thought
460 was to use ``schedule_via_backend=True``.
461 - However, that led to database deadlocks (multiple processes trying to
462 write a new ExportRecipient).
463 - With some bugfixes to equality checking and a global lock (see
464 :meth:`camcops_server.cc_modules.cc_config.CamcopsConfig.get_master_export_recipient_lockfilename`),
465 we can try again with ``True``.
466 - Yup, works nicely.
468 Args:
469 self: the Celery task, :class:`celery.app.task.Task`
470 recipient_name: export recipient name (as per the config file)
471 """
472 from camcops_server.cc_modules.cc_export import export # delayed import
473 from camcops_server.cc_modules.cc_request import (
474 command_line_request_context,
475 ) # delayed import
477 with retry_backoff_if_raises(self):
478 with command_line_request_context() as req:
479 export(
480 req,
481 recipient_names=[recipient_name],
482 schedule_via_backend=True,
483 )
486@celery_app.task(
487 bind=True,
488 ignore_result=True,
489 max_retries=MAX_RETRIES,
490 soft_time_limit=CELERY_SOFT_TIME_LIMIT_SEC,
491)
492def email_basic_dump(
493 self: "CeleryTask",
494 collection: "TaskCollection",
495 options: "DownloadOptions",
496) -> None:
497 """
498 Send a research dump to the user via e-mail.
500 Args:
501 self:
502 the Celery task, :class:`celery.app.task.Task`
503 collection:
504 a
505 :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection`
506 options:
507 :class:`camcops_server.cc_modules.cc_export.DownloadOptions`
508 governing the download
509 """
510 from camcops_server.cc_modules.cc_export import (
511 make_exporter,
512 ) # delayed import
513 from camcops_server.cc_modules.cc_request import (
514 command_line_request_context,
515 ) # delayed import
517 with retry_backoff_if_raises(self):
518 # Create request for a specific user, so the auditing is correct.
519 with command_line_request_context(user_id=options.user_id) as req:
520 collection.set_request(req)
521 exporter = make_exporter(
522 req=req, collection=collection, options=options
523 )
524 exporter.send_by_email()
527@celery_app.task(
528 bind=True,
529 ignore_result=True,
530 max_retries=MAX_RETRIES,
531 soft_time_limit=CELERY_SOFT_TIME_LIMIT_SEC,
532)
533def create_user_download(
534 self: "CeleryTask",
535 collection: "TaskCollection",
536 options: "DownloadOptions",
537) -> None:
538 """
539 Create a research dump file for the user to download later.
540 Let them know by e-mail.
542 Args:
543 self:
544 the Celery task, :class:`celery.app.task.Task`
545 collection:
546 a
547 :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection`
548 options:
549 :class:`camcops_server.cc_modules.cc_export.DownloadOptions`
550 governing the download
551 """
552 from camcops_server.cc_modules.cc_export import (
553 make_exporter,
554 ) # delayed import
555 from camcops_server.cc_modules.cc_request import (
556 command_line_request_context,
557 ) # delayed import
559 with retry_backoff_if_raises(self):
560 # Create request for a specific user, so the auditing is correct.
561 with command_line_request_context(user_id=options.user_id) as req:
562 collection.set_request(req)
563 exporter = make_exporter(
564 req=req, collection=collection, options=options
565 )
566 exporter.create_user_download_and_email()
569# =============================================================================
570# Housekeeping
571# =============================================================================
574def delete_old_user_downloads(req: "CamcopsRequest") -> None:
575 """
576 Deletes user download files that are past their expiry time.
578 Args:
579 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
580 """
581 from camcops_server.cc_modules.cc_export import (
582 UserDownloadFile,
583 ) # delayed import
585 now = req.now
586 lifetime = req.user_download_lifetime_duration
587 oldest_allowed = now - lifetime
588 log.debug(f"Deleting any user download files older than {oldest_allowed}")
589 for root, dirs, files in os.walk(req.config.user_download_dir):
590 for f in files:
591 udf = UserDownloadFile(filename=f, directory=root)
592 if udf.older_than(oldest_allowed):
593 udf.delete()
596@celery_app.task(
597 bind=False, ignore_result=True, soft_time_limit=CELERY_SOFT_TIME_LIMIT_SEC
598)
599def housekeeping() -> None:
600 """
601 Function that is run regularly to do cleanup tasks.
603 (Remember that the ``bind`` parameter to ``@celery_app.task()`` means that
604 the first argument to the function, typically called ``self``, is the
605 Celery task. We don't need it here. See
606 https://docs.celeryproject.org/en/latest/userguide/tasks.html#bound-tasks.)
607 """
608 from camcops_server.cc_modules.cc_request import (
609 command_line_request_context,
610 ) # delayed import
611 from camcops_server.cc_modules.cc_session import (
612 CamcopsSession,
613 ) # delayed import
614 from camcops_server.cc_modules.cc_user import (
615 SecurityAccountLockout,
616 SecurityLoginFailure,
617 ) # delayed import
619 log.debug("Housekeeping!")
620 with command_line_request_context() as req:
621 # ---------------------------------------------------------------------
622 # Housekeeping tasks
623 # ---------------------------------------------------------------------
624 # We had a problem with MySQL locking here (two locks open for what
625 # appeared to be a single delete, followed by a lock timeout). Seems to
626 # be working now.
627 CamcopsSession.delete_old_sessions(req)
628 SecurityAccountLockout.delete_old_account_lockouts(req)
629 SecurityLoginFailure.clear_dummy_login_failures_if_necessary(req)
630 delete_old_user_downloads(req)