Coverage for cc_modules/cc_taskindex.py: 34%
290 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
1#!/usr/bin/env python
3"""
4camcops_server/cc_modules/cc_taskindex.py
6===============================================================================
8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry.
9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk).
11 This file is part of CamCOPS.
13 CamCOPS is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
18 CamCOPS is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
26===============================================================================
28**Server-side task index.**
30Note in particular that if you, as a developer, change the ``is_complete()``
31criteria for a task, you should cause the server index to be rebuilt (because
32it caches ``is_complete()`` information).
34"""
36import logging
37from typing import List, Optional, Type, TYPE_CHECKING
39from cardinal_pythonlib.logs import BraceStyleAdapter
40from cardinal_pythonlib.reprfunc import simple_repr
41from cardinal_pythonlib.sqlalchemy.session import get_engine_from_session
42from cardinal_pythonlib.sqlalchemy.schema import table_exists
43from cardinal_pythonlib.sqlalchemy.sqlserver import (
44 if_sqlserver_disable_constraints_triggers,
45)
46from pendulum import DateTime as Pendulum
47import pyramid.httpexceptions as exc
48from sqlalchemy.orm import relationship, Session as SqlASession
49from sqlalchemy.sql.expression import and_, exists, join, literal, select
50from sqlalchemy.sql.schema import Column, ForeignKey, Table
51from sqlalchemy.sql.sqltypes import BigInteger, Boolean, DateTime, Integer
53from camcops_server.cc_modules.cc_client_api_core import (
54 BatchDetails,
55 fail_user_error,
56 UploadTableChanges,
57)
58from camcops_server.cc_modules.cc_constants import ERA_NOW
59from camcops_server.cc_modules.cc_idnumdef import IdNumDefinition
60from camcops_server.cc_modules.cc_patient import Patient
61from camcops_server.cc_modules.cc_patientidnum import PatientIdNum
62from camcops_server.cc_modules.cc_sqla_coltypes import (
63 EraColType,
64 isotzdatetime_to_utcdatetime,
65 PendulumDateTimeAsIsoTextColType,
66 TableNameColType,
67)
68from camcops_server.cc_modules.cc_sqlalchemy import Base
69from camcops_server.cc_modules.cc_task import (
70 all_task_tablenames,
71 tablename_to_task_class_dict,
72 Task,
73)
74from camcops_server.cc_modules.cc_user import User
76if TYPE_CHECKING:
77 from camcops_server.cc_modules.cc_request import CamcopsRequest
79log = BraceStyleAdapter(logging.getLogger(__name__))
82# =============================================================================
83# Helper functions
84# =============================================================================
87def task_factory_unfiltered(
88 dbsession: SqlASession, basetable: str, serverpk: int
89) -> Optional[Task]:
90 """
91 Load a task from the database and return it.
92 No permission filtering is performed. (Used by
93 :class:`camcops_server.cc_modules.cc_taskindex.TaskIndexEntry`.)
95 Args:
96 dbsession: a :class:`sqlalchemy.orm.session.Session`
97 basetable: name of the task's base table
98 serverpk: server PK of the task
100 Returns:
101 the task, or ``None`` if the PK doesn't exist
103 Raises:
104 :exc:`HTTPBadRequest` if the table doesn't exist
105 """
106 d = tablename_to_task_class_dict()
107 try:
108 cls = d[basetable] # may raise KeyError
109 except KeyError:
110 raise exc.HTTPBadRequest(f"No such task table: {basetable!r}")
111 # noinspection PyProtectedMember
112 q = dbsession.query(cls).filter(cls._pk == serverpk)
113 return q.first()
116# =============================================================================
117# PatientIdNumIndexEntry
118# =============================================================================
121class PatientIdNumIndexEntry(Base):
122 """
123 Represents a server index entry for a
124 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`.
126 - Only current ID numbers are indexed.
127 """
129 __tablename__ = "_idnum_index"
131 idnum_pk = Column(
132 "idnum_pk",
133 Integer,
134 primary_key=True,
135 index=True,
136 comment="Server primary key of the PatientIdNum "
137 "(and of the PatientIdNumIndexEntry)",
138 )
139 indexed_at_utc = Column(
140 "indexed_at_utc",
141 DateTime,
142 nullable=False,
143 comment="When this index entry was created",
144 )
146 # noinspection PyProtectedMember
147 patient_pk = Column(
148 "patient_pk",
149 Integer,
150 ForeignKey(Patient._pk),
151 index=True,
152 comment="Server primary key of the Patient",
153 )
154 which_idnum = Column(
155 "which_idnum",
156 Integer,
157 ForeignKey(IdNumDefinition.which_idnum),
158 nullable=False,
159 index=True,
160 comment="Which of the server's ID numbers is this?",
161 )
162 idnum_value = Column(
163 "idnum_value", BigInteger, comment="The value of the ID number"
164 )
166 # Relationships:
167 patient = relationship(Patient)
169 def __repr__(self) -> str:
170 return simple_repr(
171 self, ["idnum_pk", "patient_pk", "which_idnum", "idnum_value"]
172 )
174 # -------------------------------------------------------------------------
175 # Create
176 # -------------------------------------------------------------------------
178 @classmethod
179 def make_from_idnum(cls, idnum: PatientIdNum) -> "PatientIdNumIndexEntry":
180 """
181 Returns an ID index entry for the specified
182 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`. The
183 returned index requires inserting into a database session.
184 """
185 # noinspection PyProtectedMember
186 assert idnum._current, "Only index current PatientIdNum objects"
187 index = cls()
188 index.idnum_pk = idnum.pk
189 index.patient_pk = idnum.get_patient_server_pk()
190 index.which_idnum = idnum.which_idnum
191 index.idnum_value = idnum.idnum_value
192 index.indexed_at_utc = Pendulum.now()
193 return index
195 @classmethod
196 def index_idnum(cls, idnum: PatientIdNum, session: SqlASession) -> None:
197 """
198 Indexes an ID number and inserts the index into the database.
200 Args:
201 idnum: a
202 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`
203 session:
204 an SQLAlchemy Session
205 """
206 index = cls.make_from_idnum(idnum)
207 session.add(index)
209 @classmethod
210 def unindex_patient(cls, patient: Patient, session: SqlASession) -> None:
211 """
212 Removes all ID number indexes from the database for a patient.
214 Args:
215 patient:
216 :class:`camcops_server.cc_modules.cc_patient.Patient`
217 session:
218 an SQLAlchemy Session
219 """
221 # noinspection PyUnresolvedReferences
222 idxtable = cls.__table__ # type: Table
223 idxcols = idxtable.columns
224 # noinspection PyProtectedMember
225 session.execute(
226 idxtable.delete().where(idxcols.patient_pk == patient._pk)
227 )
229 # -------------------------------------------------------------------------
230 # Regenerate index
231 # -------------------------------------------------------------------------
233 @classmethod
234 def rebuild_idnum_index(
235 cls, session: SqlASession, indexed_at_utc: Pendulum
236 ) -> None:
237 """
238 Rebuilds the index entirely. Uses SQLAlchemy Core (not ORM) for speed.
240 Args:
241 session: an SQLAlchemy Session
242 indexed_at_utc: current time in UTC
243 """
244 log.info("Rebuilding patient ID number index")
245 # noinspection PyUnresolvedReferences
246 indextable = PatientIdNumIndexEntry.__table__ # type: Table
247 indexcols = indextable.columns
248 # noinspection PyUnresolvedReferences
249 idnumtable = PatientIdNum.__table__ # type: Table
250 idnumcols = idnumtable.columns
251 # noinspection PyUnresolvedReferences
252 patienttable = Patient.__table__ # type: Table
253 patientcols = patienttable.columns
255 # Delete all entries
256 with if_sqlserver_disable_constraints_triggers(
257 session, indextable.name
258 ):
259 session.execute(indextable.delete())
261 # Create new ones
262 # noinspection PyProtectedMember,PyPep8
263 session.execute(
264 indextable.insert().from_select(
265 # Target:
266 [
267 indexcols.idnum_pk,
268 indexcols.indexed_at_utc,
269 indexcols.patient_pk,
270 indexcols.which_idnum,
271 indexcols.idnum_value,
272 ],
273 # Source:
274 (
275 select(
276 [
277 idnumcols._pk,
278 literal(indexed_at_utc),
279 patientcols._pk,
280 idnumcols.which_idnum,
281 idnumcols.idnum_value,
282 ]
283 )
284 .select_from(
285 join(
286 idnumtable,
287 patienttable,
288 and_(
289 idnumcols._device_id == patientcols._device_id,
290 idnumcols._era == patientcols._era,
291 idnumcols.patient_id == patientcols.id,
292 ),
293 )
294 )
295 .where(idnumcols._current == True) # noqa: E712
296 .where(idnumcols.idnum_value.isnot(None))
297 .where(patientcols._current == True) # noqa: E712
298 ),
299 )
300 )
302 # -------------------------------------------------------------------------
303 # Check index
304 # -------------------------------------------------------------------------
305 @classmethod
306 def check_index(
307 cls, session: SqlASession, show_all_bad: bool = False
308 ) -> bool:
309 """
310 Checks the index.
312 Args:
313 session:
314 an SQLAlchemy Session
315 show_all_bad:
316 show all bad entries? (If false, return upon the first)
318 Returns:
319 bool: is the index OK?
320 """
321 ok = True
323 log.info(
324 "Checking all patient ID number indexes represent valid entries"
325 )
326 # noinspection PyUnresolvedReferences,PyProtectedMember
327 q_idx_without_original = session.query(PatientIdNumIndexEntry).filter(
328 ~exists()
329 .select_from(
330 PatientIdNum.__table__.join(
331 Patient.__table__,
332 Patient.id == PatientIdNum.patient_id,
333 Patient._device_id == PatientIdNum._device_id,
334 Patient._era == PatientIdNum._era,
335 )
336 )
337 .where(
338 and_(
339 PatientIdNum._pk == PatientIdNumIndexEntry.idnum_pk,
340 PatientIdNum._current == True, # noqa: E712
341 PatientIdNum.which_idnum
342 == PatientIdNumIndexEntry.which_idnum,
343 PatientIdNum.idnum_value
344 == PatientIdNumIndexEntry.idnum_value,
345 Patient._pk == PatientIdNumIndexEntry.patient_pk,
346 Patient._current == True, # noqa: E712
347 )
348 )
349 )
350 for index in q_idx_without_original:
351 log.error(
352 "Patient ID number index without matching " "original: {!r}",
353 index,
354 )
355 ok = False
356 if not show_all_bad:
357 return ok
359 log.info("Checking all patient ID numbers have an index")
360 # noinspection PyUnresolvedReferences,PyProtectedMember
361 q_original_with_idx = session.query(PatientIdNum).filter(
362 PatientIdNum._current == True, # noqa: E712
363 PatientIdNum.idnum_value.isnot(None),
364 ~exists()
365 .select_from(PatientIdNumIndexEntry.__table__)
366 .where(
367 and_(
368 PatientIdNum._pk == PatientIdNumIndexEntry.idnum_pk,
369 PatientIdNum.which_idnum
370 == PatientIdNumIndexEntry.which_idnum, # noqa
371 PatientIdNum.idnum_value
372 == PatientIdNumIndexEntry.idnum_value, # noqa
373 )
374 ),
375 )
376 for orig in q_original_with_idx:
377 log.error("ID number without index entry: {!r}", orig)
378 ok = False
379 if not show_all_bad:
380 return ok
382 return ok
384 # -------------------------------------------------------------------------
385 # Update index at the point of upload from a device
386 # -------------------------------------------------------------------------
388 @classmethod
389 def update_idnum_index_for_upload(
390 cls,
391 session: SqlASession,
392 indexed_at_utc: Pendulum,
393 tablechanges: UploadTableChanges,
394 ) -> None:
395 """
396 Updates the index for a device's upload.
398 - Deletes index entries for records that are on the way out.
399 - Creates index entries for records that are on the way in.
400 - Should be called after both the Patient and PatientIdNum tables are
401 committed; see special ordering in
402 :func:`camcops_server.cc_modules.client_api.commit_all`.
404 Args:
405 session:
406 an SQLAlchemy Session
407 indexed_at_utc:
408 current time in UTC
409 tablechanges:
410 a :class:`camcops_server.cc_modules.cc_client_api_core.UploadTableChanges`
411 object describing the changes to a table
412 """ # noqa
413 # noinspection PyUnresolvedReferences
414 indextable = PatientIdNumIndexEntry.__table__ # type: Table
415 indexcols = indextable.columns
416 # noinspection PyUnresolvedReferences
417 idnumtable = PatientIdNum.__table__ # type: Table
418 idnumcols = idnumtable.columns
419 # noinspection PyUnresolvedReferences
420 patienttable = Patient.__table__ # type: Table
421 patientcols = patienttable.columns
423 # Delete the old
424 removal_pks = tablechanges.idnum_delete_index_pks
425 if removal_pks:
426 log.debug(
427 "Deleting old ID number indexes: server PKs {}", removal_pks
428 )
429 session.execute(
430 indextable.delete().where(
431 indextable.c.idnum_pk.in_(removal_pks)
432 )
433 )
435 # Create the new
436 addition_pks = tablechanges.idnum_add_index_pks
437 if addition_pks:
438 log.debug("Adding ID number indexes: server PKs {}", addition_pks)
439 # noinspection PyPep8,PyProtectedMember
440 session.execute(
441 indextable.insert().from_select(
442 # Target:
443 [
444 indexcols.idnum_pk,
445 indexcols.indexed_at_utc,
446 indexcols.patient_pk,
447 indexcols.which_idnum,
448 indexcols.idnum_value,
449 ],
450 # Source:
451 (
452 select(
453 [
454 idnumcols._pk,
455 literal(indexed_at_utc),
456 patientcols._pk,
457 idnumcols.which_idnum,
458 idnumcols.idnum_value,
459 ]
460 )
461 .select_from(
462 join(
463 idnumtable,
464 patienttable,
465 and_(
466 idnumcols._device_id
467 == patientcols._device_id, # noqa
468 idnumcols._era == patientcols._era,
469 idnumcols.patient_id == patientcols.id,
470 ),
471 )
472 )
473 .where(idnumcols._pk.in_(addition_pks))
474 .where(patientcols._current == True) # noqa: E712
475 ),
476 )
477 )
480# =============================================================================
481# TaskIndexEntry
482# =============================================================================
485class TaskIndexEntry(Base):
486 """
487 Represents a server index entry for a
488 :class:`camcops_server.cc_modules.cc_task.Task`.
490 - Only current tasks are indexed. This simplifies direct linking to patient
491 PKs.
492 """
494 __tablename__ = "_task_index"
496 index_entry_pk = Column(
497 "index_entry_pk",
498 Integer,
499 primary_key=True,
500 autoincrement=True,
501 comment="Arbitrary primary key of this index entry",
502 )
503 indexed_at_utc = Column(
504 "indexed_at_utc",
505 DateTime,
506 nullable=False,
507 comment="When this index entry was created",
508 )
510 # The next two fields link to our task:
511 task_table_name = Column(
512 "task_table_name",
513 TableNameColType,
514 index=True,
515 comment="Table name of the task's base table",
516 )
517 task_pk = Column(
518 "task_pk",
519 Integer,
520 index=True,
521 comment="Server primary key of the task",
522 )
523 # We can probably even represent this with an SQLAlchemy ORM relationship.
524 # This is polymorphic loading (we'll return objects of different types)
525 # based on concrete table inheritance (each type of object -- each task --
526 # has its own standalone table).
527 # However, there are warnings about the inefficiency of this; see
528 # https://docs.sqlalchemy.org/en/latest/orm/inheritance.html#concrete-table-inheritance
529 # and we are trying to be efficient. So let's do via task() below.
531 # This links to the task's patient, if there is one:
532 # noinspection PyProtectedMember
533 patient_pk = Column(
534 "patient_pk",
535 Integer,
536 ForeignKey(Patient._pk),
537 index=True,
538 comment="Server primary key of the patient (if applicable)",
539 )
541 # These fields allow us to filter tasks efficiently:
542 device_id = Column(
543 "device_id",
544 Integer,
545 ForeignKey("_security_devices.id"),
546 nullable=False,
547 index=True,
548 comment="ID of the source tablet device",
549 )
550 era = Column(
551 "era",
552 EraColType,
553 nullable=False,
554 index=True,
555 comment="Era (_era) field of the source record",
556 )
557 when_created_utc = Column(
558 "when_created_utc",
559 DateTime,
560 nullable=False,
561 index=True,
562 comment="Date/time this task instance was created (UTC)",
563 )
564 when_created_iso = Column(
565 "when_created_iso",
566 PendulumDateTimeAsIsoTextColType,
567 nullable=False,
568 index=True,
569 comment="Date/time this task instance was created (ISO 8601)",
570 ) # Pendulum on the Python side
571 when_added_batch_utc = Column(
572 "when_added_batch_utc",
573 DateTime,
574 nullable=False,
575 index=True,
576 comment="Date/time this task index was uploaded (UTC)",
577 )
578 adding_user_id = Column(
579 "adding_user_id",
580 Integer,
581 ForeignKey("_security_users.id"),
582 comment="ID of user that added this task",
583 )
584 group_id = Column(
585 "group_id",
586 Integer,
587 ForeignKey("_security_groups.id"),
588 nullable=False,
589 index=True,
590 comment="ID of group to which this task belongs",
591 )
592 task_is_complete = Column(
593 "task_is_complete",
594 Boolean,
595 nullable=False,
596 comment="Is the task complete (as judged by the server when the index "
597 "entry was created)?",
598 )
600 # Relationships:
601 patient = relationship(Patient)
602 _adding_user = relationship(User)
604 def __repr__(self) -> str:
605 return simple_repr(
606 self,
607 [
608 "index_entry_pk",
609 "task_table_name",
610 "task_pk",
611 "patient_pk",
612 "device_id",
613 "era",
614 "when_created_utc",
615 "when_created_iso",
616 "when_added_batch_utc",
617 "adding_user_id",
618 "group_id",
619 "task_is_complete",
620 ],
621 )
623 # -------------------------------------------------------------------------
624 # Fetch the task
625 # -------------------------------------------------------------------------
627 @property
628 def task(self) -> Optional[Task]:
629 """
630 Returns:
631 the associated :class:`camcops_server.cc_modules.cc_task.Task`, or
632 ``None`` if none exists.
634 Raises:
635 :exc:`HTTPBadRequest` if the table doesn't exist
636 """
637 dbsession = SqlASession.object_session(self)
638 assert dbsession, (
639 "TaskIndexEntry.task called on a TaskIndexEntry "
640 "that's not yet in a database session"
641 )
642 return task_factory_unfiltered(
643 dbsession, self.task_table_name, self.task_pk
644 )
646 # -------------------------------------------------------------------------
647 # Other properties mirroring those of Task, for duck typing
648 # -------------------------------------------------------------------------
650 @property
651 def is_anonymous(self) -> bool:
652 """
653 Is the task anonymous?
654 """
655 return self.patient_pk is None
657 def is_complete(self) -> bool:
658 """
659 Is the task complete?
660 """
661 return self.task_is_complete
663 @property
664 def _current(self) -> bool:
665 """
666 All task index entries represent complete tasks, so this always returns
667 ``True``.
668 """
669 return True
671 @property
672 def pk(self) -> int:
673 """
674 Return's the task's server PK.
675 """
676 return self.task_pk
678 @property
679 def tablename(self) -> str:
680 """
681 Returns the base table name of the task.
682 """
683 return self.task_table_name
685 @property
686 def shortname(self) -> str:
687 """
688 Returns the task's shortname.
689 """
690 d = tablename_to_task_class_dict()
691 taskclass = d[self.task_table_name]
692 return taskclass.shortname
694 def is_live_on_tablet(self) -> bool:
695 """
696 Is the task live on the source device (e.g. tablet)?
697 """
698 return self.era == ERA_NOW
700 @property
701 def when_created(self) -> Pendulum:
702 """
703 Returns the creation date/time as a Pendulum DateTime object.
704 """
705 return self.when_created_iso
707 def any_patient_idnums_invalid(self, req: "CamcopsRequest") -> bool:
708 """
709 Do we have a patient who has any invalid ID numbers?
711 Args:
712 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
713 """
714 idnums = self.get_patient_idnum_objects()
715 for idnum in idnums:
716 if not idnum.is_fully_valid(req):
717 return True
718 return False
720 def get_patient_idnum_objects(self) -> List[PatientIdNum]:
721 """
722 Gets all :class:`PatientIdNum` objects for the patient.
723 """
724 return self.patient.get_idnum_objects() if self.patient else []
726 # -------------------------------------------------------------------------
727 # Create
728 # -------------------------------------------------------------------------
730 @classmethod
731 def make_from_task(
732 cls, task: Task, indexed_at_utc: Pendulum
733 ) -> "TaskIndexEntry":
734 """
735 Returns a task index entry for the specified
736 :class:`camcops_server.cc_modules.cc_task.Task`. The
737 returned index requires inserting into a database session.
739 Args:
740 task:
741 a :class:`camcops_server.cc_modules.cc_task.Task`
742 indexed_at_utc:
743 current time in UTC
744 """
745 assert indexed_at_utc is not None, "Missing indexed_at_utc"
747 index = cls()
749 index.indexed_at_utc = indexed_at_utc
751 index.task_table_name = task.tablename
752 index.task_pk = task.pk
754 patient = task.patient
755 index.patient_pk = patient.pk if patient else None
757 index.device_id = task.device_id
758 index.era = task.era
759 index.when_created_utc = task.get_creation_datetime_utc()
760 index.when_created_iso = task.when_created
761 # noinspection PyProtectedMember
762 index.when_added_batch_utc = task._when_added_batch_utc
763 index.adding_user_id = task.get_adding_user_id()
764 index.group_id = task.group_id
765 index.task_is_complete = task.is_complete()
767 return index
769 @classmethod
770 def index_task(
771 cls, task: Task, session: SqlASession, indexed_at_utc: Pendulum
772 ) -> None:
773 """
774 Indexes a task and inserts the index into the database.
776 Args:
777 task:
778 a :class:`camcops_server.cc_modules.cc_task.Task`
779 session:
780 an SQLAlchemy Session
781 indexed_at_utc:
782 current time in UTC
783 """
784 index = cls.make_from_task(task, indexed_at_utc=indexed_at_utc)
785 session.add(index)
787 @classmethod
788 def unindex_task(cls, task: Task, session: SqlASession) -> None:
789 """
790 Removes a task index from the database.
792 Args:
793 task:
794 a :class:`camcops_server.cc_modules.cc_task.Task`
795 session:
796 an SQLAlchemy Session
797 """
799 # noinspection PyUnresolvedReferences
800 idxtable = cls.__table__ # type: Table
801 idxcols = idxtable.columns
802 tasktablename = task.__class__.tablename
803 session.execute(
804 idxtable.delete()
805 .where(idxcols.task_table_name == tasktablename)
806 .where(idxcols.task_pk == task.pk)
807 )
809 # -------------------------------------------------------------------------
810 # Regenerate index
811 # -------------------------------------------------------------------------
813 @classmethod
814 def rebuild_index_for_task_type(
815 cls,
816 session: SqlASession,
817 taskclass: Type[Task],
818 indexed_at_utc: Pendulum,
819 delete_first: bool = True,
820 ) -> None:
821 """
822 Rebuilds the index for a particular task type.
824 Args:
825 session: an SQLAlchemy Session
826 taskclass: a subclass of
827 :class:`camcops_server.cc_modules.cc_task.Task`
828 indexed_at_utc: current time in UTC
829 delete_first: delete old index entries first? Should always be True
830 unless called as part of a master rebuild that deletes
831 everything first.
832 """
833 # noinspection PyUnresolvedReferences
834 idxtable = cls.__table__ # type: Table
835 idxcols = idxtable.columns
836 tasktablename = taskclass.tablename
837 log.info("Rebuilding task index for {}", tasktablename)
838 # Delete all entries for this task
839 if delete_first:
840 session.execute(
841 idxtable.delete().where(idxcols.table_name == tasktablename)
842 )
843 # Create new entries
844 # noinspection PyPep8,PyUnresolvedReferences,PyProtectedMember
845 q = (
846 session.query(taskclass)
847 .filter(taskclass._current == True) # noqa: E712
848 .order_by(isotzdatetime_to_utcdatetime(taskclass.when_created))
849 )
850 for task in q:
851 cls.index_task(task, session, indexed_at_utc)
853 @classmethod
854 def rebuild_entire_task_index(
855 cls,
856 session: SqlASession,
857 indexed_at_utc: Pendulum,
858 skip_tasks_with_missing_tables: bool = False,
859 ) -> None:
860 """
861 Rebuilds the entire index.
863 Args:
864 session: an SQLAlchemy Session
865 indexed_at_utc: current time in UTC
866 skip_tasks_with_missing_tables: should we skip over tasks if their
867 tables are not in the database? (This is so we can rebuild an
868 index from a database upgrade, but not crash because newer
869 tasks haven't had their tables created yet.)
870 """
871 log.info("Rebuilding entire task index")
872 # noinspection PyUnresolvedReferences
873 idxtable = cls.__table__ # type: Table
875 # Delete all entries
876 with if_sqlserver_disable_constraints_triggers(session, idxtable.name):
877 session.execute(idxtable.delete())
879 # Now rebuild:
880 for taskclass in Task.all_subclasses_by_tablename():
881 if skip_tasks_with_missing_tables:
882 basetable = taskclass.tablename
883 engine = get_engine_from_session(session)
884 if not table_exists(engine, basetable):
885 continue
886 cls.rebuild_index_for_task_type(
887 session, taskclass, indexed_at_utc, delete_first=False
888 )
890 # -------------------------------------------------------------------------
891 # Update index at the point of upload from a device
892 # -------------------------------------------------------------------------
894 @classmethod
895 def update_task_index_for_upload(
896 cls,
897 session: SqlASession,
898 tablechanges: UploadTableChanges,
899 indexed_at_utc: Pendulum,
900 ) -> None:
901 """
902 Updates the index for a device's upload.
904 - Deletes index entries for records that are on the way out.
905 - Creates index entries for records that are on the way in.
906 - Deletes/recreates index entries for records being preserved.
908 Args:
909 session:
910 an SQLAlchemy Session
911 tablechanges:
912 a :class:`camcops_server.cc_modules.cc_client_api_core.UploadTableChanges`
913 object describing the changes to a table
914 indexed_at_utc:
915 current time in UTC
916 """ # noqa
917 tasktablename = tablechanges.tablename
918 d = tablename_to_task_class_dict()
919 try:
920 taskclass = d[tasktablename] # may raise KeyError
921 except KeyError:
922 fail_user_error(f"Bug: no such task table: {tasktablename!r}")
924 # noinspection PyUnresolvedReferences
925 idxtable = cls.__table__ # type: Table
926 idxcols = idxtable.columns
928 # Delete the old.
929 delete_index_pks = tablechanges.task_delete_index_pks
930 if delete_index_pks:
931 log.debug(
932 "Deleting old task indexes: {}, server PKs {}",
933 tasktablename,
934 delete_index_pks,
935 )
936 # noinspection PyProtectedMember
937 session.execute(
938 idxtable.delete()
939 .where(idxcols.task_table_name == tasktablename)
940 .where(idxcols.task_pk.in_(delete_index_pks))
941 )
943 # Create the new.
944 reindex_pks = tablechanges.task_reindex_pks
945 if reindex_pks:
946 log.debug(
947 "Recreating task indexes: {}, server PKs {}",
948 tasktablename,
949 reindex_pks,
950 )
951 # noinspection PyUnboundLocalVariable,PyProtectedMember
952 q = session.query(taskclass).filter(taskclass._pk.in_(reindex_pks))
953 for task in q:
954 cls.index_task(task, session, indexed_at_utc=indexed_at_utc)
956 # -------------------------------------------------------------------------
957 # Check index
958 # -------------------------------------------------------------------------
959 @classmethod
960 def check_index(
961 cls, session: SqlASession, show_all_bad: bool = False
962 ) -> bool:
963 """
964 Checks the index.
966 Args:
967 session:
968 an SQLAlchemy Session
969 show_all_bad:
970 show all bad entries? (If false, return upon the first)
972 Returns:
973 bool: is the index OK?
974 """
975 ok = True
977 log.info("Checking all task indexes represent valid entries")
978 for taskclass in Task.all_subclasses_by_tablename():
979 tasktablename = taskclass.tablename
980 log.debug("Checking {}", tasktablename)
981 # noinspection PyUnresolvedReferences,PyProtectedMember
982 q_idx_without_original = session.query(TaskIndexEntry).filter(
983 TaskIndexEntry.task_table_name == tasktablename,
984 ~exists()
985 .select_from(taskclass.__table__)
986 .where(
987 and_(
988 TaskIndexEntry.task_pk == taskclass._pk,
989 taskclass._current == True, # noqa: E712
990 )
991 ),
992 )
993 # No check for a valid patient at this time.
994 for index in q_idx_without_original:
995 log.error("Task index without matching original: {!r}", index)
996 ok = False
997 if not show_all_bad:
998 return ok
1000 log.info("Checking all tasks have an index")
1001 for taskclass in Task.all_subclasses_by_tablename():
1002 tasktablename = taskclass.tablename
1003 log.debug("Checking {}", tasktablename)
1004 # noinspection PyUnresolvedReferences,PyProtectedMember
1005 q_original_with_idx = session.query(taskclass).filter(
1006 taskclass._current == True, # noqa: E712
1007 ~exists()
1008 .select_from(TaskIndexEntry.__table__)
1009 .where(
1010 and_(
1011 TaskIndexEntry.task_pk == taskclass._pk,
1012 TaskIndexEntry.task_table_name == tasktablename,
1013 )
1014 ),
1015 )
1016 for orig in q_original_with_idx:
1017 log.error("Task without index entry: {!r}", orig)
1018 ok = False
1019 if not show_all_bad:
1020 return ok
1022 return ok
1025# =============================================================================
1026# Wide-ranging index update functions
1027# =============================================================================
1030def reindex_everything(
1031 session: SqlASession, skip_tasks_with_missing_tables: bool = False
1032) -> None:
1033 """
1034 Deletes from and rebuilds all server index tables.
1036 Args:
1037 session: an SQLAlchemy Session
1038 skip_tasks_with_missing_tables: should we skip over tasks if their
1039 tables are not in the database? (This is so we can rebuild an index
1040 from a database upgrade, but not crash because newer tasks haven't
1041 had their tables created yet.)
1042 """
1043 now = Pendulum.utcnow()
1044 log.info("Reindexing database; indexed_at_utc = {}", now)
1045 PatientIdNumIndexEntry.rebuild_idnum_index(session, now)
1046 TaskIndexEntry.rebuild_entire_task_index(
1047 session,
1048 now,
1049 skip_tasks_with_missing_tables=skip_tasks_with_missing_tables,
1050 )
1053def update_indexes_and_push_exports(
1054 req: "CamcopsRequest",
1055 batchdetails: BatchDetails,
1056 tablechanges: UploadTableChanges,
1057) -> None:
1058 """
1059 Update server indexes, if required.
1061 Also triggers background jobs to export "new arrivals", if required.
1063 Args:
1064 req: the :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
1065 batchdetails: the :class:`BatchDetails`
1066 tablechanges:
1067 a :class:`camcops_server.cc_modules.cc_client_api_core.UploadTableChanges`
1068 object describing the changes to a table
1069 """ # noqa
1070 tablename = tablechanges.tablename
1071 if tablename == PatientIdNum.__tablename__:
1072 # Update idnum index
1073 PatientIdNumIndexEntry.update_idnum_index_for_upload(
1074 session=req.dbsession,
1075 indexed_at_utc=batchdetails.batchtime,
1076 tablechanges=tablechanges,
1077 )
1078 elif tablename in all_task_tablenames():
1079 # Update task index
1080 TaskIndexEntry.update_task_index_for_upload(
1081 session=req.dbsession,
1082 tablechanges=tablechanges,
1083 indexed_at_utc=batchdetails.batchtime,
1084 )
1085 # Push exports
1086 recipients = req.all_push_recipients
1087 uploading_group_id = req.user.upload_group_id
1088 for recipient in recipients:
1089 recipient_name = recipient.recipient_name
1090 for pk in tablechanges.get_task_push_export_pks(
1091 recipient=recipient, uploading_group_id=uploading_group_id
1092 ):
1093 req.add_export_push_request(recipient_name, tablename, pk)
1094 # ... will be transmitted *after* the request performs COMMIT
1097def check_indexes(session: SqlASession, show_all_bad: bool = False) -> bool:
1098 """
1099 Checks all server index tables.
1101 Args:
1102 session:
1103 an SQLAlchemy Session
1104 show_all_bad:
1105 show all bad entries? (If false, return upon the first)
1107 Returns:
1108 bool: are the indexes OK?
1109 """
1110 p_ok = PatientIdNumIndexEntry.check_index(session, show_all_bad)
1111 if p_ok:
1112 log.info("Patient ID number index is good")
1113 else:
1114 log.error("Patient ID number index is bad")
1115 if not show_all_bad:
1116 return False
1117 t_ok = TaskIndexEntry.check_index(session, show_all_bad)
1118 if t_ok:
1119 log.info("Task index is good")
1120 else:
1121 log.error("Task index is bad")
1122 return p_ok and t_ok