Coverage for cc_modules/cc_taskindex.py: 34%

290 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-08 23:14 +0000

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_taskindex.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry. 

9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**Server-side task index.** 

29 

30Note in particular that if you, as a developer, change the ``is_complete()`` 

31criteria for a task, you should cause the server index to be rebuilt (because 

32it caches ``is_complete()`` information). 

33 

34""" 

35 

36import logging 

37from typing import List, Optional, Type, TYPE_CHECKING 

38 

39from cardinal_pythonlib.logs import BraceStyleAdapter 

40from cardinal_pythonlib.reprfunc import simple_repr 

41from cardinal_pythonlib.sqlalchemy.session import get_engine_from_session 

42from cardinal_pythonlib.sqlalchemy.schema import table_exists 

43from cardinal_pythonlib.sqlalchemy.sqlserver import ( 

44 if_sqlserver_disable_constraints_triggers, 

45) 

46from pendulum import DateTime as Pendulum 

47import pyramid.httpexceptions as exc 

48from sqlalchemy.orm import relationship, Session as SqlASession 

49from sqlalchemy.sql.expression import and_, exists, join, literal, select 

50from sqlalchemy.sql.schema import Column, ForeignKey, Table 

51from sqlalchemy.sql.sqltypes import BigInteger, Boolean, DateTime, Integer 

52 

53from camcops_server.cc_modules.cc_client_api_core import ( 

54 BatchDetails, 

55 fail_user_error, 

56 UploadTableChanges, 

57) 

58from camcops_server.cc_modules.cc_constants import ERA_NOW 

59from camcops_server.cc_modules.cc_idnumdef import IdNumDefinition 

60from camcops_server.cc_modules.cc_patient import Patient 

61from camcops_server.cc_modules.cc_patientidnum import PatientIdNum 

62from camcops_server.cc_modules.cc_sqla_coltypes import ( 

63 EraColType, 

64 isotzdatetime_to_utcdatetime, 

65 PendulumDateTimeAsIsoTextColType, 

66 TableNameColType, 

67) 

68from camcops_server.cc_modules.cc_sqlalchemy import Base 

69from camcops_server.cc_modules.cc_task import ( 

70 all_task_tablenames, 

71 tablename_to_task_class_dict, 

72 Task, 

73) 

74from camcops_server.cc_modules.cc_user import User 

75 

76if TYPE_CHECKING: 

77 from camcops_server.cc_modules.cc_request import CamcopsRequest 

78 

79log = BraceStyleAdapter(logging.getLogger(__name__)) 

80 

81 

82# ============================================================================= 

83# Helper functions 

84# ============================================================================= 

85 

86 

87def task_factory_unfiltered( 

88 dbsession: SqlASession, basetable: str, serverpk: int 

89) -> Optional[Task]: 

90 """ 

91 Load a task from the database and return it. 

92 No permission filtering is performed. (Used by 

93 :class:`camcops_server.cc_modules.cc_taskindex.TaskIndexEntry`.) 

94 

95 Args: 

96 dbsession: a :class:`sqlalchemy.orm.session.Session` 

97 basetable: name of the task's base table 

98 serverpk: server PK of the task 

99 

100 Returns: 

101 the task, or ``None`` if the PK doesn't exist 

102 

103 Raises: 

104 :exc:`HTTPBadRequest` if the table doesn't exist 

105 """ 

106 d = tablename_to_task_class_dict() 

107 try: 

108 cls = d[basetable] # may raise KeyError 

109 except KeyError: 

110 raise exc.HTTPBadRequest(f"No such task table: {basetable!r}") 

111 # noinspection PyProtectedMember 

112 q = dbsession.query(cls).filter(cls._pk == serverpk) 

113 return q.first() 

114 

115 

116# ============================================================================= 

117# PatientIdNumIndexEntry 

118# ============================================================================= 

119 

120 

121class PatientIdNumIndexEntry(Base): 

122 """ 

123 Represents a server index entry for a 

124 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`. 

125 

126 - Only current ID numbers are indexed. 

127 """ 

128 

129 __tablename__ = "_idnum_index" 

130 

131 idnum_pk = Column( 

132 "idnum_pk", 

133 Integer, 

134 primary_key=True, 

135 index=True, 

136 comment="Server primary key of the PatientIdNum " 

137 "(and of the PatientIdNumIndexEntry)", 

138 ) 

139 indexed_at_utc = Column( 

140 "indexed_at_utc", 

141 DateTime, 

142 nullable=False, 

143 comment="When this index entry was created", 

144 ) 

145 

146 # noinspection PyProtectedMember 

147 patient_pk = Column( 

148 "patient_pk", 

149 Integer, 

150 ForeignKey(Patient._pk), 

151 index=True, 

152 comment="Server primary key of the Patient", 

153 ) 

154 which_idnum = Column( 

155 "which_idnum", 

156 Integer, 

157 ForeignKey(IdNumDefinition.which_idnum), 

158 nullable=False, 

159 index=True, 

160 comment="Which of the server's ID numbers is this?", 

161 ) 

162 idnum_value = Column( 

163 "idnum_value", BigInteger, comment="The value of the ID number" 

164 ) 

165 

166 # Relationships: 

167 patient = relationship(Patient) 

168 

169 def __repr__(self) -> str: 

170 return simple_repr( 

171 self, ["idnum_pk", "patient_pk", "which_idnum", "idnum_value"] 

172 ) 

173 

174 # ------------------------------------------------------------------------- 

175 # Create 

176 # ------------------------------------------------------------------------- 

177 

178 @classmethod 

179 def make_from_idnum(cls, idnum: PatientIdNum) -> "PatientIdNumIndexEntry": 

180 """ 

181 Returns an ID index entry for the specified 

182 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`. The 

183 returned index requires inserting into a database session. 

184 """ 

185 # noinspection PyProtectedMember 

186 assert idnum._current, "Only index current PatientIdNum objects" 

187 index = cls() 

188 index.idnum_pk = idnum.pk 

189 index.patient_pk = idnum.get_patient_server_pk() 

190 index.which_idnum = idnum.which_idnum 

191 index.idnum_value = idnum.idnum_value 

192 index.indexed_at_utc = Pendulum.now() 

193 return index 

194 

195 @classmethod 

196 def index_idnum(cls, idnum: PatientIdNum, session: SqlASession) -> None: 

197 """ 

198 Indexes an ID number and inserts the index into the database. 

199 

200 Args: 

201 idnum: a 

202 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum` 

203 session: 

204 an SQLAlchemy Session 

205 """ 

206 index = cls.make_from_idnum(idnum) 

207 session.add(index) 

208 

209 @classmethod 

210 def unindex_patient(cls, patient: Patient, session: SqlASession) -> None: 

211 """ 

212 Removes all ID number indexes from the database for a patient. 

213 

214 Args: 

215 patient: 

216 :class:`camcops_server.cc_modules.cc_patient.Patient` 

217 session: 

218 an SQLAlchemy Session 

219 """ 

220 

221 # noinspection PyUnresolvedReferences 

222 idxtable = cls.__table__ # type: Table 

223 idxcols = idxtable.columns 

224 # noinspection PyProtectedMember 

225 session.execute( 

226 idxtable.delete().where(idxcols.patient_pk == patient._pk) 

227 ) 

228 

229 # ------------------------------------------------------------------------- 

230 # Regenerate index 

231 # ------------------------------------------------------------------------- 

232 

233 @classmethod 

234 def rebuild_idnum_index( 

235 cls, session: SqlASession, indexed_at_utc: Pendulum 

236 ) -> None: 

237 """ 

238 Rebuilds the index entirely. Uses SQLAlchemy Core (not ORM) for speed. 

239 

240 Args: 

241 session: an SQLAlchemy Session 

242 indexed_at_utc: current time in UTC 

243 """ 

244 log.info("Rebuilding patient ID number index") 

245 # noinspection PyUnresolvedReferences 

246 indextable = PatientIdNumIndexEntry.__table__ # type: Table 

247 indexcols = indextable.columns 

248 # noinspection PyUnresolvedReferences 

249 idnumtable = PatientIdNum.__table__ # type: Table 

250 idnumcols = idnumtable.columns 

251 # noinspection PyUnresolvedReferences 

252 patienttable = Patient.__table__ # type: Table 

253 patientcols = patienttable.columns 

254 

255 # Delete all entries 

256 with if_sqlserver_disable_constraints_triggers( 

257 session, indextable.name 

258 ): 

259 session.execute(indextable.delete()) 

260 

261 # Create new ones 

262 # noinspection PyProtectedMember,PyPep8 

263 session.execute( 

264 indextable.insert().from_select( 

265 # Target: 

266 [ 

267 indexcols.idnum_pk, 

268 indexcols.indexed_at_utc, 

269 indexcols.patient_pk, 

270 indexcols.which_idnum, 

271 indexcols.idnum_value, 

272 ], 

273 # Source: 

274 ( 

275 select( 

276 [ 

277 idnumcols._pk, 

278 literal(indexed_at_utc), 

279 patientcols._pk, 

280 idnumcols.which_idnum, 

281 idnumcols.idnum_value, 

282 ] 

283 ) 

284 .select_from( 

285 join( 

286 idnumtable, 

287 patienttable, 

288 and_( 

289 idnumcols._device_id == patientcols._device_id, 

290 idnumcols._era == patientcols._era, 

291 idnumcols.patient_id == patientcols.id, 

292 ), 

293 ) 

294 ) 

295 .where(idnumcols._current == True) # noqa: E712 

296 .where(idnumcols.idnum_value.isnot(None)) 

297 .where(patientcols._current == True) # noqa: E712 

298 ), 

299 ) 

300 ) 

301 

302 # ------------------------------------------------------------------------- 

303 # Check index 

304 # ------------------------------------------------------------------------- 

305 @classmethod 

306 def check_index( 

307 cls, session: SqlASession, show_all_bad: bool = False 

308 ) -> bool: 

309 """ 

310 Checks the index. 

311 

312 Args: 

313 session: 

314 an SQLAlchemy Session 

315 show_all_bad: 

316 show all bad entries? (If false, return upon the first) 

317 

318 Returns: 

319 bool: is the index OK? 

320 """ 

321 ok = True 

322 

323 log.info( 

324 "Checking all patient ID number indexes represent valid entries" 

325 ) 

326 # noinspection PyUnresolvedReferences,PyProtectedMember 

327 q_idx_without_original = session.query(PatientIdNumIndexEntry).filter( 

328 ~exists() 

329 .select_from( 

330 PatientIdNum.__table__.join( 

331 Patient.__table__, 

332 Patient.id == PatientIdNum.patient_id, 

333 Patient._device_id == PatientIdNum._device_id, 

334 Patient._era == PatientIdNum._era, 

335 ) 

336 ) 

337 .where( 

338 and_( 

339 PatientIdNum._pk == PatientIdNumIndexEntry.idnum_pk, 

340 PatientIdNum._current == True, # noqa: E712 

341 PatientIdNum.which_idnum 

342 == PatientIdNumIndexEntry.which_idnum, 

343 PatientIdNum.idnum_value 

344 == PatientIdNumIndexEntry.idnum_value, 

345 Patient._pk == PatientIdNumIndexEntry.patient_pk, 

346 Patient._current == True, # noqa: E712 

347 ) 

348 ) 

349 ) 

350 for index in q_idx_without_original: 

351 log.error( 

352 "Patient ID number index without matching " "original: {!r}", 

353 index, 

354 ) 

355 ok = False 

356 if not show_all_bad: 

357 return ok 

358 

359 log.info("Checking all patient ID numbers have an index") 

360 # noinspection PyUnresolvedReferences,PyProtectedMember 

361 q_original_with_idx = session.query(PatientIdNum).filter( 

362 PatientIdNum._current == True, # noqa: E712 

363 PatientIdNum.idnum_value.isnot(None), 

364 ~exists() 

365 .select_from(PatientIdNumIndexEntry.__table__) 

366 .where( 

367 and_( 

368 PatientIdNum._pk == PatientIdNumIndexEntry.idnum_pk, 

369 PatientIdNum.which_idnum 

370 == PatientIdNumIndexEntry.which_idnum, # noqa 

371 PatientIdNum.idnum_value 

372 == PatientIdNumIndexEntry.idnum_value, # noqa 

373 ) 

374 ), 

375 ) 

376 for orig in q_original_with_idx: 

377 log.error("ID number without index entry: {!r}", orig) 

378 ok = False 

379 if not show_all_bad: 

380 return ok 

381 

382 return ok 

383 

384 # ------------------------------------------------------------------------- 

385 # Update index at the point of upload from a device 

386 # ------------------------------------------------------------------------- 

387 

388 @classmethod 

389 def update_idnum_index_for_upload( 

390 cls, 

391 session: SqlASession, 

392 indexed_at_utc: Pendulum, 

393 tablechanges: UploadTableChanges, 

394 ) -> None: 

395 """ 

396 Updates the index for a device's upload. 

397 

398 - Deletes index entries for records that are on the way out. 

399 - Creates index entries for records that are on the way in. 

400 - Should be called after both the Patient and PatientIdNum tables are 

401 committed; see special ordering in 

402 :func:`camcops_server.cc_modules.client_api.commit_all`. 

403 

404 Args: 

405 session: 

406 an SQLAlchemy Session 

407 indexed_at_utc: 

408 current time in UTC 

409 tablechanges: 

410 a :class:`camcops_server.cc_modules.cc_client_api_core.UploadTableChanges` 

411 object describing the changes to a table 

412 """ # noqa 

413 # noinspection PyUnresolvedReferences 

414 indextable = PatientIdNumIndexEntry.__table__ # type: Table 

415 indexcols = indextable.columns 

416 # noinspection PyUnresolvedReferences 

417 idnumtable = PatientIdNum.__table__ # type: Table 

418 idnumcols = idnumtable.columns 

419 # noinspection PyUnresolvedReferences 

420 patienttable = Patient.__table__ # type: Table 

421 patientcols = patienttable.columns 

422 

423 # Delete the old 

424 removal_pks = tablechanges.idnum_delete_index_pks 

425 if removal_pks: 

426 log.debug( 

427 "Deleting old ID number indexes: server PKs {}", removal_pks 

428 ) 

429 session.execute( 

430 indextable.delete().where( 

431 indextable.c.idnum_pk.in_(removal_pks) 

432 ) 

433 ) 

434 

435 # Create the new 

436 addition_pks = tablechanges.idnum_add_index_pks 

437 if addition_pks: 

438 log.debug("Adding ID number indexes: server PKs {}", addition_pks) 

439 # noinspection PyPep8,PyProtectedMember 

440 session.execute( 

441 indextable.insert().from_select( 

442 # Target: 

443 [ 

444 indexcols.idnum_pk, 

445 indexcols.indexed_at_utc, 

446 indexcols.patient_pk, 

447 indexcols.which_idnum, 

448 indexcols.idnum_value, 

449 ], 

450 # Source: 

451 ( 

452 select( 

453 [ 

454 idnumcols._pk, 

455 literal(indexed_at_utc), 

456 patientcols._pk, 

457 idnumcols.which_idnum, 

458 idnumcols.idnum_value, 

459 ] 

460 ) 

461 .select_from( 

462 join( 

463 idnumtable, 

464 patienttable, 

465 and_( 

466 idnumcols._device_id 

467 == patientcols._device_id, # noqa 

468 idnumcols._era == patientcols._era, 

469 idnumcols.patient_id == patientcols.id, 

470 ), 

471 ) 

472 ) 

473 .where(idnumcols._pk.in_(addition_pks)) 

474 .where(patientcols._current == True) # noqa: E712 

475 ), 

476 ) 

477 ) 

478 

479 

480# ============================================================================= 

481# TaskIndexEntry 

482# ============================================================================= 

483 

484 

485class TaskIndexEntry(Base): 

486 """ 

487 Represents a server index entry for a 

488 :class:`camcops_server.cc_modules.cc_task.Task`. 

489 

490 - Only current tasks are indexed. This simplifies direct linking to patient 

491 PKs. 

492 """ 

493 

494 __tablename__ = "_task_index" 

495 

496 index_entry_pk = Column( 

497 "index_entry_pk", 

498 Integer, 

499 primary_key=True, 

500 autoincrement=True, 

501 comment="Arbitrary primary key of this index entry", 

502 ) 

503 indexed_at_utc = Column( 

504 "indexed_at_utc", 

505 DateTime, 

506 nullable=False, 

507 comment="When this index entry was created", 

508 ) 

509 

510 # The next two fields link to our task: 

511 task_table_name = Column( 

512 "task_table_name", 

513 TableNameColType, 

514 index=True, 

515 comment="Table name of the task's base table", 

516 ) 

517 task_pk = Column( 

518 "task_pk", 

519 Integer, 

520 index=True, 

521 comment="Server primary key of the task", 

522 ) 

523 # We can probably even represent this with an SQLAlchemy ORM relationship. 

524 # This is polymorphic loading (we'll return objects of different types) 

525 # based on concrete table inheritance (each type of object -- each task -- 

526 # has its own standalone table). 

527 # However, there are warnings about the inefficiency of this; see 

528 # https://docs.sqlalchemy.org/en/latest/orm/inheritance.html#concrete-table-inheritance 

529 # and we are trying to be efficient. So let's do via task() below. 

530 

531 # This links to the task's patient, if there is one: 

532 # noinspection PyProtectedMember 

533 patient_pk = Column( 

534 "patient_pk", 

535 Integer, 

536 ForeignKey(Patient._pk), 

537 index=True, 

538 comment="Server primary key of the patient (if applicable)", 

539 ) 

540 

541 # These fields allow us to filter tasks efficiently: 

542 device_id = Column( 

543 "device_id", 

544 Integer, 

545 ForeignKey("_security_devices.id"), 

546 nullable=False, 

547 index=True, 

548 comment="ID of the source tablet device", 

549 ) 

550 era = Column( 

551 "era", 

552 EraColType, 

553 nullable=False, 

554 index=True, 

555 comment="Era (_era) field of the source record", 

556 ) 

557 when_created_utc = Column( 

558 "when_created_utc", 

559 DateTime, 

560 nullable=False, 

561 index=True, 

562 comment="Date/time this task instance was created (UTC)", 

563 ) 

564 when_created_iso = Column( 

565 "when_created_iso", 

566 PendulumDateTimeAsIsoTextColType, 

567 nullable=False, 

568 index=True, 

569 comment="Date/time this task instance was created (ISO 8601)", 

570 ) # Pendulum on the Python side 

571 when_added_batch_utc = Column( 

572 "when_added_batch_utc", 

573 DateTime, 

574 nullable=False, 

575 index=True, 

576 comment="Date/time this task index was uploaded (UTC)", 

577 ) 

578 adding_user_id = Column( 

579 "adding_user_id", 

580 Integer, 

581 ForeignKey("_security_users.id"), 

582 comment="ID of user that added this task", 

583 ) 

584 group_id = Column( 

585 "group_id", 

586 Integer, 

587 ForeignKey("_security_groups.id"), 

588 nullable=False, 

589 index=True, 

590 comment="ID of group to which this task belongs", 

591 ) 

592 task_is_complete = Column( 

593 "task_is_complete", 

594 Boolean, 

595 nullable=False, 

596 comment="Is the task complete (as judged by the server when the index " 

597 "entry was created)?", 

598 ) 

599 

600 # Relationships: 

601 patient = relationship(Patient) 

602 _adding_user = relationship(User) 

603 

604 def __repr__(self) -> str: 

605 return simple_repr( 

606 self, 

607 [ 

608 "index_entry_pk", 

609 "task_table_name", 

610 "task_pk", 

611 "patient_pk", 

612 "device_id", 

613 "era", 

614 "when_created_utc", 

615 "when_created_iso", 

616 "when_added_batch_utc", 

617 "adding_user_id", 

618 "group_id", 

619 "task_is_complete", 

620 ], 

621 ) 

622 

623 # ------------------------------------------------------------------------- 

624 # Fetch the task 

625 # ------------------------------------------------------------------------- 

626 

627 @property 

628 def task(self) -> Optional[Task]: 

629 """ 

630 Returns: 

631 the associated :class:`camcops_server.cc_modules.cc_task.Task`, or 

632 ``None`` if none exists. 

633 

634 Raises: 

635 :exc:`HTTPBadRequest` if the table doesn't exist 

636 """ 

637 dbsession = SqlASession.object_session(self) 

638 assert dbsession, ( 

639 "TaskIndexEntry.task called on a TaskIndexEntry " 

640 "that's not yet in a database session" 

641 ) 

642 return task_factory_unfiltered( 

643 dbsession, self.task_table_name, self.task_pk 

644 ) 

645 

646 # ------------------------------------------------------------------------- 

647 # Other properties mirroring those of Task, for duck typing 

648 # ------------------------------------------------------------------------- 

649 

650 @property 

651 def is_anonymous(self) -> bool: 

652 """ 

653 Is the task anonymous? 

654 """ 

655 return self.patient_pk is None 

656 

657 def is_complete(self) -> bool: 

658 """ 

659 Is the task complete? 

660 """ 

661 return self.task_is_complete 

662 

663 @property 

664 def _current(self) -> bool: 

665 """ 

666 All task index entries represent complete tasks, so this always returns 

667 ``True``. 

668 """ 

669 return True 

670 

671 @property 

672 def pk(self) -> int: 

673 """ 

674 Return's the task's server PK. 

675 """ 

676 return self.task_pk 

677 

678 @property 

679 def tablename(self) -> str: 

680 """ 

681 Returns the base table name of the task. 

682 """ 

683 return self.task_table_name 

684 

685 @property 

686 def shortname(self) -> str: 

687 """ 

688 Returns the task's shortname. 

689 """ 

690 d = tablename_to_task_class_dict() 

691 taskclass = d[self.task_table_name] 

692 return taskclass.shortname 

693 

694 def is_live_on_tablet(self) -> bool: 

695 """ 

696 Is the task live on the source device (e.g. tablet)? 

697 """ 

698 return self.era == ERA_NOW 

699 

700 @property 

701 def when_created(self) -> Pendulum: 

702 """ 

703 Returns the creation date/time as a Pendulum DateTime object. 

704 """ 

705 return self.when_created_iso 

706 

707 def any_patient_idnums_invalid(self, req: "CamcopsRequest") -> bool: 

708 """ 

709 Do we have a patient who has any invalid ID numbers? 

710 

711 Args: 

712 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

713 """ 

714 idnums = self.get_patient_idnum_objects() 

715 for idnum in idnums: 

716 if not idnum.is_fully_valid(req): 

717 return True 

718 return False 

719 

720 def get_patient_idnum_objects(self) -> List[PatientIdNum]: 

721 """ 

722 Gets all :class:`PatientIdNum` objects for the patient. 

723 """ 

724 return self.patient.get_idnum_objects() if self.patient else [] 

725 

726 # ------------------------------------------------------------------------- 

727 # Create 

728 # ------------------------------------------------------------------------- 

729 

730 @classmethod 

731 def make_from_task( 

732 cls, task: Task, indexed_at_utc: Pendulum 

733 ) -> "TaskIndexEntry": 

734 """ 

735 Returns a task index entry for the specified 

736 :class:`camcops_server.cc_modules.cc_task.Task`. The 

737 returned index requires inserting into a database session. 

738 

739 Args: 

740 task: 

741 a :class:`camcops_server.cc_modules.cc_task.Task` 

742 indexed_at_utc: 

743 current time in UTC 

744 """ 

745 assert indexed_at_utc is not None, "Missing indexed_at_utc" 

746 

747 index = cls() 

748 

749 index.indexed_at_utc = indexed_at_utc 

750 

751 index.task_table_name = task.tablename 

752 index.task_pk = task.pk 

753 

754 patient = task.patient 

755 index.patient_pk = patient.pk if patient else None 

756 

757 index.device_id = task.device_id 

758 index.era = task.era 

759 index.when_created_utc = task.get_creation_datetime_utc() 

760 index.when_created_iso = task.when_created 

761 # noinspection PyProtectedMember 

762 index.when_added_batch_utc = task._when_added_batch_utc 

763 index.adding_user_id = task.get_adding_user_id() 

764 index.group_id = task.group_id 

765 index.task_is_complete = task.is_complete() 

766 

767 return index 

768 

769 @classmethod 

770 def index_task( 

771 cls, task: Task, session: SqlASession, indexed_at_utc: Pendulum 

772 ) -> None: 

773 """ 

774 Indexes a task and inserts the index into the database. 

775 

776 Args: 

777 task: 

778 a :class:`camcops_server.cc_modules.cc_task.Task` 

779 session: 

780 an SQLAlchemy Session 

781 indexed_at_utc: 

782 current time in UTC 

783 """ 

784 index = cls.make_from_task(task, indexed_at_utc=indexed_at_utc) 

785 session.add(index) 

786 

787 @classmethod 

788 def unindex_task(cls, task: Task, session: SqlASession) -> None: 

789 """ 

790 Removes a task index from the database. 

791 

792 Args: 

793 task: 

794 a :class:`camcops_server.cc_modules.cc_task.Task` 

795 session: 

796 an SQLAlchemy Session 

797 """ 

798 

799 # noinspection PyUnresolvedReferences 

800 idxtable = cls.__table__ # type: Table 

801 idxcols = idxtable.columns 

802 tasktablename = task.__class__.tablename 

803 session.execute( 

804 idxtable.delete() 

805 .where(idxcols.task_table_name == tasktablename) 

806 .where(idxcols.task_pk == task.pk) 

807 ) 

808 

809 # ------------------------------------------------------------------------- 

810 # Regenerate index 

811 # ------------------------------------------------------------------------- 

812 

813 @classmethod 

814 def rebuild_index_for_task_type( 

815 cls, 

816 session: SqlASession, 

817 taskclass: Type[Task], 

818 indexed_at_utc: Pendulum, 

819 delete_first: bool = True, 

820 ) -> None: 

821 """ 

822 Rebuilds the index for a particular task type. 

823 

824 Args: 

825 session: an SQLAlchemy Session 

826 taskclass: a subclass of 

827 :class:`camcops_server.cc_modules.cc_task.Task` 

828 indexed_at_utc: current time in UTC 

829 delete_first: delete old index entries first? Should always be True 

830 unless called as part of a master rebuild that deletes 

831 everything first. 

832 """ 

833 # noinspection PyUnresolvedReferences 

834 idxtable = cls.__table__ # type: Table 

835 idxcols = idxtable.columns 

836 tasktablename = taskclass.tablename 

837 log.info("Rebuilding task index for {}", tasktablename) 

838 # Delete all entries for this task 

839 if delete_first: 

840 session.execute( 

841 idxtable.delete().where(idxcols.table_name == tasktablename) 

842 ) 

843 # Create new entries 

844 # noinspection PyPep8,PyUnresolvedReferences,PyProtectedMember 

845 q = ( 

846 session.query(taskclass) 

847 .filter(taskclass._current == True) # noqa: E712 

848 .order_by(isotzdatetime_to_utcdatetime(taskclass.when_created)) 

849 ) 

850 for task in q: 

851 cls.index_task(task, session, indexed_at_utc) 

852 

853 @classmethod 

854 def rebuild_entire_task_index( 

855 cls, 

856 session: SqlASession, 

857 indexed_at_utc: Pendulum, 

858 skip_tasks_with_missing_tables: bool = False, 

859 ) -> None: 

860 """ 

861 Rebuilds the entire index. 

862 

863 Args: 

864 session: an SQLAlchemy Session 

865 indexed_at_utc: current time in UTC 

866 skip_tasks_with_missing_tables: should we skip over tasks if their 

867 tables are not in the database? (This is so we can rebuild an 

868 index from a database upgrade, but not crash because newer 

869 tasks haven't had their tables created yet.) 

870 """ 

871 log.info("Rebuilding entire task index") 

872 # noinspection PyUnresolvedReferences 

873 idxtable = cls.__table__ # type: Table 

874 

875 # Delete all entries 

876 with if_sqlserver_disable_constraints_triggers(session, idxtable.name): 

877 session.execute(idxtable.delete()) 

878 

879 # Now rebuild: 

880 for taskclass in Task.all_subclasses_by_tablename(): 

881 if skip_tasks_with_missing_tables: 

882 basetable = taskclass.tablename 

883 engine = get_engine_from_session(session) 

884 if not table_exists(engine, basetable): 

885 continue 

886 cls.rebuild_index_for_task_type( 

887 session, taskclass, indexed_at_utc, delete_first=False 

888 ) 

889 

890 # ------------------------------------------------------------------------- 

891 # Update index at the point of upload from a device 

892 # ------------------------------------------------------------------------- 

893 

894 @classmethod 

895 def update_task_index_for_upload( 

896 cls, 

897 session: SqlASession, 

898 tablechanges: UploadTableChanges, 

899 indexed_at_utc: Pendulum, 

900 ) -> None: 

901 """ 

902 Updates the index for a device's upload. 

903 

904 - Deletes index entries for records that are on the way out. 

905 - Creates index entries for records that are on the way in. 

906 - Deletes/recreates index entries for records being preserved. 

907 

908 Args: 

909 session: 

910 an SQLAlchemy Session 

911 tablechanges: 

912 a :class:`camcops_server.cc_modules.cc_client_api_core.UploadTableChanges` 

913 object describing the changes to a table 

914 indexed_at_utc: 

915 current time in UTC 

916 """ # noqa 

917 tasktablename = tablechanges.tablename 

918 d = tablename_to_task_class_dict() 

919 try: 

920 taskclass = d[tasktablename] # may raise KeyError 

921 except KeyError: 

922 fail_user_error(f"Bug: no such task table: {tasktablename!r}") 

923 

924 # noinspection PyUnresolvedReferences 

925 idxtable = cls.__table__ # type: Table 

926 idxcols = idxtable.columns 

927 

928 # Delete the old. 

929 delete_index_pks = tablechanges.task_delete_index_pks 

930 if delete_index_pks: 

931 log.debug( 

932 "Deleting old task indexes: {}, server PKs {}", 

933 tasktablename, 

934 delete_index_pks, 

935 ) 

936 # noinspection PyProtectedMember 

937 session.execute( 

938 idxtable.delete() 

939 .where(idxcols.task_table_name == tasktablename) 

940 .where(idxcols.task_pk.in_(delete_index_pks)) 

941 ) 

942 

943 # Create the new. 

944 reindex_pks = tablechanges.task_reindex_pks 

945 if reindex_pks: 

946 log.debug( 

947 "Recreating task indexes: {}, server PKs {}", 

948 tasktablename, 

949 reindex_pks, 

950 ) 

951 # noinspection PyUnboundLocalVariable,PyProtectedMember 

952 q = session.query(taskclass).filter(taskclass._pk.in_(reindex_pks)) 

953 for task in q: 

954 cls.index_task(task, session, indexed_at_utc=indexed_at_utc) 

955 

956 # ------------------------------------------------------------------------- 

957 # Check index 

958 # ------------------------------------------------------------------------- 

959 @classmethod 

960 def check_index( 

961 cls, session: SqlASession, show_all_bad: bool = False 

962 ) -> bool: 

963 """ 

964 Checks the index. 

965 

966 Args: 

967 session: 

968 an SQLAlchemy Session 

969 show_all_bad: 

970 show all bad entries? (If false, return upon the first) 

971 

972 Returns: 

973 bool: is the index OK? 

974 """ 

975 ok = True 

976 

977 log.info("Checking all task indexes represent valid entries") 

978 for taskclass in Task.all_subclasses_by_tablename(): 

979 tasktablename = taskclass.tablename 

980 log.debug("Checking {}", tasktablename) 

981 # noinspection PyUnresolvedReferences,PyProtectedMember 

982 q_idx_without_original = session.query(TaskIndexEntry).filter( 

983 TaskIndexEntry.task_table_name == tasktablename, 

984 ~exists() 

985 .select_from(taskclass.__table__) 

986 .where( 

987 and_( 

988 TaskIndexEntry.task_pk == taskclass._pk, 

989 taskclass._current == True, # noqa: E712 

990 ) 

991 ), 

992 ) 

993 # No check for a valid patient at this time. 

994 for index in q_idx_without_original: 

995 log.error("Task index without matching original: {!r}", index) 

996 ok = False 

997 if not show_all_bad: 

998 return ok 

999 

1000 log.info("Checking all tasks have an index") 

1001 for taskclass in Task.all_subclasses_by_tablename(): 

1002 tasktablename = taskclass.tablename 

1003 log.debug("Checking {}", tasktablename) 

1004 # noinspection PyUnresolvedReferences,PyProtectedMember 

1005 q_original_with_idx = session.query(taskclass).filter( 

1006 taskclass._current == True, # noqa: E712 

1007 ~exists() 

1008 .select_from(TaskIndexEntry.__table__) 

1009 .where( 

1010 and_( 

1011 TaskIndexEntry.task_pk == taskclass._pk, 

1012 TaskIndexEntry.task_table_name == tasktablename, 

1013 ) 

1014 ), 

1015 ) 

1016 for orig in q_original_with_idx: 

1017 log.error("Task without index entry: {!r}", orig) 

1018 ok = False 

1019 if not show_all_bad: 

1020 return ok 

1021 

1022 return ok 

1023 

1024 

1025# ============================================================================= 

1026# Wide-ranging index update functions 

1027# ============================================================================= 

1028 

1029 

1030def reindex_everything( 

1031 session: SqlASession, skip_tasks_with_missing_tables: bool = False 

1032) -> None: 

1033 """ 

1034 Deletes from and rebuilds all server index tables. 

1035 

1036 Args: 

1037 session: an SQLAlchemy Session 

1038 skip_tasks_with_missing_tables: should we skip over tasks if their 

1039 tables are not in the database? (This is so we can rebuild an index 

1040 from a database upgrade, but not crash because newer tasks haven't 

1041 had their tables created yet.) 

1042 """ 

1043 now = Pendulum.utcnow() 

1044 log.info("Reindexing database; indexed_at_utc = {}", now) 

1045 PatientIdNumIndexEntry.rebuild_idnum_index(session, now) 

1046 TaskIndexEntry.rebuild_entire_task_index( 

1047 session, 

1048 now, 

1049 skip_tasks_with_missing_tables=skip_tasks_with_missing_tables, 

1050 ) 

1051 

1052 

1053def update_indexes_and_push_exports( 

1054 req: "CamcopsRequest", 

1055 batchdetails: BatchDetails, 

1056 tablechanges: UploadTableChanges, 

1057) -> None: 

1058 """ 

1059 Update server indexes, if required. 

1060 

1061 Also triggers background jobs to export "new arrivals", if required. 

1062 

1063 Args: 

1064 req: the :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

1065 batchdetails: the :class:`BatchDetails` 

1066 tablechanges: 

1067 a :class:`camcops_server.cc_modules.cc_client_api_core.UploadTableChanges` 

1068 object describing the changes to a table 

1069 """ # noqa 

1070 tablename = tablechanges.tablename 

1071 if tablename == PatientIdNum.__tablename__: 

1072 # Update idnum index 

1073 PatientIdNumIndexEntry.update_idnum_index_for_upload( 

1074 session=req.dbsession, 

1075 indexed_at_utc=batchdetails.batchtime, 

1076 tablechanges=tablechanges, 

1077 ) 

1078 elif tablename in all_task_tablenames(): 

1079 # Update task index 

1080 TaskIndexEntry.update_task_index_for_upload( 

1081 session=req.dbsession, 

1082 tablechanges=tablechanges, 

1083 indexed_at_utc=batchdetails.batchtime, 

1084 ) 

1085 # Push exports 

1086 recipients = req.all_push_recipients 

1087 uploading_group_id = req.user.upload_group_id 

1088 for recipient in recipients: 

1089 recipient_name = recipient.recipient_name 

1090 for pk in tablechanges.get_task_push_export_pks( 

1091 recipient=recipient, uploading_group_id=uploading_group_id 

1092 ): 

1093 req.add_export_push_request(recipient_name, tablename, pk) 

1094 # ... will be transmitted *after* the request performs COMMIT 

1095 

1096 

1097def check_indexes(session: SqlASession, show_all_bad: bool = False) -> bool: 

1098 """ 

1099 Checks all server index tables. 

1100 

1101 Args: 

1102 session: 

1103 an SQLAlchemy Session 

1104 show_all_bad: 

1105 show all bad entries? (If false, return upon the first) 

1106 

1107 Returns: 

1108 bool: are the indexes OK? 

1109 """ 

1110 p_ok = PatientIdNumIndexEntry.check_index(session, show_all_bad) 

1111 if p_ok: 

1112 log.info("Patient ID number index is good") 

1113 else: 

1114 log.error("Patient ID number index is bad") 

1115 if not show_all_bad: 

1116 return False 

1117 t_ok = TaskIndexEntry.check_index(session, show_all_bad) 

1118 if t_ok: 

1119 log.info("Task index is good") 

1120 else: 

1121 log.error("Task index is bad") 

1122 return p_ok and t_ok