Coverage for cc_modules/cc_taskfilter.py: 39%

212 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-08 23:14 +0000

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_taskfilter.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry. 

9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**Representation of filtering criteria for tasks.** 

29 

30""" 

31 

32import datetime 

33from enum import Enum 

34import logging 

35from typing import Dict, List, Optional, Type, TYPE_CHECKING, Union 

36 

37from cardinal_pythonlib.datetimefunc import convert_datetime_to_utc 

38from cardinal_pythonlib.json.serialize import register_class_for_json 

39from cardinal_pythonlib.logs import BraceStyleAdapter 

40from cardinal_pythonlib.reprfunc import auto_repr 

41from cardinal_pythonlib.sqlalchemy.list_types import ( 

42 IntListType, 

43 StringListType, 

44) 

45from pendulum import DateTime as Pendulum 

46from sqlalchemy.orm import Query, reconstructor 

47from sqlalchemy.sql.functions import func 

48from sqlalchemy.sql.expression import and_, or_ 

49from sqlalchemy.sql.schema import Column 

50from sqlalchemy.sql.sqltypes import Boolean, Date, Integer 

51 

52from camcops_server.cc_modules.cc_cache import cache_region_static, fkg 

53from camcops_server.cc_modules.cc_device import Device 

54from camcops_server.cc_modules.cc_group import Group 

55from camcops_server.cc_modules.cc_patient import Patient 

56from camcops_server.cc_modules.cc_patientidnum import PatientIdNum 

57from camcops_server.cc_modules.cc_sqla_coltypes import ( 

58 PendulumDateTimeAsIsoTextColType, 

59 IdNumReferenceListColType, 

60 PatientNameColType, 

61 SexColType, 

62) 

63from camcops_server.cc_modules.cc_sqlalchemy import Base 

64from camcops_server.cc_modules.cc_task import ( 

65 tablename_to_task_class_dict, 

66 Task, 

67) 

68from camcops_server.cc_modules.cc_taskindex import PatientIdNumIndexEntry 

69from camcops_server.cc_modules.cc_user import User 

70 

71if TYPE_CHECKING: 

72 from sqlalchemy.sql.elements import ColumnElement 

73 from camcops_server.cc_modules.cc_request import CamcopsRequest 

74 from camcops_server.cc_modules.cc_simpleobjects import IdNumReference 

75 

76log = BraceStyleAdapter(logging.getLogger(__name__)) 

77 

78 

79# ============================================================================= 

80# Sorting helpers 

81# ============================================================================= 

82 

83 

84class TaskClassSortMethod(Enum): 

85 """ 

86 Enum to represent ways to sort task types (classes). 

87 """ 

88 

89 NONE = 0 

90 TABLENAME = 1 

91 SHORTNAME = 2 

92 LONGNAME = 3 

93 

94 

95def sort_task_classes_in_place( 

96 classlist: List[Type[Task]], 

97 sortmethod: TaskClassSortMethod, 

98 req: "CamcopsRequest" = None, 

99) -> None: 

100 """ 

101 Sort a list of task classes in place. 

102 

103 Args: 

104 classlist: the list of task classes 

105 sortmethod: a :class:`TaskClassSortMethod` enum 

106 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

107 """ 

108 if sortmethod == TaskClassSortMethod.TABLENAME: 

109 classlist.sort(key=lambda c: c.tablename) 

110 elif sortmethod == TaskClassSortMethod.SHORTNAME: 

111 classlist.sort(key=lambda c: c.shortname) 

112 elif sortmethod == TaskClassSortMethod.LONGNAME: 

113 assert req is not None 

114 classlist.sort(key=lambda c: c.longname(req)) 

115 

116 

117# ============================================================================= 

118# Cache task class mapping 

119# ============================================================================= 

120# Function, staticmethod, classmethod? 

121# https://stackoverflow.com/questions/8108688/in-python-when-should-i-use-a-function-instead-of-a-method # noqa 

122# https://stackoverflow.com/questions/11788195/module-function-vs-staticmethod-vs-classmethod-vs-no-decorators-which-idiom-is # noqa 

123# https://stackoverflow.com/questions/15017734/using-static-methods-in-python-best-practice # noqa 

124 

125 

126def task_classes_from_table_names( 

127 tablenames: List[str], 

128 sortmethod: TaskClassSortMethod = TaskClassSortMethod.NONE, 

129) -> List[Type[Task]]: 

130 """ 

131 Transforms a list of task base tablenames into a list of task classes, 

132 appropriately sorted. 

133 

134 Args: 

135 tablenames: list of task base table names 

136 sortmethod: a :class:`TaskClassSortMethod` enum 

137 

138 Returns: 

139 a list of task classes, in the order requested 

140 

141 Raises: 

142 :exc:`KeyError` if a table name is invalid 

143 

144 """ 

145 assert sortmethod != TaskClassSortMethod.LONGNAME 

146 d = tablename_to_task_class_dict() 

147 classes = [] # type: List[Type[Task]] 

148 for tablename in tablenames: 

149 cls = d[tablename] 

150 classes.append(cls) 

151 sort_task_classes_in_place(classes, sortmethod) 

152 return classes 

153 

154 

155@cache_region_static.cache_on_arguments(function_key_generator=fkg) 

156def all_tracker_task_classes() -> List[Type[Task]]: 

157 """ 

158 Returns a list of all task classes that provide tracker information. 

159 """ 

160 return [ 

161 cls 

162 for cls in Task.all_subclasses_by_shortname() 

163 if cls.provides_trackers 

164 ] 

165 

166 

167# ============================================================================= 

168# Define a filter to apply to tasks 

169# ============================================================================= 

170 

171 

172class TaskFilter(Base): 

173 """ 

174 SQLAlchemy ORM object representing task filter criteria. 

175 """ 

176 

177 __tablename__ = "_task_filters" 

178 

179 # Lots of these could be changed into lists; for example, filtering to 

180 # multiple devices, multiple users, multiple text patterns. For 

181 # AND-joining, there is little clear benefit (one could always AND-join 

182 # multiple filters with SQL). For OR-joining, this is more useful. 

183 # - surname: use ID numbers instead; not very likely to have >1 surname 

184 # - forename: ditto 

185 # - DOB: ditto 

186 # - sex: just eliminate the filter if you don't care about sex 

187 # - task_types: needs a list 

188 # - device_id: might as well make it a list 

189 # - user_id: might as well make it a list 

190 # - group_id: might as well make it a list 

191 # - start_datetime: single only 

192 # - end_datetime: single only 

193 # - text_contents: might as well make it a list 

194 # - ID numbers: a list, joined with OR. 

195 id = Column( 

196 "id", 

197 Integer, 

198 primary_key=True, 

199 autoincrement=True, 

200 index=True, 

201 comment="Task filter ID (arbitrary integer)", 

202 ) 

203 # Task type filters 

204 task_types = Column( 

205 "task_types", 

206 StringListType, 

207 comment="Task filter: task type(s), as CSV list of table names", 

208 ) 

209 tasks_offering_trackers_only = Column( 

210 "tasks_offering_trackers_only", 

211 Boolean, 

212 comment="Task filter: restrict to tasks offering trackers only?", 

213 ) 

214 tasks_with_patient_only = Column( 

215 "tasks_with_patient_only", 

216 Boolean, 

217 comment="Task filter: restrict to tasks with a patient (non-anonymous " 

218 "tasks) only?", 

219 ) 

220 # Patient-related filters 

221 surname = Column( 

222 "surname", PatientNameColType, comment="Task filter: surname" 

223 ) 

224 forename = Column( 

225 "forename", PatientNameColType, comment="Task filter: forename" 

226 ) 

227 dob = Column( 

228 "dob", Date, comment="Task filter: DOB" 

229 ) # type: Optional[datetime.date] 

230 sex = Column("sex", SexColType, comment="Task filter: sex") 

231 idnum_criteria = Column( # new in v2.0.1 

232 "idnum_criteria", 

233 IdNumReferenceListColType, 

234 comment="ID filters as JSON; the ID number definitions are joined " 

235 "with OR", 

236 ) 

237 # Other filters 

238 device_ids = Column( 

239 "device_ids", 

240 IntListType, 

241 comment="Task filter: source device ID(s), as CSV", 

242 ) 

243 adding_user_ids = Column( 

244 "user_ids", 

245 IntListType, 

246 comment="Task filter: adding (uploading) user ID(s), as CSV", 

247 ) 

248 group_ids = Column( 

249 "group_ids", IntListType, comment="Task filter: group ID(s), as CSV" 

250 ) 

251 start_datetime = Column( 

252 "start_datetime_iso8601", 

253 PendulumDateTimeAsIsoTextColType, 

254 comment="Task filter: start date/time (UTC as ISO8601)", 

255 ) # type: Union[None, Pendulum, datetime.datetime] 

256 end_datetime = Column( 

257 "end_datetime_iso8601", 

258 PendulumDateTimeAsIsoTextColType, 

259 comment="Task filter: end date/time (UTC as ISO8601)", 

260 ) # type: Union[None, Pendulum, datetime.datetime] 

261 # Implemented on the Python side for indexed lookup: 

262 text_contents = Column( 

263 "text_contents", 

264 StringListType, 

265 comment="Task filter: filter text fields", 

266 ) # task must contain ALL the strings in AT LEAST ONE of its text columns 

267 # Implemented on the Python side for non-indexed lookup: 

268 complete_only = Column( 

269 "complete_only", Boolean, comment="Task filter: task complete?" 

270 ) 

271 

272 def __init__(self) -> None: 

273 # We need to initialize these explicitly, because if we create an 

274 # instance via "x = TaskFilter()", they will be initialized to None, 

275 # without any recourse to our database to-and-fro conversion code for 

276 # each fieldtype. 

277 # (If we load from a database, things will be fine.) 

278 self.idnum_criteria = [] # type: List[IdNumReference] 

279 self.device_ids = [] # type: List[int] 

280 self.adding_user_ids = [] # type: List[int] 

281 self.group_ids = [] # type: List[int] 

282 self.text_contents = [] # type: List[str] 

283 

284 # ANYTHING YOU ADD BELOW HERE MUST ALSO BE IN init_on_load(). 

285 # Or call it, of course, but we like to keep on the happy side of the 

286 # PyCharm type checker. 

287 

288 # Python-only filtering attributes (i.e. not saved to database) 

289 self.era = None # type: Optional[str] 

290 self.finalized_only = False # used for exports 

291 self.must_have_idnum_type = None # type: Optional[int] 

292 

293 # Other Python-only attributes 

294 self._sort_method = TaskClassSortMethod.NONE 

295 self._task_classes = None # type: Optional[List[Type[Task]]] 

296 

297 @reconstructor 

298 def init_on_load(self): 

299 """ 

300 SQLAlchemy function to recreate after loading from the database. 

301 """ 

302 self.era = None # type: Optional[str] 

303 self.finalized_only = False 

304 self.must_have_idnum_type = None # type: Optional[int] 

305 

306 self._sort_method = TaskClassSortMethod.NONE 

307 self._task_classes = None # type: Optional[List[Type[Task]]] 

308 

309 def __repr__(self) -> str: 

310 return auto_repr(self, with_addr=True) 

311 

312 def set_sort_method(self, sort_method: TaskClassSortMethod) -> None: 

313 """ 

314 Sets the sorting method for task types. 

315 """ 

316 assert sort_method != TaskClassSortMethod.LONGNAME, ( 

317 "If you want to use that sorting method, you need to save a " 

318 "request object, because long task names use translation" 

319 ) 

320 self._sort_method = sort_method 

321 

322 @property 

323 def task_classes(self) -> List[Type[Task]]: 

324 """ 

325 Return a list of task classes permitted by the filter. 

326 

327 Uses caching, since the filter will be called repeatedly. 

328 """ 

329 if self._task_classes is None: 

330 self._task_classes = [] # type: List[Type[Task]] 

331 if self.task_types: 

332 starting_classes = task_classes_from_table_names( 

333 self.task_types 

334 ) 

335 else: 

336 starting_classes = Task.all_subclasses_by_shortname() 

337 skip_anonymous_tasks = self.skip_anonymous_tasks() 

338 for cls in starting_classes: 

339 if ( 

340 self.tasks_offering_trackers_only 

341 and not cls.provides_trackers 

342 ): 

343 # Class doesn't provide trackers; skip 

344 continue 

345 if skip_anonymous_tasks and not cls.has_patient: 

346 # Anonymous task; skip 

347 continue 

348 if self.text_contents and not cls.get_text_filter_columns(): 

349 # Text filter and task has no text columns; skip 

350 continue 

351 self._task_classes.append(cls) 

352 sort_task_classes_in_place(self._task_classes, self._sort_method) 

353 return self._task_classes 

354 

355 def skip_anonymous_tasks(self) -> bool: 

356 """ 

357 Should we skip anonymous tasks? 

358 """ 

359 return self.tasks_with_patient_only or self.any_patient_filtering() 

360 

361 def offers_all_task_types(self) -> bool: 

362 """ 

363 Does this filter offer every single task class? Used for efficiency 

364 when using indexes. (Since ignored.) 

365 """ 

366 if self.tasks_offering_trackers_only: 

367 return False 

368 if self.skip_anonymous_tasks(): 

369 return False 

370 if not self.task_types: 

371 return True 

372 return set(self.task_classes) == set(Task.all_subclasses_by_shortname) 

373 

374 def offers_all_non_anonymous_task_types(self) -> bool: 

375 """ 

376 Does this filter offer every single non-anonymous task class? Used for 

377 efficiency when using indexes. 

378 """ 

379 offered_task_classes = self.task_classes 

380 for taskclass in Task.all_subclasses_by_shortname(): 

381 if taskclass.is_anonymous: 

382 continue 

383 if taskclass not in offered_task_classes: 

384 return False 

385 return True 

386 

387 @property 

388 def task_tablename_list(self) -> List[str]: 

389 """ 

390 Returns the base table names for all task types permitted by the 

391 filter. 

392 """ 

393 return [cls.__tablename__ for cls in self.task_classes] 

394 

395 def any_patient_filtering(self) -> bool: 

396 """ 

397 Is some sort of patient filtering being applied? 

398 """ 

399 return ( 

400 bool(self.surname) 

401 or bool(self.forename) 

402 or (self.dob is not None) 

403 or bool(self.sex) 

404 or bool(self.idnum_criteria) 

405 ) 

406 

407 def any_specific_patient_filtering(self) -> bool: 

408 """ 

409 Are there filters that would restrict to one or a few patients? 

410 

411 (Differs from :func:`any_patient_filtering` with respect to sex.) 

412 """ 

413 return ( 

414 bool(self.surname) 

415 or bool(self.forename) 

416 or self.dob is not None 

417 or bool(self.idnum_criteria) 

418 ) 

419 

420 def get_only_iddef(self) -> Optional["IdNumReference"]: 

421 """ 

422 If a single ID number type/value restriction is being applied, return 

423 it, as an 

424 :class:`camcops_server.cc_modules.cc_simpleobjects.IdNumReference`. 

425 Otherwise, return ``None``. 

426 """ 

427 if len(self.idnum_criteria) != 1: 

428 return None 

429 return self.idnum_criteria[0] 

430 

431 def get_group_names(self, req: "CamcopsRequest") -> List[str]: 

432 """ 

433 Get the names of any groups to which we are restricting. 

434 """ 

435 groups = ( 

436 req.dbsession.query(Group) 

437 .filter(Group.id.in_(self.group_ids)) 

438 .all() 

439 ) # type: List[Group] 

440 return [g.name if g and g.name else "" for g in groups] 

441 

442 def get_user_names(self, req: "CamcopsRequest") -> List[str]: 

443 """ 

444 Get the usernames of any uploading users to which we are restricting. 

445 """ 

446 users = ( 

447 req.dbsession.query(User) 

448 .filter(User.id.in_(self.adding_user_ids)) 

449 .all() 

450 ) # type: List[User] 

451 return [u.username if u and u.username else "" for u in users] 

452 

453 def get_device_names(self, req: "CamcopsRequest") -> List[str]: 

454 """ 

455 Get the names of any devices to which we are restricting. 

456 """ 

457 devices = ( 

458 req.dbsession.query(Device) 

459 .filter(Device.id.in_(self.device_ids)) 

460 .all() 

461 ) # type: List[Device] 

462 return [d.name if d and d.name else "" for d in devices] 

463 

464 def clear(self) -> None: 

465 """ 

466 Clear all parts of the filter. 

467 """ 

468 self.task_types = [] # type: List[str] 

469 

470 self.surname = None 

471 self.forename = None 

472 self.dob = None # type: Optional[datetime.date] 

473 self.sex = None 

474 self.idnum_criteria = [] # type: List[IdNumReference] 

475 

476 self.device_ids = [] # type: List[int] 

477 self.adding_user_ids = [] # type: List[int] 

478 self.group_ids = [] # type: List[int] 

479 self.start_datetime = ( 

480 None 

481 ) # type: Union[None, Pendulum, datetime.datetime] 

482 self.end_datetime = ( 

483 None 

484 ) # type: Union[None, Pendulum, datetime.datetime] 

485 self.text_contents = [] # type: List[str] 

486 

487 self.complete_only = None # type: Optional[bool] 

488 

489 def dates_inconsistent(self) -> bool: 

490 """ 

491 Are inconsistent dates specified, such that no tasks should be 

492 returned? 

493 """ 

494 return ( 

495 self.start_datetime 

496 and self.end_datetime 

497 and self.end_datetime < self.start_datetime 

498 ) 

499 

500 def filter_query_by_patient(self, q: Query, via_index: bool) -> Query: 

501 """ 

502 Restricts an query that has *already been joined* to the 

503 :class:`camcops_server.cc_modules.cc_patient.Patient` class, according 

504 to the patient filtering criteria. 

505 

506 Args: 

507 q: the starting SQLAlchemy ORM Query 

508 via_index: 

509 If ``True``, the query relates to a 

510 :class:`camcops_server.cc_modules.cc_taskindex.TaskIndexEntry` 

511 and we should restrict it according to the 

512 :class:`camcops_server.cc_modules.cc_taskindex.PatientIdNumIndexEntry` 

513 class. If ``False``, the query relates to a 

514 :class:`camcops_server.cc_modules.cc_taskindex.Task` and we 

515 should restrict according to 

516 :class:`camcops_server.cc_modules.cc_patientidnum.PatientIdNum`. 

517 

518 Returns: 

519 a revised Query 

520 

521 """ 

522 if self.surname: 

523 q = q.filter(func.upper(Patient.surname) == self.surname.upper()) 

524 if self.forename: 

525 q = q.filter(func.upper(Patient.forename) == self.forename.upper()) 

526 if self.dob is not None: 

527 q = q.filter(Patient.dob == self.dob) 

528 if self.sex: 

529 q = q.filter(func.upper(Patient.sex) == self.sex.upper()) 

530 

531 if self.idnum_criteria: 

532 id_filter_parts = [] # type: List[ColumnElement] 

533 if via_index: 

534 q = q.join(PatientIdNumIndexEntry) 

535 # "Specify possible ID number values" 

536 for iddef in self.idnum_criteria: 

537 id_filter_parts.append( 

538 and_( 

539 PatientIdNumIndexEntry.which_idnum 

540 == iddef.which_idnum, # noqa 

541 PatientIdNumIndexEntry.idnum_value 

542 == iddef.idnum_value, 

543 ) 

544 ) 

545 # Use OR (disjunction) of the specified values: 

546 q = q.filter(or_(*id_filter_parts)) 

547 # "Must have a value for a given ID number type" 

548 if self.must_have_idnum_type: 

549 # noinspection PyComparisonWithNone,PyPep8 

550 q = q.filter( 

551 and_( 

552 PatientIdNumIndexEntry.which_idnum 

553 == self.must_have_idnum_type, # noqa 

554 PatientIdNumIndexEntry.idnum_value 

555 != None, # noqa: E711 

556 ) 

557 ) 

558 else: 

559 # q = q.join(PatientIdNum) # fails 

560 q = q.join(Patient.idnums) 

561 # "Specify possible ID number values" 

562 for iddef in self.idnum_criteria: 

563 id_filter_parts.append( 

564 and_( 

565 PatientIdNum.which_idnum == iddef.which_idnum, 

566 PatientIdNum.idnum_value == iddef.idnum_value, 

567 ) 

568 ) 

569 # Use OR (disjunction) of the specified values: 

570 q = q.filter(or_(*id_filter_parts)) 

571 # "Must have a value for a given ID number type" 

572 if self.must_have_idnum_type: 

573 # noinspection PyComparisonWithNone,PyPep8 

574 q = q.filter( 

575 and_( 

576 PatientIdNum.which_idnum 

577 == self.must_have_idnum_type, 

578 PatientIdNum.idnum_value != None, # noqa: E711 

579 ) 

580 ) 

581 

582 return q 

583 

584 @property 

585 def start_datetime_utc(self) -> Optional[Pendulum]: 

586 if not self.start_datetime: 

587 return None 

588 return convert_datetime_to_utc(self.start_datetime) 

589 

590 @property 

591 def end_datetime_utc(self) -> Optional[Pendulum]: 

592 if not self.end_datetime: 

593 return None 

594 return convert_datetime_to_utc(self.end_datetime) 

595 

596 

597def encode_task_filter(taskfilter: TaskFilter) -> Dict: 

598 return { 

599 "task_types": taskfilter.task_types, 

600 "group_ids": taskfilter.group_ids, 

601 } 

602 

603 

604# noinspection PyUnusedLocal 

605def decode_task_filter(d: Dict, cls: Type) -> TaskFilter: 

606 taskfilter = TaskFilter() 

607 taskfilter.task_types = d["task_types"] 

608 taskfilter.group_ids = d["group_ids"] 

609 

610 return taskfilter 

611 

612 

613register_class_for_json( 

614 cls=TaskFilter, 

615 obj_to_dict_fn=encode_task_filter, 

616 dict_to_obj_fn=decode_task_filter, 

617)