Coverage for cc_modules/cc_exportrecipient.py: 59%

168 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-08 23:14 +0000

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_exportrecipient.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry. 

9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**ExportRecipient class.** 

29 

30""" 

31 

32import logging 

33from typing import List, Optional, TYPE_CHECKING 

34 

35from cardinal_pythonlib.logs import BraceStyleAdapter 

36from cardinal_pythonlib.reprfunc import simple_repr 

37from cardinal_pythonlib.sqlalchemy.list_types import ( 

38 IntListType, 

39 StringListType, 

40) 

41from cardinal_pythonlib.sqlalchemy.orm_inspect import gen_columns 

42from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_url 

43from sqlalchemy.event.api import listens_for 

44from sqlalchemy.orm import reconstructor, Session as SqlASession 

45from sqlalchemy.sql.schema import Column 

46from sqlalchemy.sql.sqltypes import ( 

47 BigInteger, 

48 Boolean, 

49 DateTime, 

50 Integer, 

51 Text, 

52) 

53 

54from camcops_server.cc_modules.cc_exportrecipientinfo import ( 

55 ExportRecipientInfo, 

56) 

57from camcops_server.cc_modules.cc_simpleobjects import TaskExportOptions 

58from camcops_server.cc_modules.cc_sqla_coltypes import ( 

59 EmailAddressColType, 

60 ExportRecipientNameColType, 

61 ExportTransmissionMethodColType, 

62 FileSpecColType, 

63 HostnameColType, 

64 UrlColType, 

65 UserNameExternalColType, 

66) 

67from camcops_server.cc_modules.cc_sqlalchemy import Base 

68 

69if TYPE_CHECKING: 

70 from sqlalchemy.engine.base import Connection 

71 from sqlalchemy.orm.mapper import Mapper 

72 from camcops_server.cc_modules.cc_task import Task 

73 

74log = BraceStyleAdapter(logging.getLogger(__name__)) 

75 

76 

77# ============================================================================= 

78# ExportRecipient class 

79# ============================================================================= 

80 

81 

82class ExportRecipient(ExportRecipientInfo, Base): 

83 """ 

84 SQLAlchemy ORM class representing an export recipient. 

85 

86 This has a close relationship with (and inherits from) 

87 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo` 

88 (q.v.). 

89 

90 Full details of parameters are in the docs for the config file. 

91 """ # noqa 

92 

93 __tablename__ = "_export_recipients" 

94 

95 IGNORE_FOR_EQ_ATTRNAMES = ExportRecipientInfo.IGNORE_FOR_EQ_ATTRNAMES + [ 

96 # Attribute names to ignore for equality comparison (is one recipient 

97 # record functionally equal to another?). 

98 "id", 

99 "current", 

100 "group_names", # Python only 

101 ] 

102 RECOPY_EACH_TIME_FROM_CONFIG_ATTRNAMES = [ 

103 # Fields representing sensitive information, not stored in the 

104 # database. See also init_on_load() function. 

105 "email_host_password", 

106 "fhir_app_secret", 

107 "fhir_launch_token", 

108 "redcap_api_key", 

109 ] 

110 

111 # ------------------------------------------------------------------------- 

112 # Identifying this object, and whether it's the "live" version 

113 # ------------------------------------------------------------------------- 

114 id = Column( 

115 "id", 

116 BigInteger, 

117 primary_key=True, 

118 autoincrement=True, 

119 index=True, 

120 comment="Export recipient ID (arbitrary primary key)", 

121 ) 

122 recipient_name = Column( 

123 "recipient_name", 

124 ExportRecipientNameColType, 

125 nullable=False, 

126 comment="Name of export recipient", 

127 ) 

128 current = Column( 

129 "current", 

130 Boolean, 

131 default=False, 

132 nullable=False, 

133 comment="Is this the current record for this recipient? (If not, it's " 

134 "a historical record for audit purposes.)", 

135 ) 

136 

137 # ------------------------------------------------------------------------- 

138 # How to export 

139 # ------------------------------------------------------------------------- 

140 transmission_method = Column( 

141 "transmission_method", 

142 ExportTransmissionMethodColType, 

143 nullable=False, 

144 comment="Export transmission method (e.g. hl7, file)", 

145 ) 

146 push = Column( 

147 "push", 

148 Boolean, 

149 default=False, 

150 nullable=False, 

151 comment="Push (support auto-export on upload)?", 

152 ) 

153 task_format = Column( 

154 "task_format", 

155 ExportTransmissionMethodColType, 

156 comment="Format that task information should be sent in (e.g. PDF), " 

157 "if not predetermined by the transmission method", 

158 ) 

159 xml_field_comments = Column( 

160 "xml_field_comments", 

161 Boolean, 

162 default=True, 

163 nullable=False, 

164 comment="Whether to include field comments in XML output", 

165 ) 

166 

167 # ------------------------------------------------------------------------- 

168 # What to export 

169 # ------------------------------------------------------------------------- 

170 all_groups = Column( 

171 "all_groups", 

172 Boolean, 

173 default=False, 

174 nullable=False, 

175 comment="Export all groups? (If not, see group_ids.)", 

176 ) 

177 group_ids = Column( 

178 "group_ids", 

179 IntListType, 

180 comment="Integer IDs of CamCOPS group to export data from (as CSV)", 

181 ) 

182 tasks = Column( 

183 "tasks", 

184 StringListType, 

185 comment="Base table names of CamCOPS tasks to export data from " 

186 "(as CSV)", 

187 ) 

188 start_datetime_utc = Column( 

189 "start_datetime_utc", 

190 DateTime, 

191 comment="Start date/time for tasks (UTC)", 

192 ) 

193 end_datetime_utc = Column( 

194 "end_datetime_utc", DateTime, comment="End date/time for tasks (UTC)" 

195 ) 

196 finalized_only = Column( 

197 "finalized_only", 

198 Boolean, 

199 default=True, 

200 nullable=False, 

201 comment="Send only finalized tasks", 

202 ) 

203 include_anonymous = Column( 

204 "include_anonymous", 

205 Boolean, 

206 default=False, 

207 nullable=False, 

208 comment="Include anonymous tasks? " 

209 "Not applicable to some methods (e.g. HL7)", 

210 ) 

211 primary_idnum = Column( 

212 "primary_idnum", 

213 Integer, 

214 nullable=False, 

215 comment="Which ID number is used as the primary ID?", 

216 ) 

217 require_idnum_mandatory = Column( 

218 "require_idnum_mandatory", 

219 Boolean, 

220 comment="Must the primary ID number be mandatory in the relevant " 

221 "policy?", 

222 ) 

223 

224 # ------------------------------------------------------------------------- 

225 # Database 

226 # ------------------------------------------------------------------------- 

227 db_url = Column( 

228 "db_url", 

229 UrlColType, 

230 comment="(DATABASE) SQLAlchemy database URL for export", 

231 ) 

232 db_echo = Column( 

233 "db_echo", 

234 Boolean, 

235 default=False, 

236 nullable=False, 

237 comment="(DATABASE) Echo SQL applied to destination database?", 

238 ) 

239 db_include_blobs = Column( 

240 "db_include_blobs", 

241 Boolean, 

242 default=True, 

243 nullable=False, 

244 comment="(DATABASE) Include BLOBs?", 

245 ) 

246 db_add_summaries = Column( 

247 "db_add_summaries", 

248 Boolean, 

249 default=True, 

250 nullable=False, 

251 comment="(DATABASE) Add summary information?", 

252 ) 

253 db_patient_id_per_row = Column( 

254 "db_patient_id_per_row", 

255 Boolean, 

256 default=True, 

257 nullable=False, 

258 comment="(DATABASE) Add patient ID information per row?", 

259 ) 

260 

261 # ------------------------------------------------------------------------- 

262 # Email 

263 # ------------------------------------------------------------------------- 

264 email_host = Column( 

265 "email_host", 

266 HostnameColType, 

267 comment="(EMAIL) E-mail (SMTP) server host name/IP address", 

268 ) 

269 email_port = Column( 

270 "email_port", 

271 Integer, 

272 comment="(EMAIL) E-mail (SMTP) server port number", 

273 ) 

274 email_use_tls = Column( 

275 "email_use_tls", 

276 Boolean, 

277 default=True, 

278 nullable=False, 

279 comment="(EMAIL) Use explicit TLS connection?", 

280 ) 

281 email_host_username = Column( 

282 "email_host_username", 

283 UserNameExternalColType, 

284 comment="(EMAIL) Username on e-mail server", 

285 ) 

286 # email_host_password: not stored in database 

287 email_from = Column( 

288 "email_from", 

289 EmailAddressColType, 

290 comment='(EMAIL) "From:" address(es)', 

291 ) 

292 email_sender = Column( 

293 "email_sender", 

294 EmailAddressColType, 

295 comment='(EMAIL) "Sender:" address(es)', 

296 ) 

297 email_reply_to = Column( 

298 "email_reply_to", 

299 EmailAddressColType, 

300 comment='(EMAIL) "Reply-To:" address(es)', 

301 ) 

302 email_to = Column( 

303 "email_to", Text, comment='(EMAIL) "To:" recipient(s), as a CSV list' 

304 ) 

305 email_cc = Column( 

306 "email_cc", Text, comment='(EMAIL) "CC:" recipient(s), as a CSV list' 

307 ) 

308 email_bcc = Column( 

309 "email_bcc", Text, comment='(EMAIL) "BCC:" recipient(s), as a CSV list' 

310 ) 

311 email_patient_spec = Column( 

312 "email_patient", 

313 FileSpecColType, 

314 comment="(EMAIL) Patient specification", 

315 ) 

316 email_patient_spec_if_anonymous = Column( 

317 "email_patient_spec_if_anonymous", 

318 FileSpecColType, 

319 comment="(EMAIL) Patient specification for anonymous tasks", 

320 ) 

321 email_subject = Column( 

322 "email_subject", 

323 FileSpecColType, 

324 comment="(EMAIL) Subject specification", 

325 ) 

326 email_body_as_html = Column( 

327 "email_body_as_html", 

328 Boolean, 

329 default=False, 

330 nullable=False, 

331 comment="(EMAIL) Is the body HTML, rather than plain text?", 

332 ) 

333 email_body = Column("email_body", Text, comment="(EMAIL) Body contents") 

334 email_keep_message = Column( 

335 "email_keep_message", 

336 Boolean, 

337 default=False, 

338 nullable=False, 

339 comment="(EMAIL) Keep entire message?", 

340 ) 

341 

342 # ------------------------------------------------------------------------- 

343 # HL7 

344 # ------------------------------------------------------------------------- 

345 hl7_host = Column( 

346 "hl7_host", 

347 HostnameColType, 

348 comment="(HL7) Destination host name/IP address", 

349 ) 

350 hl7_port = Column( 

351 "hl7_port", Integer, comment="(HL7) Destination port number" 

352 ) 

353 hl7_ping_first = Column( 

354 "hl7_ping_first", 

355 Boolean, 

356 default=False, 

357 nullable=False, 

358 comment="(HL7) Ping via TCP/IP before sending HL7 messages?", 

359 ) 

360 hl7_network_timeout_ms = Column( 

361 "hl7_network_timeout_ms", 

362 Integer, 

363 comment="(HL7) Network timeout (ms).", 

364 ) 

365 hl7_keep_message = Column( 

366 "hl7_keep_message", 

367 Boolean, 

368 default=False, 

369 nullable=False, 

370 comment="(HL7) Keep copy of message in database? (May be large!)", 

371 ) 

372 hl7_keep_reply = Column( 

373 "hl7_keep_reply", 

374 Boolean, 

375 default=False, 

376 nullable=False, 

377 comment="(HL7) Keep copy of server's reply in database?", 

378 ) 

379 hl7_debug_divert_to_file = Column( 

380 "hl7_debug_divert_to_file", 

381 Boolean, 

382 default=False, 

383 nullable=False, 

384 comment="(HL7 debugging option) Divert messages to files?", 

385 ) 

386 hl7_debug_treat_diverted_as_sent = Column( 

387 "hl7_debug_treat_diverted_as_sent", 

388 Boolean, 

389 default=False, 

390 nullable=False, 

391 comment="(HL7 debugging option) Treat messages diverted to file as sent", # noqa 

392 ) 

393 

394 # ------------------------------------------------------------------------- 

395 # File 

396 # ------------------------------------------------------------------------- 

397 file_patient_spec = Column( 

398 "file_patient_spec", 

399 FileSpecColType, 

400 comment="(FILE) Patient part of filename specification", 

401 ) 

402 file_patient_spec_if_anonymous = Column( 

403 "file_patient_spec_if_anonymous", 

404 FileSpecColType, 

405 comment="(FILE) Patient part of filename specification for anonymous tasks", # noqa: E501 

406 ) 

407 file_filename_spec = Column( 

408 "file_filename_spec", 

409 FileSpecColType, 

410 comment="(FILE) Filename specification", 

411 ) 

412 file_make_directory = Column( 

413 "file_make_directory", 

414 Boolean, 

415 default=True, 

416 nullable=False, 

417 comment=( 

418 "(FILE) Make destination directory if it doesn't already exist" 

419 ), 

420 ) 

421 file_overwrite_files = Column( 

422 "file_overwrite_files", 

423 Boolean, 

424 default=False, 

425 nullable=False, 

426 comment="(FILE) Overwrite existing files", 

427 ) 

428 file_export_rio_metadata = Column( 

429 "file_export_rio_metadata", 

430 Boolean, 

431 default=False, 

432 nullable=False, 

433 comment="(FILE) Export RiO metadata file along with main file?", 

434 ) 

435 file_script_after_export = Column( 

436 "file_script_after_export", 

437 Text, 

438 comment="(FILE) Command/script to run after file export", 

439 ) 

440 

441 # ------------------------------------------------------------------------- 

442 # File/RiO 

443 # ------------------------------------------------------------------------- 

444 rio_idnum = Column( 

445 "rio_idnum", 

446 Integer, 

447 comment="(FILE / RiO) RiO metadata: which ID number is the RiO ID?", 

448 ) 

449 rio_uploading_user = Column( 

450 "rio_uploading_user", 

451 Text, 

452 comment="(FILE / RiO) RiO metadata: name of automatic upload user", 

453 ) 

454 rio_document_type = Column( 

455 "rio_document_type", 

456 Text, 

457 comment="(FILE / RiO) RiO metadata: document type for RiO", 

458 ) 

459 

460 # ------------------------------------------------------------------------- 

461 # REDCap export 

462 # ------------------------------------------------------------------------- 

463 redcap_api_url = Column( 

464 "redcap_api_url", 

465 Text, 

466 comment="(REDCap) REDCap API URL, pointing to the REDCap server", 

467 ) 

468 redcap_fieldmap_filename = Column( 

469 "redcap_fieldmap_filename", 

470 Text, 

471 comment="(REDCap) File defining CamCOPS-to-REDCap field mapping", 

472 ) 

473 

474 # ------------------------------------------------------------------------- 

475 # FHIR export 

476 # ------------------------------------------------------------------------- 

477 fhir_api_url = Column( 

478 "fhir_api_url", 

479 Text, 

480 comment="(FHIR) FHIR API URL, pointing to the FHIR server", 

481 ) 

482 fhir_app_id = Column( 

483 "fhir_app_id", 

484 Text, 

485 comment="(FHIR) FHIR app ID, identifying CamCOPS as the data source", 

486 ) 

487 fhir_concurrent = Column( 

488 "fhir_concurrent", 

489 Boolean, 

490 default=False, 

491 nullable=True, 

492 comment="(FHIR) Server supports concurrency (parallel processing)?", 

493 ) 

494 

495 def __init__(self, *args, **kwargs) -> None: 

496 """ 

497 Creates a blank :class:`ExportRecipient` object. 

498 

499 NB not called when SQLAlchemy objects loaded from database; see 

500 :meth:`init_on_load` instead. 

501 """ 

502 super().__init__(*args, **kwargs) 

503 

504 def __hash__(self) -> int: 

505 """ 

506 Used by the ``merge_db`` function, and specifically the old-to-new map 

507 maintained by :func:`cardinal_pythonlib.sqlalchemy.merge_db.merge_db`. 

508 """ 

509 return hash(f"{self.id}_{self.recipient_name}") 

510 

511 @reconstructor 

512 def init_on_load(self) -> None: 

513 """ 

514 Called when SQLAlchemy recreates an object; see 

515 https://docs.sqlalchemy.org/en/latest/orm/constructors.html. 

516 

517 Sets Python-only attributes. 

518 

519 See also IGNORE_FOR_EQ_ATTRNAMES, 

520 NEEDS_RECOPYING_EACH_TIME_FROM_CONFIG_ATTRNAMES. 

521 """ 

522 self.group_names = [] # type: List[str] 

523 

524 # Within NEEDS_RECOPYING_EACH_TIME_FROM_CONFIG_ATTRNAMES: 

525 self.email_host_password = "" 

526 self.fhir_app_secret = "" 

527 self.fhir_launch_token = None # type: Optional[str] 

528 self.redcap_api_key = "" 

529 

530 def get_attrnames(self) -> List[str]: 

531 """ 

532 Returns all relevant attribute names. 

533 """ 

534 attrnames = set([attrname for attrname, _ in gen_columns(self)]) 

535 attrnames.update( 

536 key for key in self.__dict__ if not key.startswith("_") 

537 ) 

538 return sorted(attrnames) 

539 

540 def __repr__(self) -> str: 

541 return simple_repr(self, self.get_attrnames()) 

542 

543 def is_upload_suitable_for_push( 

544 self, tablename: str, uploading_group_id: int 

545 ) -> bool: 

546 """ 

547 Might an upload potentially give tasks to be "pushed"? 

548 

549 Called by 

550 :func:`camcops_server.cc_modules.cc_client_api_core.get_task_push_export_pks`. 

551 

552 Args: 

553 tablename: table name being uploaded 

554 uploading_group_id: group ID if the uploading user 

555 

556 Returns: 

557 whether this upload should be considered further 

558 """ 

559 if not self.push: 

560 # Not a push export recipient 

561 return False 

562 if self.tasks and tablename not in self.tasks: 

563 # Recipient is restricted to tasks that don't include the table 

564 # being uploaded (or, the table is a subtable that we don't care 

565 # about) 

566 return False 

567 if not self.all_groups: 

568 # Recipient is restricted to specific groups 

569 if uploading_group_id not in self.group_ids: 

570 # Wrong group! 

571 return False 

572 return True 

573 

574 def is_task_suitable(self, task: "Task") -> bool: 

575 """ 

576 Used as a double-check that a task remains suitable. 

577 

578 Args: 

579 task: a :class:`camcops_server.cc_modules.cc_task.Task` 

580 

581 Returns: 

582 bool: is the task suitable for this recipient? 

583 """ 

584 

585 def _warn(reason: str) -> None: 

586 log.info( 

587 "For recipient {}, task {!r} is unsuitable: {}", 

588 self, 

589 task, 

590 reason, 

591 ) 

592 # Not a warning, actually; it's normal to see these because it 

593 # allows the client API to skip some checks for speed. 

594 

595 if self.tasks and task.tablename not in self.tasks: 

596 _warn(f"Task type {task.tablename!r} not included") 

597 return False 

598 

599 if not self.all_groups: 

600 task_group_id = task.group_id 

601 if task_group_id not in self.group_ids: 

602 _warn(f"group_id {task_group_id} not permitted") 

603 return False 

604 

605 if not self.include_anonymous and task.is_anonymous: 

606 _warn("task is anonymous") 

607 return False 

608 

609 if self.finalized_only and not task.is_preserved(): 

610 _warn("task not finalized") 

611 return False 

612 

613 if self.start_datetime_utc or self.end_datetime_utc: 

614 task_dt = task.get_creation_datetime_utc_tz_unaware() 

615 if self.start_datetime_utc and task_dt < self.start_datetime_utc: 

616 _warn("task created before recipient start_datetime_utc") 

617 return False 

618 if self.end_datetime_utc and task_dt >= self.end_datetime_utc: 

619 _warn("task created at/after recipient end_datetime_utc") 

620 return False 

621 

622 if not task.is_anonymous and self.primary_idnum is not None: 

623 patient = task.patient 

624 if not patient: 

625 _warn("missing patient") 

626 return False 

627 if not patient.has_idnum_type(self.primary_idnum): 

628 _warn( 

629 f"task's patient is missing ID number type " 

630 f"{self.primary_idnum}" 

631 ) 

632 return False 

633 

634 return True 

635 

636 @classmethod 

637 def get_existing_matching_recipient( 

638 cls, dbsession: SqlASession, recipient: "ExportRecipient" 

639 ) -> Optional["ExportRecipient"]: 

640 """ 

641 Retrieves an active instance from the database that matches ``other``, 

642 if there is one. 

643 

644 Args: 

645 dbsession: a :class:`sqlalchemy.orm.session.Session` 

646 recipient: an :class:`ExportRecipient` 

647 

648 Returns: 

649 a database instance of :class:`ExportRecipient` that matches, or 

650 ``None``. 

651 """ 

652 # noinspection PyPep8 

653 q = dbsession.query(cls).filter( 

654 cls.recipient_name == recipient.recipient_name, 

655 cls.current == True, # noqa: E712 

656 ) 

657 results = q.all() 

658 if len(results) > 1: 

659 raise ValueError( 

660 "Database has gone wrong: more than one active record for " 

661 "{t}.{c} = {r}".format( 

662 t=cls.__tablename__, 

663 c=cls.recipient_name.name, # column name from Column 

664 r=recipient.recipient_name, 

665 ) 

666 ) 

667 if results: 

668 r = results[0] 

669 if recipient == r: 

670 return r 

671 return None 

672 

673 @property 

674 def db_url_obscuring_password(self) -> Optional[str]: 

675 """ 

676 Returns the database URL (if present), but with its password obscured. 

677 """ 

678 if not self.db_url: 

679 return self.db_url 

680 return get_safe_url_from_url(self.db_url) 

681 

682 def get_task_export_options(self) -> TaskExportOptions: 

683 return TaskExportOptions( 

684 xml_include_comments=self.xml_field_comments, 

685 xml_with_header_comments=self.xml_field_comments, 

686 ) 

687 

688 

689# noinspection PyUnusedLocal 

690@listens_for(ExportRecipient, "after_insert") 

691@listens_for(ExportRecipient, "after_update") 

692def _check_current( 

693 mapper: "Mapper", connection: "Connection", target: ExportRecipient 

694) -> None: 

695 """ 

696 Ensures that only one :class:`ExportRecipient` is marked as ``current`` 

697 per ``recipient_name``. 

698 

699 As per 

700 https://stackoverflow.com/questions/6269469/mark-a-single-row-in-a-table-in-sqlalchemy. 

701 """ # noqa 

702 if target.current: 

703 # noinspection PyUnresolvedReferences 

704 connection.execute( 

705 ExportRecipient.__table__.update() 

706 .values(current=False) 

707 .where(ExportRecipient.recipient_name == target.recipient_name) 

708 .where(ExportRecipient.id != target.id) 

709 )