Coverage for cc_modules/cc_exportmodels.py: 33%

460 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-08 23:14 +0000

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_exportmodels.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry. 

9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**Define models for export functions (e.g. HL7, file-based export).** 

29 

30""" 

31 

32import logging 

33import os 

34import posixpath 

35import socket 

36import subprocess 

37import sys 

38from typing import Generator, List, Optional, Tuple, TYPE_CHECKING 

39 

40from cardinal_pythonlib.datetimefunc import ( 

41 get_now_utc_datetime, 

42 get_now_utc_pendulum, 

43) 

44from cardinal_pythonlib.email.sendmail import ( 

45 CONTENT_TYPE_HTML, 

46 CONTENT_TYPE_TEXT, 

47) 

48from cardinal_pythonlib.fileops import mkdir_p 

49from cardinal_pythonlib.logs import BraceStyleAdapter 

50from cardinal_pythonlib.network import ping 

51from cardinal_pythonlib.sqlalchemy.list_types import StringListType 

52from cardinal_pythonlib.sqlalchemy.orm_query import bool_from_exists_clause 

53import hl7 

54from pendulum import DateTime as Pendulum 

55from sqlalchemy.orm import reconstructor, relationship, Session as SqlASession 

56from sqlalchemy.sql.schema import Column, ForeignKey 

57from sqlalchemy.sql.sqltypes import ( 

58 BigInteger, 

59 Boolean, 

60 DateTime, 

61 Integer, 

62 Text, 

63 UnicodeText, 

64) 

65 

66from camcops_server.cc_modules.cc_constants import ( 

67 ConfigParamExportRecipient, 

68 FileType, 

69 UTF8, 

70) 

71from camcops_server.cc_modules.cc_email import Email 

72from camcops_server.cc_modules.cc_exportrecipient import ExportRecipient 

73from camcops_server.cc_modules.cc_exportrecipientinfo import ( 

74 ExportTransmissionMethod, 

75) 

76from camcops_server.cc_modules.cc_fhir import ( 

77 FhirExportException, 

78 FhirTaskExporter, 

79) 

80from camcops_server.cc_modules.cc_filename import change_filename_ext 

81from camcops_server.cc_modules.cc_hl7 import ( 

82 make_msh_segment, 

83 MLLPTimeoutClient, 

84 msg_is_successful_ack, 

85 SEGMENT_SEPARATOR, 

86) 

87from camcops_server.cc_modules.cc_redcap import ( 

88 RedcapExportException, 

89 RedcapTaskExporter, 

90) 

91from camcops_server.cc_modules.cc_sqla_coltypes import ( 

92 LongText, 

93 TableNameColType, 

94) 

95from camcops_server.cc_modules.cc_sqlalchemy import Base 

96from camcops_server.cc_modules.cc_taskcollection import ( 

97 TaskCollection, 

98 TaskSortMethod, 

99) 

100from camcops_server.cc_modules.cc_taskfactory import ( 

101 task_factory_no_security_checks, 

102) 

103 

104if TYPE_CHECKING: 

105 from camcops_server.cc_modules.cc_request import CamcopsRequest 

106 from camcops_server.cc_modules.cc_task import Task 

107 

108log = BraceStyleAdapter(logging.getLogger(__name__)) 

109 

110 

111# ============================================================================= 

112# Constants 

113# ============================================================================= 

114 

115DOS_NEWLINE = "\r\n" 

116 

117 

118# ============================================================================= 

119# Create task collections for export 

120# ============================================================================= 

121 

122 

123def get_collection_for_export( 

124 req: "CamcopsRequest", 

125 recipient: ExportRecipient, 

126 via_index: bool = True, 

127 debug: bool = False, 

128) -> TaskCollection: 

129 """ 

130 Returns an appropriate task collection for this export recipient, namely 

131 those tasks that are desired and (in the case of incremental exports) 

132 haven't already been sent. 

133 

134 "Not already sent" means "not already sent to an export recipient with 

135 the same name (even if other aspects of the export recipient have 

136 changed)". 

137 

138 Args: 

139 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

140 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient` 

141 via_index: use the task index (faster)? 

142 debug: report details? 

143 

144 Returns: 

145 a :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection` 

146 """ # noqa 

147 if not via_index: 

148 log.debug("Task index disabled for get_collection_for_export()") 

149 collection = TaskCollection( 

150 req=req, 

151 sort_method_by_class=TaskSortMethod.CREATION_DATE_ASC, 

152 current_only=True, 

153 via_index=via_index, 

154 export_recipient=recipient, 

155 ) 

156 if debug: 

157 log.debug( 

158 "get_collection_for_export(): recipient={!r}, " "collection={!r}", 

159 recipient, 

160 collection, 

161 ) 

162 return collection 

163 

164 

165def gen_exportedtasks( 

166 collection: TaskCollection, 

167) -> Generator["ExportedTask", None, None]: 

168 """ 

169 Generates task export entries from a collection. 

170 

171 Args: 

172 collection: a :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection` 

173 

174 Yields: 

175 :class:`ExportedTask` objects 

176 

177 """ # noqa 

178 dbsession = collection.dbsession 

179 recipient = collection.export_recipient 

180 assert recipient is not None, "TaskCollection has no export_recipient" 

181 for task in collection.gen_tasks_by_class(): 

182 et = ExportedTask(recipient, task) 

183 dbsession.add(et) 

184 yield et 

185 

186 

187def gen_tasks_having_exportedtasks( 

188 collection: TaskCollection, 

189) -> Generator["Task", None, None]: 

190 """ 

191 Generates tasks from a collection, creating export logs as we go. 

192 

193 Used for database exports. 

194 

195 Args: 

196 collection: a :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection` 

197 

198 Yields: 

199 :class:`camcops_server.cc_modules.cc_task.Task` objects 

200 

201 """ # noqa 

202 for et in gen_exportedtasks(collection): 

203 yield et.task 

204 et.succeed() 

205 

206 

207# ============================================================================= 

208# ExportedTask class 

209# ============================================================================= 

210 

211 

212class ExportedTask(Base): 

213 """ 

214 Class representing an attempt to exported a task (as part of a 

215 :class:`ExportRun`) to a specific 

216 :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient`. 

217 """ 

218 

219 __tablename__ = "_exported_tasks" 

220 

221 id = Column( 

222 "id", 

223 BigInteger, 

224 primary_key=True, 

225 autoincrement=True, 

226 comment="Arbitrary primary key", 

227 ) 

228 recipient_id = Column( 

229 "recipient_id", 

230 BigInteger, 

231 ForeignKey(ExportRecipient.id), 

232 nullable=False, 

233 comment=f"FK to {ExportRecipient.__tablename__}.{ExportRecipient.id.name}", # noqa 

234 ) 

235 basetable = Column( 

236 "basetable", 

237 TableNameColType, 

238 nullable=False, 

239 index=True, 

240 comment="Base table of task concerned", 

241 ) 

242 task_server_pk = Column( 

243 "task_server_pk", 

244 Integer, 

245 nullable=False, 

246 index=True, 

247 comment="Server PK of task in basetable (_pk field)", 

248 ) 

249 start_at_utc = Column( 

250 "start_at_utc", 

251 DateTime, 

252 nullable=False, 

253 index=True, 

254 comment="Time export was started (UTC)", 

255 ) 

256 finish_at_utc = Column( 

257 "finish_at_utc", DateTime, comment="Time export was finished (UTC)" 

258 ) 

259 success = Column( 

260 "success", 

261 Boolean, 

262 default=False, 

263 nullable=False, 

264 comment="Task exported successfully?", 

265 ) 

266 failure_reasons = Column( 

267 "failure_reasons", StringListType, comment="Reasons for failure" 

268 ) 

269 cancelled = Column( 

270 "cancelled", 

271 Boolean, 

272 default=False, 

273 nullable=False, 

274 comment="Export subsequently cancelled/invalidated (may trigger resend)", # noqa 

275 ) 

276 cancelled_at_utc = Column( 

277 "cancelled_at_utc", 

278 DateTime, 

279 comment="Time export was cancelled at (UTC)", 

280 ) 

281 

282 recipient = relationship(ExportRecipient) 

283 

284 emails = relationship("ExportedTaskEmail") 

285 fhir_exports = relationship("ExportedTaskFhir") 

286 filegroups = relationship("ExportedTaskFileGroup") 

287 hl7_messages = relationship("ExportedTaskHL7Message") 

288 redcap_exports = relationship("ExportedTaskRedcap") 

289 

290 def __init__( 

291 self, 

292 recipient: ExportRecipient = None, 

293 task: "Task" = None, 

294 basetable: str = None, 

295 task_server_pk: int = None, 

296 *args, 

297 **kwargs, 

298 ) -> None: 

299 """ 

300 Can initialize with a task, or a basetable/task_server_pk combination. 

301 

302 Args: 

303 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient` 

304 task: a :class:`camcops_server.cc_modules.cc_task.Task` object 

305 basetable: base table name of the task 

306 task_server_pk: server PK of the task 

307 

308 (However, we must also support a no-parameter constructor, not least 

309 for our :func:`merge_db` function.) 

310 """ # noqa 

311 super().__init__(*args, **kwargs) 

312 self.recipient = recipient 

313 self.start_at_utc = get_now_utc_datetime() 

314 if task: 

315 assert ( 

316 not basetable 

317 ) and task_server_pk is None, ( 

318 "Task specified; mustn't specify basetable/task_server_pk" 

319 ) 

320 self.basetable = task.tablename 

321 self.task_server_pk = task.pk 

322 self._task = task 

323 else: 

324 self.basetable = basetable 

325 self.task_server_pk = task_server_pk 

326 self._task = None # type: Optional[Task] 

327 

328 @reconstructor 

329 def init_on_load(self) -> None: 

330 """ 

331 Called when SQLAlchemy recreates an object; see 

332 https://docs.sqlalchemy.org/en/latest/orm/constructors.html. 

333 """ 

334 self._task = None # type: Optional[Task] 

335 

336 @property 

337 def task(self) -> "Task": 

338 """ 

339 Returns the associated task. 

340 """ 

341 if self._task is None: 

342 dbsession = SqlASession.object_session(self) 

343 try: 

344 self._task = task_factory_no_security_checks( 

345 dbsession, self.basetable, self.task_server_pk 

346 ) 

347 except KeyError: 

348 log.warning( 

349 "Failed to retrieve task for basetable={!r}, " "PK={!r}", 

350 self.basetable, 

351 self.task_server_pk, 

352 ) 

353 self._task = None 

354 return self._task 

355 

356 def succeed(self) -> None: 

357 """ 

358 Register success. 

359 """ 

360 self.success = True 

361 self.finish() 

362 

363 def abort(self, msg: str) -> None: 

364 """ 

365 Record failure, and why. 

366 

367 (Called ``abort`` not ``fail`` because PyCharm has a bug relating to 

368 functions named ``fail``: 

369 https://stackoverflow.com/questions/21954959/pycharm-unreachable-code.) 

370 

371 Args: 

372 msg: why 

373 """ 

374 self.success = False 

375 log.error("Task export failed: {}", msg) 

376 self._add_failure_reason(msg) 

377 self.finish() 

378 

379 def _add_failure_reason(self, msg: str) -> None: 

380 """ 

381 Writes to our ``failure_reasons`` list in a way that (a) obviates the 

382 need to create an empty list via ``__init__()``, and (b) will 

383 definitely mark it as dirty, so it gets saved to the database. 

384 

385 See :class:`cardinal_pythonlib.sqlalchemy.list_types.StringListType`. 

386 

387 Args: 

388 msg: the message 

389 """ 

390 if self.failure_reasons is None: 

391 self.failure_reasons = [msg] 

392 else: 

393 # Do not use .append(); that won't mark the record as dirty. 

394 # Don't use "+="; similarly, that calls list.__iadd__(), not 

395 # InstrumentedAttribute.__set__(). 

396 # noinspection PyAugmentAssignment 

397 self.failure_reasons = self.failure_reasons + [msg] 

398 

399 def finish(self) -> None: 

400 """ 

401 Records the finish time. 

402 """ 

403 self.finish_at_utc = get_now_utc_datetime() 

404 

405 def export(self, req: "CamcopsRequest") -> None: 

406 """ 

407 Performs an export of the specific task. 

408 

409 Args: 

410 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

411 """ 

412 dbsession = req.dbsession 

413 recipient = self.recipient 

414 transmission_method = recipient.transmission_method 

415 log.info("Exporting task {!r} to recipient {}", self.task, recipient) 

416 

417 if transmission_method == ExportTransmissionMethod.EMAIL: 

418 email = ExportedTaskEmail(self) 

419 dbsession.add(email) 

420 email.export_task(req) 

421 

422 elif transmission_method == ExportTransmissionMethod.FHIR: 

423 efhir = ExportedTaskFhir(self) 

424 dbsession.add(efhir) 

425 dbsession.flush() 

426 efhir.export_task(req) 

427 

428 elif transmission_method == ExportTransmissionMethod.FILE: 

429 efg = ExportedTaskFileGroup(self) 

430 dbsession.add(efg) 

431 efg.export_task(req) 

432 

433 elif transmission_method == ExportTransmissionMethod.HL7: 

434 ehl7 = ExportedTaskHL7Message(self) 

435 if ehl7.valid(): 

436 dbsession.add(ehl7) 

437 ehl7.export_task(req) 

438 else: 

439 self.abort("Task not valid for HL7 export") 

440 

441 elif transmission_method == ExportTransmissionMethod.REDCAP: 

442 eredcap = ExportedTaskRedcap(self) 

443 dbsession.add(eredcap) 

444 eredcap.export_task(req) 

445 

446 else: 

447 raise AssertionError("Bug: bad transmission_method") 

448 

449 @property 

450 def filegroup(self) -> "ExportedTaskFileGroup": 

451 """ 

452 Returns a :class:`ExportedTaskFileGroup`, creating it if necessary. 

453 """ 

454 if self.filegroups: 

455 # noinspection PyUnresolvedReferences 

456 filegroup = self.filegroups[0] # type: ExportedTaskFileGroup 

457 else: 

458 filegroup = ExportedTaskFileGroup(self) 

459 # noinspection PyUnresolvedReferences 

460 self.filegroups.append(filegroup) 

461 return filegroup 

462 

463 def export_file( 

464 self, 

465 filename: str, 

466 text: str = None, 

467 binary: bytes = None, 

468 text_encoding: str = UTF8, 

469 ) -> bool: 

470 """ 

471 Exports a file. 

472 

473 Args: 

474 filename: 

475 text: text contents (specify this XOR ``binary``) 

476 binary: binary contents (specify this XOR ``text``) 

477 text_encoding: encoding to use when writing text 

478 

479 Returns: was it exported? 

480 """ 

481 filegroup = self.filegroup 

482 return filegroup.export_file( 

483 filename=filename, 

484 text=text, 

485 binary=binary, 

486 text_encoding=text_encoding, 

487 ) 

488 

489 def cancel(self) -> None: 

490 """ 

491 Marks the task export as cancelled/invalidated. 

492 

493 May trigger a resend (which is the point). 

494 """ 

495 self.cancelled = True 

496 self.cancelled_at_utc = get_now_utc_datetime() 

497 

498 @classmethod 

499 def task_already_exported( 

500 cls, 

501 dbsession: SqlASession, 

502 recipient_name: str, 

503 basetable: str, 

504 task_pk: int, 

505 ) -> bool: 

506 """ 

507 Has the specified task already been successfully exported? 

508 

509 Args: 

510 dbsession: a :class:`sqlalchemy.orm.session.Session` 

511 recipient_name: 

512 basetable: name of the task's base table 

513 task_pk: server PK of the task 

514 

515 Returns: 

516 does a successful export record exist for this task? 

517 

518 """ 

519 exists_q = ( 

520 dbsession.query(cls) 

521 .join(cls.recipient) 

522 .filter(ExportRecipient.recipient_name == recipient_name) 

523 .filter(cls.basetable == basetable) 

524 .filter(cls.task_server_pk == task_pk) 

525 .filter(cls.success == True) # noqa: E712 

526 .filter(cls.cancelled == False) # noqa: E712 

527 .exists() 

528 ) 

529 return bool_from_exists_clause(dbsession, exists_q) 

530 

531 

532# ============================================================================= 

533# HL7 export 

534# ============================================================================= 

535 

536 

537class ExportedTaskHL7Message(Base): 

538 """ 

539 Represents an individual HL7 message. 

540 """ 

541 

542 __tablename__ = "_exported_task_hl7msg" 

543 

544 id = Column( 

545 "id", 

546 BigInteger, 

547 primary_key=True, 

548 autoincrement=True, 

549 comment="Arbitrary primary key", 

550 ) 

551 exported_task_id = Column( 

552 "exported_task_id", 

553 BigInteger, 

554 ForeignKey(ExportedTask.id), 

555 nullable=False, 

556 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}", 

557 ) 

558 sent_at_utc = Column( 

559 "sent_at_utc", DateTime, comment="Time message was sent at (UTC)" 

560 ) 

561 reply_at_utc = Column( 

562 "reply_at_utc", DateTime, comment="Time message was replied to (UTC)" 

563 ) 

564 success = Column( 

565 "success", 

566 Boolean, 

567 comment="Message sent successfully and acknowledged by HL7 server", 

568 ) 

569 failure_reason = Column( 

570 "failure_reason", Text, comment="Reason for failure" 

571 ) 

572 message = Column("message", LongText, comment="Message body, if kept") 

573 reply = Column("reply", Text, comment="Server's reply, if kept") 

574 

575 exported_task = relationship(ExportedTask) 

576 

577 def __init__( 

578 self, exported_task: ExportedTask = None, *args, **kwargs 

579 ) -> None: 

580 """ 

581 Must support parameter-free construction, not least for 

582 :func:`merge_db`. 

583 """ 

584 super().__init__(*args, **kwargs) 

585 self.exported_task = exported_task 

586 

587 self._hl7_msg = None # type: Optional[hl7.Message] 

588 

589 @reconstructor 

590 def init_on_load(self) -> None: 

591 """ 

592 Called when SQLAlchemy recreates an object; see 

593 https://docs.sqlalchemy.org/en/latest/orm/constructors.html. 

594 """ 

595 self._hl7_msg = None 

596 

597 @staticmethod 

598 def task_acceptable_for_hl7( 

599 recipient: ExportRecipient, task: "Task" 

600 ) -> bool: 

601 """ 

602 Is the task valid for HL7 export. (For example, anonymous tasks and 

603 tasks missing key ID information may not be.) 

604 

605 Args: 

606 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient` 

607 task: a :class:`camcops_server.cc_modules.cc_task.Task` object 

608 

609 Returns: 

610 bool: valid? 

611 

612 """ # noqa 

613 if not task: 

614 return False 

615 if task.is_anonymous: 

616 return False # Cannot send anonymous tasks via HL7 

617 patient = task.patient 

618 if not patient: 

619 return False 

620 if not recipient.primary_idnum: 

621 return False # required for HL7 

622 if not patient.has_idnum_type(recipient.primary_idnum): 

623 return False 

624 return True 

625 

626 def valid(self) -> bool: 

627 """ 

628 Checks for internal validity; returns a bool. 

629 """ 

630 exported_task = self.exported_task 

631 task = exported_task.task 

632 recipient = exported_task.recipient 

633 return self.task_acceptable_for_hl7(recipient, task) 

634 

635 def succeed(self, now: Pendulum = None) -> None: 

636 """ 

637 Record that we succeeded, and so did our associated task export. 

638 """ 

639 now = now or get_now_utc_datetime() 

640 self.success = True 

641 self.sent_at_utc = now 

642 self.exported_task.succeed() 

643 

644 def abort(self, msg: str, diverted_not_sent: bool = False) -> None: 

645 """ 

646 Record that we failed, and so did our associated task export. 

647 

648 (Called ``abort`` not ``fail`` because PyCharm has a bug relating to 

649 functions named ``fail``: 

650 https://stackoverflow.com/questions/21954959/pycharm-unreachable-code.) 

651 

652 Args: 

653 msg: reason for failure 

654 diverted_not_sent: deliberately diverted (and not counted as sent) 

655 rather than a sending failure? 

656 """ 

657 self.success = False 

658 self.failure_reason = msg 

659 self.exported_task.abort( 

660 "HL7 message deliberately not sent; diverted to file" 

661 if diverted_not_sent 

662 else "HL7 sending failed" 

663 ) 

664 

665 def export_task(self, req: "CamcopsRequest") -> None: 

666 """ 

667 Exports the task itself to an HL7 message. 

668 

669 Args: 

670 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

671 """ 

672 if not self.valid(): 

673 self.abort( 

674 "Unsuitable for HL7; should have been filtered out earlier" 

675 ) 

676 return 

677 self.make_hl7_message(req) 

678 recipient = self.exported_task.recipient 

679 if recipient.hl7_debug_divert_to_file: 

680 self.divert_to_file(req) 

681 else: 

682 # Proper HL7 message 

683 self.transmit_hl7() 

684 

685 def divert_to_file(self, req: "CamcopsRequest") -> None: 

686 """ 

687 Write an HL7 message to a file. For debugging. 

688 

689 Args: 

690 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

691 """ 

692 exported_task = self.exported_task 

693 recipient = exported_task.recipient 

694 filename = recipient.get_filename( 

695 req, exported_task.task, override_task_format="hl7" 

696 ) 

697 now_utc = get_now_utc_pendulum() 

698 log.info("Diverting HL7 message to file {!r}", filename) 

699 written = exported_task.export_file( 

700 filename=filename, text=str(self._hl7_msg) 

701 ) 

702 if not written: 

703 return 

704 

705 if recipient.hl7_debug_treat_diverted_as_sent: 

706 self.sent_at_utc = now_utc 

707 self.succeed(now_utc) 

708 else: 

709 self.abort( 

710 "Exported to file as requested but not sent via HL7", 

711 diverted_not_sent=True, 

712 ) 

713 

714 def make_hl7_message(self, req: "CamcopsRequest") -> None: 

715 """ 

716 Makes an HL7 message and stores it in ``self._hl7_msg``. 

717 

718 May also store it in ``self.message`` (which is saved to the database), 

719 if we're saving HL7 messages. 

720 

721 See 

722 

723 - https://python-hl7.readthedocs.org/en/latest/index.html 

724 """ 

725 task = self.exported_task.task 

726 recipient = self.exported_task.recipient 

727 

728 # --------------------------------------------------------------------- 

729 # Parts 

730 # --------------------------------------------------------------------- 

731 msh_segment = make_msh_segment( 

732 message_datetime=req.now, message_control_id=str(self.id) 

733 ) 

734 pid_segment = task.get_patient_hl7_pid_segment(req, recipient) 

735 other_segments = task.get_hl7_data_segments(req, recipient) 

736 

737 # --------------------------------------------------------------------- 

738 # Whole message 

739 # --------------------------------------------------------------------- 

740 segments = [msh_segment, pid_segment] + other_segments 

741 self._hl7_msg = hl7.Message(SEGMENT_SEPARATOR, segments) 

742 if recipient.hl7_keep_message: 

743 self.message = str(self._hl7_msg) 

744 

745 def transmit_hl7(self) -> None: 

746 """ 

747 Sends the HL7 message over TCP/IP. 

748 

749 - Default MLLP/HL7 port is 2575 

750 - MLLP = minimum lower layer protocol 

751 

752 - https://www.cleo.com/support/byproduct/lexicom/usersguide/mllp_configuration.htm 

753 - https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=hl7 

754 - Essentially just a TCP socket with a minimal wrapper: 

755 https://stackoverflow.com/questions/11126918 

756 

757 - https://python-hl7.readthedocs.org/en/latest/api.html; however, 

758 we've modified that 

759 """ # noqa 

760 recipient = self.exported_task.recipient 

761 

762 if recipient.hl7_ping_first: 

763 pinged = self.ping_hl7_server(recipient) 

764 if not pinged: 

765 self.abort("Could not ping HL7 host") 

766 return 

767 

768 try: 

769 log.info( 

770 "Sending HL7 message to {}:{}", 

771 recipient.hl7_host, 

772 recipient.hl7_port, 

773 ) 

774 with MLLPTimeoutClient( 

775 recipient.hl7_host, 

776 recipient.hl7_port, 

777 recipient.hl7_network_timeout_ms, 

778 ) as client: 

779 server_replied, reply = client.send_message(self._hl7_msg) 

780 except socket.timeout: 

781 self.abort("Failed to send message via MLLP: timeout") 

782 return 

783 except Exception as e: 

784 self.abort(f"Failed to send message via MLLP: {e}") 

785 return 

786 

787 if not server_replied: 

788 self.abort("No response from server") 

789 return 

790 

791 self.reply_at_utc = get_now_utc_datetime() 

792 if recipient.hl7_keep_reply: 

793 self.reply = reply 

794 

795 try: 

796 replymsg = hl7.parse(reply) 

797 except Exception as e: 

798 self.abort(f"Malformed reply: {e}") 

799 return 

800 

801 success, failure_reason = msg_is_successful_ack(replymsg) 

802 if success: 

803 self.succeed() 

804 else: 

805 self.abort(failure_reason) 

806 

807 @staticmethod 

808 def ping_hl7_server(recipient: ExportRecipient) -> bool: 

809 # noinspection HttpUrlsUsage 

810 """ 

811 Performs a TCP/IP ping on our HL7 server; returns success. If we've 

812 already pinged successfully during this run, don't bother doing it 

813 again. 

814 

815 (No HL7 PING method yet. Proposal is 

816 http://hl7tsc.org/wiki/index.php?title=FTSD-ConCalls-20081028 

817 So use TCP/IP ping.) 

818 

819 Args: 

820 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient` 

821 

822 Returns: 

823 bool: success 

824 

825 """ # noqa 

826 timeout_s = min(recipient.hl7_network_timeout_ms // 1000, 1) 

827 if ping(hostname=recipient.hl7_host, timeout_s=timeout_s): 

828 return True 

829 else: 

830 log.error("Failed to ping {!r}", recipient.hl7_host) 

831 return False 

832 

833 

834# ============================================================================= 

835# File export 

836# ============================================================================= 

837 

838 

839class ExportedTaskFileGroup(Base): 

840 """ 

841 Represents a small set of files exported in relation to a single task. 

842 """ 

843 

844 __tablename__ = "_exported_task_filegroup" 

845 

846 id = Column( 

847 "id", 

848 BigInteger, 

849 primary_key=True, 

850 autoincrement=True, 

851 comment="Arbitrary primary key", 

852 ) 

853 exported_task_id = Column( 

854 "exported_task_id", 

855 BigInteger, 

856 ForeignKey(ExportedTask.id), 

857 nullable=False, 

858 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}", 

859 ) 

860 filenames = Column( 

861 "filenames", StringListType, comment="List of filenames exported" 

862 ) 

863 script_called = Column( 

864 "script_called", 

865 Boolean, 

866 default=False, 

867 nullable=False, 

868 comment=( 

869 f"Was the {ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} " 

870 f"script called?" 

871 ), 

872 ) 

873 script_retcode = Column( 

874 "script_retcode", 

875 Integer, 

876 comment=( 

877 f"Return code from the " 

878 f"{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} script" 

879 ), 

880 ) 

881 script_stdout = Column( 

882 "script_stdout", 

883 UnicodeText, 

884 comment=( 

885 f"stdout from the " 

886 f"{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} script" 

887 ), 

888 ) 

889 script_stderr = Column( 

890 "script_stderr", 

891 UnicodeText, 

892 comment=( 

893 f"stderr from the " 

894 f"{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} script" 

895 ), 

896 ) 

897 

898 exported_task = relationship(ExportedTask) 

899 

900 def __init__(self, exported_task: ExportedTask = None) -> None: 

901 """ 

902 Args: 

903 exported_task: :class:`ExportedTask` object 

904 """ 

905 self.exported_task = exported_task 

906 

907 def export_file( 

908 self, 

909 filename: str, 

910 text: str = None, 

911 binary: bytes = None, 

912 text_encoding: str = UTF8, 

913 ) -> False: 

914 """ 

915 Exports the file. 

916 

917 Args: 

918 filename: 

919 text: text contents (specify this XOR ``binary``) 

920 binary: binary contents (specify this XOR ``text``) 

921 text_encoding: encoding to use when writing text 

922 

923 Returns: 

924 bool: was it exported? 

925 """ 

926 assert bool(text) != bool(binary), "Specify text XOR binary" 

927 exported_task = self.exported_task 

928 filename = os.path.abspath(filename) 

929 directory = os.path.dirname(filename) 

930 recipient = exported_task.recipient 

931 

932 if not recipient.file_overwrite_files and os.path.isfile(filename): 

933 self.abort(f"File already exists: {filename!r}") 

934 return False 

935 

936 if recipient.file_make_directory: 

937 try: 

938 mkdir_p(directory) 

939 except Exception as e: 

940 self.abort(f"Couldn't make directory {directory!r}: {e}") 

941 return False 

942 

943 try: 

944 log.debug("Writing to {!r}", filename) 

945 if text: 

946 with open(filename, mode="w", encoding=text_encoding) as f: 

947 f.write(text) 

948 else: 

949 with open(filename, mode="wb") as f: 

950 f.write(binary) 

951 except Exception as e: 

952 self.abort(f"Failed to open or write file {filename!r}: {e}") 

953 return False 

954 

955 self.note_exported_file(filename) 

956 return True 

957 

958 def note_exported_file(self, *filenames: str) -> None: 

959 """ 

960 Records a filename that has been exported, or several. 

961 

962 Args: 

963 *filenames: filenames 

964 """ 

965 if self.filenames is None: 

966 self.filenames = list(filenames) 

967 else: 

968 # See ExportedTask._add_failure_reason() above: 

969 # noinspection PyAugmentAssignment,PyTypeChecker 

970 self.filenames = self.filenames + list(filenames) 

971 

972 def export_task(self, req: "CamcopsRequest") -> None: 

973 """ 

974 Exports the task itself to a file. 

975 

976 Args: 

977 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

978 """ 

979 exported_task = self.exported_task 

980 task = exported_task.task 

981 recipient = exported_task.recipient 

982 task_format = recipient.task_format 

983 task_filename = recipient.get_filename(req, task) 

984 rio_metadata_filename = change_filename_ext( 

985 task_filename, ".metadata" 

986 ).replace(" ", "") 

987 # ... in case we use it. No spaces in its filename. 

988 

989 # Before we calculate the PDF, etc., we can pre-check for existing 

990 # files. 

991 if not recipient.file_overwrite_files: 

992 target_filenames = [task_filename] 

993 if recipient.file_export_rio_metadata: 

994 target_filenames.append(rio_metadata_filename) 

995 for fname in target_filenames: 

996 if os.path.isfile(os.path.abspath(fname)): 

997 self.abort(f"File already exists: {fname!r}") 

998 return 

999 

1000 # Export task 

1001 if task_format == FileType.PDF: 

1002 binary = task.get_pdf(req) 

1003 text = None 

1004 elif task_format == FileType.HTML: 

1005 binary = None 

1006 text = task.get_html(req) 

1007 elif task_format == FileType.XML: 

1008 binary = None 

1009 text = task.get_xml(req) 

1010 else: 

1011 raise AssertionError("Unknown task_format") 

1012 written = self.export_file( 

1013 task_filename, text=text, binary=binary, text_encoding=UTF8 

1014 ) 

1015 if not written: 

1016 return 

1017 

1018 # RiO metadata too? 

1019 if recipient.file_export_rio_metadata: 

1020 

1021 metadata = task.get_rio_metadata( 

1022 req, 

1023 recipient.rio_idnum, 

1024 recipient.rio_uploading_user, 

1025 recipient.rio_document_type, 

1026 ) 

1027 # We're going to write in binary mode, to get the newlines right. 

1028 # One way is: 

1029 # with codecs.open(filename, mode="w", encoding="ascii") as f: 

1030 # f.write(metadata.replace("\n", DOS_NEWLINE)) 

1031 # Here's another. 

1032 metadata = metadata.replace("\n", DOS_NEWLINE) 

1033 # ... Servelec say CR = "\r", but DOS is \r\n. 

1034 metadata_binary = metadata.encode("ascii") 

1035 # UTF-8 is NOT supported by RiO for metadata. 

1036 written_metadata = self.export_file( 

1037 rio_metadata_filename, binary=metadata_binary 

1038 ) 

1039 if not written_metadata: 

1040 return 

1041 

1042 self.finish_run_script_if_necessary() 

1043 

1044 def succeed(self) -> None: 

1045 """ 

1046 Register success. 

1047 """ 

1048 self.exported_task.succeed() 

1049 

1050 def abort(self, msg: str) -> None: 

1051 """ 

1052 Record failure, and why. 

1053 

1054 (Called ``abort`` not ``fail`` because PyCharm has a bug relating to 

1055 functions named ``fail``: 

1056 https://stackoverflow.com/questions/21954959/pycharm-unreachable-code.) 

1057 

1058 Args: 

1059 msg: why 

1060 """ 

1061 self.exported_task.abort(msg) 

1062 

1063 def finish_run_script_if_necessary(self) -> None: 

1064 """ 

1065 Completes the file export by running the external script, if required. 

1066 """ 

1067 recipient = self.exported_task.recipient 

1068 if self.filenames and recipient.file_script_after_export: 

1069 args = [recipient.file_script_after_export] + self.filenames 

1070 try: 

1071 encoding = sys.getdefaultencoding() 

1072 p = subprocess.Popen( 

1073 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE 

1074 ) 

1075 out, err = p.communicate() 

1076 self.script_called = True 

1077 self.script_stdout = out.decode(encoding) 

1078 self.script_stderr = err.decode(encoding) 

1079 self.script_retcode = p.returncode 

1080 except Exception as e: 

1081 self.script_called = False 

1082 self.script_stdout = "" 

1083 self.script_stderr = str(e) 

1084 self.abort("Failed to run script") 

1085 return 

1086 self.succeed() 

1087 

1088 

1089# ============================================================================= 

1090# E-mail export 

1091# ============================================================================= 

1092 

1093 

1094class ExportedTaskEmail(Base): 

1095 """ 

1096 Represents an individual email export. 

1097 """ 

1098 

1099 __tablename__ = "_exported_task_email" 

1100 

1101 id = Column( 

1102 "id", 

1103 BigInteger, 

1104 primary_key=True, 

1105 autoincrement=True, 

1106 comment="Arbitrary primary key", 

1107 ) 

1108 exported_task_id = Column( 

1109 "exported_task_id", 

1110 BigInteger, 

1111 ForeignKey(ExportedTask.id), 

1112 nullable=False, 

1113 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}", 

1114 ) 

1115 email_id = Column( 

1116 "email_id", 

1117 BigInteger, 

1118 ForeignKey(Email.id), 

1119 comment=f"FK to {Email.__tablename__}.{Email.id.name}", 

1120 ) 

1121 

1122 exported_task = relationship(ExportedTask) 

1123 email = relationship(Email) 

1124 

1125 def __init__(self, exported_task: ExportedTask = None) -> None: 

1126 """ 

1127 Args: 

1128 exported_task: :class:`ExportedTask` object 

1129 """ 

1130 self.exported_task = exported_task 

1131 

1132 def export_task(self, req: "CamcopsRequest") -> None: 

1133 """ 

1134 Exports the task itself to an email. 

1135 

1136 Args: 

1137 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

1138 """ 

1139 exported_task = self.exported_task 

1140 task = exported_task.task 

1141 recipient = exported_task.recipient 

1142 task_format = recipient.task_format 

1143 task_filename = os.path.basename(recipient.get_filename(req, task)) 

1144 # ... we don't want a full path for e-mail! 

1145 encoding = "utf8" 

1146 

1147 # Export task 

1148 attachments = [] # type: List[Tuple[str, bytes]] 

1149 if task_format == FileType.PDF: 

1150 binary = task.get_pdf(req) 

1151 elif task_format == FileType.HTML: 

1152 binary = task.get_html(req).encode(encoding) 

1153 elif task_format == FileType.XML: 

1154 binary = task.get_xml(req).encode(encoding) 

1155 else: 

1156 raise AssertionError("Unknown task_format") 

1157 attachments.append((task_filename, binary)) 

1158 

1159 self.email = Email( 

1160 from_addr=recipient.email_from, 

1161 # date: automatic 

1162 sender=recipient.email_sender, 

1163 reply_to=recipient.email_reply_to, 

1164 to=recipient.email_to, 

1165 cc=recipient.email_cc, 

1166 bcc=recipient.email_bcc, 

1167 subject=recipient.get_email_subject(req, task), 

1168 body=recipient.get_email_body(req, task), 

1169 content_type=( 

1170 CONTENT_TYPE_HTML 

1171 if recipient.email_body_as_html 

1172 else CONTENT_TYPE_TEXT 

1173 ), 

1174 charset=encoding, 

1175 attachments_binary=attachments, 

1176 save_msg_string=recipient.email_keep_message, 

1177 ) 

1178 self.email.send( 

1179 host=recipient.email_host, 

1180 username=recipient.email_host_username, 

1181 password=recipient.email_host_password, 

1182 port=recipient.email_port, 

1183 use_tls=recipient.email_use_tls, 

1184 ) 

1185 if self.email.sent: 

1186 exported_task.succeed() 

1187 else: 

1188 exported_task.abort("Failed to send e-mail") 

1189 

1190 

1191# ============================================================================= 

1192# REDCap export 

1193# ============================================================================= 

1194 

1195 

1196class ExportedTaskRedcap(Base): 

1197 """ 

1198 Represents an individual REDCap export. 

1199 """ 

1200 

1201 __tablename__ = "_exported_task_redcap" 

1202 

1203 id = Column( 

1204 "id", 

1205 Integer, 

1206 primary_key=True, 

1207 autoincrement=True, 

1208 comment="Arbitrary primary key", 

1209 ) 

1210 exported_task_id = Column( 

1211 "exported_task_id", 

1212 BigInteger, 

1213 ForeignKey(ExportedTask.id), 

1214 nullable=False, 

1215 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}", 

1216 ) 

1217 

1218 exported_task = relationship(ExportedTask) 

1219 

1220 # We store these just as an audit trail 

1221 redcap_record_id = Column( 

1222 "redcap_record_id", 

1223 UnicodeText, 

1224 comment=( 

1225 "ID of the (patient) record on the REDCap instance where " 

1226 "this task has been exported" 

1227 ), 

1228 ) 

1229 

1230 redcap_instrument_name = Column( 

1231 "redcap_instrument_name", 

1232 UnicodeText, 

1233 comment=( 

1234 "The name of the REDCap instrument name (form) where this " 

1235 "task has been exported" 

1236 ), 

1237 ) 

1238 

1239 redcap_instance_id = Column( 

1240 "redcap_instance_id", 

1241 Integer, 

1242 comment=( 

1243 "1-based index of this particular task within the patient " 

1244 "record. Increments on every repeat attempt." 

1245 ), 

1246 ) 

1247 

1248 def __init__(self, exported_task: ExportedTask = None) -> None: 

1249 """ 

1250 Args: 

1251 exported_task: :class:`ExportedTask` object 

1252 """ 

1253 self.exported_task = exported_task 

1254 

1255 def export_task(self, req: "CamcopsRequest") -> None: 

1256 """ 

1257 Exports the task to REDCap. 

1258 

1259 Args: 

1260 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

1261 """ 

1262 exported_task = self.exported_task 

1263 exporter = RedcapTaskExporter() 

1264 

1265 try: 

1266 exporter.export_task(req, self) 

1267 exported_task.succeed() 

1268 except RedcapExportException as e: 

1269 exported_task.abort(str(e)) 

1270 

1271 

1272# ============================================================================= 

1273# FHIR export 

1274# ============================================================================= 

1275 

1276 

1277class ExportedTaskFhir(Base): 

1278 """ 

1279 Represents an individual FHIR export. 

1280 """ 

1281 

1282 __tablename__ = "_exported_task_fhir" 

1283 

1284 id = Column( 

1285 "id", 

1286 Integer, 

1287 primary_key=True, 

1288 autoincrement=True, 

1289 comment="Arbitrary primary key", 

1290 ) 

1291 

1292 exported_task_id = Column( 

1293 "exported_task_id", 

1294 BigInteger, 

1295 ForeignKey(ExportedTask.id), 

1296 nullable=False, 

1297 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}", 

1298 ) 

1299 

1300 exported_task = relationship(ExportedTask) 

1301 

1302 entries = relationship("ExportedTaskFhirEntry") 

1303 

1304 def __init__(self, exported_task: ExportedTask = None) -> None: 

1305 """ 

1306 Args: 

1307 exported_task: :class:`ExportedTask` object 

1308 """ 

1309 self.exported_task = exported_task 

1310 

1311 def export_task(self, req: "CamcopsRequest") -> None: 

1312 """ 

1313 Exports the task to FHIR. 

1314 

1315 Args: 

1316 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

1317 """ 

1318 exported_task = self.exported_task 

1319 

1320 try: 

1321 exporter = FhirTaskExporter(req, self) 

1322 exporter.export_task() 

1323 exported_task.succeed() 

1324 except FhirExportException as e: 

1325 exported_task.abort(str(e)) 

1326 

1327 

1328class ExportedTaskFhirEntry(Base): 

1329 """ 

1330 Details of Patients, Questionnaires, QuestionnaireResponses exported to 

1331 a FHIR server for a single task. 

1332 """ 

1333 

1334 __tablename__ = "_exported_task_fhir_entry" 

1335 

1336 id = Column( 

1337 "id", 

1338 Integer, 

1339 primary_key=True, 

1340 autoincrement=True, 

1341 comment="Arbitrary primary key", 

1342 ) 

1343 

1344 exported_task_fhir_id = Column( 

1345 "exported_task_fhir_id", 

1346 Integer, 

1347 ForeignKey(ExportedTaskFhir.id), 

1348 nullable=False, 

1349 comment="FK to {}.{}".format( 

1350 ExportedTaskFhir.__tablename__, ExportedTaskFhir.id.name 

1351 ), 

1352 ) 

1353 

1354 etag = Column( 

1355 "etag", UnicodeText, comment="The ETag for the resource (if relevant)" 

1356 ) 

1357 

1358 last_modified = Column( 

1359 "last_modified", DateTime, comment="Server's date/time modified." 

1360 ) 

1361 

1362 location = Column( 

1363 "location", 

1364 UnicodeText, 

1365 comment="The location (if the operation returns a location).", 

1366 ) 

1367 

1368 status = Column( 

1369 "status", UnicodeText, comment="Status response code (text optional)." 

1370 ) 

1371 

1372 # TODO: outcome? 

1373 

1374 exported_task_fhir = relationship(ExportedTaskFhir) 

1375 

1376 @property 

1377 def location_url(self) -> str: 

1378 """ 

1379 Puts the FHIR server API URL together with the returned location, so 

1380 we can hyperlink to the resource. 

1381 """ 

1382 if not self.location: 

1383 return "" 

1384 try: 

1385 api_url = ( 

1386 self.exported_task_fhir.exported_task.recipient.fhir_api_url 

1387 ) 

1388 except AttributeError: 

1389 return "" 

1390 # Avoid urllib.parse.urljoin; it does complex (and for our purposes 

1391 # wrong) things. See 

1392 # https://stackoverflow.com/questions/10893374/python-confusions-with-urljoin 

1393 return posixpath.join(api_url, self.location)