Coverage for cc_modules/cc_exportmodels.py: 33%
460 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
1#!/usr/bin/env python
3"""
4camcops_server/cc_modules/cc_exportmodels.py
6===============================================================================
8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry.
9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk).
11 This file is part of CamCOPS.
13 CamCOPS is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
18 CamCOPS is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
26===============================================================================
28**Define models for export functions (e.g. HL7, file-based export).**
30"""
32import logging
33import os
34import posixpath
35import socket
36import subprocess
37import sys
38from typing import Generator, List, Optional, Tuple, TYPE_CHECKING
40from cardinal_pythonlib.datetimefunc import (
41 get_now_utc_datetime,
42 get_now_utc_pendulum,
43)
44from cardinal_pythonlib.email.sendmail import (
45 CONTENT_TYPE_HTML,
46 CONTENT_TYPE_TEXT,
47)
48from cardinal_pythonlib.fileops import mkdir_p
49from cardinal_pythonlib.logs import BraceStyleAdapter
50from cardinal_pythonlib.network import ping
51from cardinal_pythonlib.sqlalchemy.list_types import StringListType
52from cardinal_pythonlib.sqlalchemy.orm_query import bool_from_exists_clause
53import hl7
54from pendulum import DateTime as Pendulum
55from sqlalchemy.orm import reconstructor, relationship, Session as SqlASession
56from sqlalchemy.sql.schema import Column, ForeignKey
57from sqlalchemy.sql.sqltypes import (
58 BigInteger,
59 Boolean,
60 DateTime,
61 Integer,
62 Text,
63 UnicodeText,
64)
66from camcops_server.cc_modules.cc_constants import (
67 ConfigParamExportRecipient,
68 FileType,
69 UTF8,
70)
71from camcops_server.cc_modules.cc_email import Email
72from camcops_server.cc_modules.cc_exportrecipient import ExportRecipient
73from camcops_server.cc_modules.cc_exportrecipientinfo import (
74 ExportTransmissionMethod,
75)
76from camcops_server.cc_modules.cc_fhir import (
77 FhirExportException,
78 FhirTaskExporter,
79)
80from camcops_server.cc_modules.cc_filename import change_filename_ext
81from camcops_server.cc_modules.cc_hl7 import (
82 make_msh_segment,
83 MLLPTimeoutClient,
84 msg_is_successful_ack,
85 SEGMENT_SEPARATOR,
86)
87from camcops_server.cc_modules.cc_redcap import (
88 RedcapExportException,
89 RedcapTaskExporter,
90)
91from camcops_server.cc_modules.cc_sqla_coltypes import (
92 LongText,
93 TableNameColType,
94)
95from camcops_server.cc_modules.cc_sqlalchemy import Base
96from camcops_server.cc_modules.cc_taskcollection import (
97 TaskCollection,
98 TaskSortMethod,
99)
100from camcops_server.cc_modules.cc_taskfactory import (
101 task_factory_no_security_checks,
102)
104if TYPE_CHECKING:
105 from camcops_server.cc_modules.cc_request import CamcopsRequest
106 from camcops_server.cc_modules.cc_task import Task
108log = BraceStyleAdapter(logging.getLogger(__name__))
111# =============================================================================
112# Constants
113# =============================================================================
115DOS_NEWLINE = "\r\n"
118# =============================================================================
119# Create task collections for export
120# =============================================================================
123def get_collection_for_export(
124 req: "CamcopsRequest",
125 recipient: ExportRecipient,
126 via_index: bool = True,
127 debug: bool = False,
128) -> TaskCollection:
129 """
130 Returns an appropriate task collection for this export recipient, namely
131 those tasks that are desired and (in the case of incremental exports)
132 haven't already been sent.
134 "Not already sent" means "not already sent to an export recipient with
135 the same name (even if other aspects of the export recipient have
136 changed)".
138 Args:
139 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
140 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient`
141 via_index: use the task index (faster)?
142 debug: report details?
144 Returns:
145 a :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection`
146 """ # noqa
147 if not via_index:
148 log.debug("Task index disabled for get_collection_for_export()")
149 collection = TaskCollection(
150 req=req,
151 sort_method_by_class=TaskSortMethod.CREATION_DATE_ASC,
152 current_only=True,
153 via_index=via_index,
154 export_recipient=recipient,
155 )
156 if debug:
157 log.debug(
158 "get_collection_for_export(): recipient={!r}, " "collection={!r}",
159 recipient,
160 collection,
161 )
162 return collection
165def gen_exportedtasks(
166 collection: TaskCollection,
167) -> Generator["ExportedTask", None, None]:
168 """
169 Generates task export entries from a collection.
171 Args:
172 collection: a :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection`
174 Yields:
175 :class:`ExportedTask` objects
177 """ # noqa
178 dbsession = collection.dbsession
179 recipient = collection.export_recipient
180 assert recipient is not None, "TaskCollection has no export_recipient"
181 for task in collection.gen_tasks_by_class():
182 et = ExportedTask(recipient, task)
183 dbsession.add(et)
184 yield et
187def gen_tasks_having_exportedtasks(
188 collection: TaskCollection,
189) -> Generator["Task", None, None]:
190 """
191 Generates tasks from a collection, creating export logs as we go.
193 Used for database exports.
195 Args:
196 collection: a :class:`camcops_server.cc_modules.cc_taskcollection.TaskCollection`
198 Yields:
199 :class:`camcops_server.cc_modules.cc_task.Task` objects
201 """ # noqa
202 for et in gen_exportedtasks(collection):
203 yield et.task
204 et.succeed()
207# =============================================================================
208# ExportedTask class
209# =============================================================================
212class ExportedTask(Base):
213 """
214 Class representing an attempt to exported a task (as part of a
215 :class:`ExportRun`) to a specific
216 :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient`.
217 """
219 __tablename__ = "_exported_tasks"
221 id = Column(
222 "id",
223 BigInteger,
224 primary_key=True,
225 autoincrement=True,
226 comment="Arbitrary primary key",
227 )
228 recipient_id = Column(
229 "recipient_id",
230 BigInteger,
231 ForeignKey(ExportRecipient.id),
232 nullable=False,
233 comment=f"FK to {ExportRecipient.__tablename__}.{ExportRecipient.id.name}", # noqa
234 )
235 basetable = Column(
236 "basetable",
237 TableNameColType,
238 nullable=False,
239 index=True,
240 comment="Base table of task concerned",
241 )
242 task_server_pk = Column(
243 "task_server_pk",
244 Integer,
245 nullable=False,
246 index=True,
247 comment="Server PK of task in basetable (_pk field)",
248 )
249 start_at_utc = Column(
250 "start_at_utc",
251 DateTime,
252 nullable=False,
253 index=True,
254 comment="Time export was started (UTC)",
255 )
256 finish_at_utc = Column(
257 "finish_at_utc", DateTime, comment="Time export was finished (UTC)"
258 )
259 success = Column(
260 "success",
261 Boolean,
262 default=False,
263 nullable=False,
264 comment="Task exported successfully?",
265 )
266 failure_reasons = Column(
267 "failure_reasons", StringListType, comment="Reasons for failure"
268 )
269 cancelled = Column(
270 "cancelled",
271 Boolean,
272 default=False,
273 nullable=False,
274 comment="Export subsequently cancelled/invalidated (may trigger resend)", # noqa
275 )
276 cancelled_at_utc = Column(
277 "cancelled_at_utc",
278 DateTime,
279 comment="Time export was cancelled at (UTC)",
280 )
282 recipient = relationship(ExportRecipient)
284 emails = relationship("ExportedTaskEmail")
285 fhir_exports = relationship("ExportedTaskFhir")
286 filegroups = relationship("ExportedTaskFileGroup")
287 hl7_messages = relationship("ExportedTaskHL7Message")
288 redcap_exports = relationship("ExportedTaskRedcap")
290 def __init__(
291 self,
292 recipient: ExportRecipient = None,
293 task: "Task" = None,
294 basetable: str = None,
295 task_server_pk: int = None,
296 *args,
297 **kwargs,
298 ) -> None:
299 """
300 Can initialize with a task, or a basetable/task_server_pk combination.
302 Args:
303 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient`
304 task: a :class:`camcops_server.cc_modules.cc_task.Task` object
305 basetable: base table name of the task
306 task_server_pk: server PK of the task
308 (However, we must also support a no-parameter constructor, not least
309 for our :func:`merge_db` function.)
310 """ # noqa
311 super().__init__(*args, **kwargs)
312 self.recipient = recipient
313 self.start_at_utc = get_now_utc_datetime()
314 if task:
315 assert (
316 not basetable
317 ) and task_server_pk is None, (
318 "Task specified; mustn't specify basetable/task_server_pk"
319 )
320 self.basetable = task.tablename
321 self.task_server_pk = task.pk
322 self._task = task
323 else:
324 self.basetable = basetable
325 self.task_server_pk = task_server_pk
326 self._task = None # type: Optional[Task]
328 @reconstructor
329 def init_on_load(self) -> None:
330 """
331 Called when SQLAlchemy recreates an object; see
332 https://docs.sqlalchemy.org/en/latest/orm/constructors.html.
333 """
334 self._task = None # type: Optional[Task]
336 @property
337 def task(self) -> "Task":
338 """
339 Returns the associated task.
340 """
341 if self._task is None:
342 dbsession = SqlASession.object_session(self)
343 try:
344 self._task = task_factory_no_security_checks(
345 dbsession, self.basetable, self.task_server_pk
346 )
347 except KeyError:
348 log.warning(
349 "Failed to retrieve task for basetable={!r}, " "PK={!r}",
350 self.basetable,
351 self.task_server_pk,
352 )
353 self._task = None
354 return self._task
356 def succeed(self) -> None:
357 """
358 Register success.
359 """
360 self.success = True
361 self.finish()
363 def abort(self, msg: str) -> None:
364 """
365 Record failure, and why.
367 (Called ``abort`` not ``fail`` because PyCharm has a bug relating to
368 functions named ``fail``:
369 https://stackoverflow.com/questions/21954959/pycharm-unreachable-code.)
371 Args:
372 msg: why
373 """
374 self.success = False
375 log.error("Task export failed: {}", msg)
376 self._add_failure_reason(msg)
377 self.finish()
379 def _add_failure_reason(self, msg: str) -> None:
380 """
381 Writes to our ``failure_reasons`` list in a way that (a) obviates the
382 need to create an empty list via ``__init__()``, and (b) will
383 definitely mark it as dirty, so it gets saved to the database.
385 See :class:`cardinal_pythonlib.sqlalchemy.list_types.StringListType`.
387 Args:
388 msg: the message
389 """
390 if self.failure_reasons is None:
391 self.failure_reasons = [msg]
392 else:
393 # Do not use .append(); that won't mark the record as dirty.
394 # Don't use "+="; similarly, that calls list.__iadd__(), not
395 # InstrumentedAttribute.__set__().
396 # noinspection PyAugmentAssignment
397 self.failure_reasons = self.failure_reasons + [msg]
399 def finish(self) -> None:
400 """
401 Records the finish time.
402 """
403 self.finish_at_utc = get_now_utc_datetime()
405 def export(self, req: "CamcopsRequest") -> None:
406 """
407 Performs an export of the specific task.
409 Args:
410 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
411 """
412 dbsession = req.dbsession
413 recipient = self.recipient
414 transmission_method = recipient.transmission_method
415 log.info("Exporting task {!r} to recipient {}", self.task, recipient)
417 if transmission_method == ExportTransmissionMethod.EMAIL:
418 email = ExportedTaskEmail(self)
419 dbsession.add(email)
420 email.export_task(req)
422 elif transmission_method == ExportTransmissionMethod.FHIR:
423 efhir = ExportedTaskFhir(self)
424 dbsession.add(efhir)
425 dbsession.flush()
426 efhir.export_task(req)
428 elif transmission_method == ExportTransmissionMethod.FILE:
429 efg = ExportedTaskFileGroup(self)
430 dbsession.add(efg)
431 efg.export_task(req)
433 elif transmission_method == ExportTransmissionMethod.HL7:
434 ehl7 = ExportedTaskHL7Message(self)
435 if ehl7.valid():
436 dbsession.add(ehl7)
437 ehl7.export_task(req)
438 else:
439 self.abort("Task not valid for HL7 export")
441 elif transmission_method == ExportTransmissionMethod.REDCAP:
442 eredcap = ExportedTaskRedcap(self)
443 dbsession.add(eredcap)
444 eredcap.export_task(req)
446 else:
447 raise AssertionError("Bug: bad transmission_method")
449 @property
450 def filegroup(self) -> "ExportedTaskFileGroup":
451 """
452 Returns a :class:`ExportedTaskFileGroup`, creating it if necessary.
453 """
454 if self.filegroups:
455 # noinspection PyUnresolvedReferences
456 filegroup = self.filegroups[0] # type: ExportedTaskFileGroup
457 else:
458 filegroup = ExportedTaskFileGroup(self)
459 # noinspection PyUnresolvedReferences
460 self.filegroups.append(filegroup)
461 return filegroup
463 def export_file(
464 self,
465 filename: str,
466 text: str = None,
467 binary: bytes = None,
468 text_encoding: str = UTF8,
469 ) -> bool:
470 """
471 Exports a file.
473 Args:
474 filename:
475 text: text contents (specify this XOR ``binary``)
476 binary: binary contents (specify this XOR ``text``)
477 text_encoding: encoding to use when writing text
479 Returns: was it exported?
480 """
481 filegroup = self.filegroup
482 return filegroup.export_file(
483 filename=filename,
484 text=text,
485 binary=binary,
486 text_encoding=text_encoding,
487 )
489 def cancel(self) -> None:
490 """
491 Marks the task export as cancelled/invalidated.
493 May trigger a resend (which is the point).
494 """
495 self.cancelled = True
496 self.cancelled_at_utc = get_now_utc_datetime()
498 @classmethod
499 def task_already_exported(
500 cls,
501 dbsession: SqlASession,
502 recipient_name: str,
503 basetable: str,
504 task_pk: int,
505 ) -> bool:
506 """
507 Has the specified task already been successfully exported?
509 Args:
510 dbsession: a :class:`sqlalchemy.orm.session.Session`
511 recipient_name:
512 basetable: name of the task's base table
513 task_pk: server PK of the task
515 Returns:
516 does a successful export record exist for this task?
518 """
519 exists_q = (
520 dbsession.query(cls)
521 .join(cls.recipient)
522 .filter(ExportRecipient.recipient_name == recipient_name)
523 .filter(cls.basetable == basetable)
524 .filter(cls.task_server_pk == task_pk)
525 .filter(cls.success == True) # noqa: E712
526 .filter(cls.cancelled == False) # noqa: E712
527 .exists()
528 )
529 return bool_from_exists_clause(dbsession, exists_q)
532# =============================================================================
533# HL7 export
534# =============================================================================
537class ExportedTaskHL7Message(Base):
538 """
539 Represents an individual HL7 message.
540 """
542 __tablename__ = "_exported_task_hl7msg"
544 id = Column(
545 "id",
546 BigInteger,
547 primary_key=True,
548 autoincrement=True,
549 comment="Arbitrary primary key",
550 )
551 exported_task_id = Column(
552 "exported_task_id",
553 BigInteger,
554 ForeignKey(ExportedTask.id),
555 nullable=False,
556 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}",
557 )
558 sent_at_utc = Column(
559 "sent_at_utc", DateTime, comment="Time message was sent at (UTC)"
560 )
561 reply_at_utc = Column(
562 "reply_at_utc", DateTime, comment="Time message was replied to (UTC)"
563 )
564 success = Column(
565 "success",
566 Boolean,
567 comment="Message sent successfully and acknowledged by HL7 server",
568 )
569 failure_reason = Column(
570 "failure_reason", Text, comment="Reason for failure"
571 )
572 message = Column("message", LongText, comment="Message body, if kept")
573 reply = Column("reply", Text, comment="Server's reply, if kept")
575 exported_task = relationship(ExportedTask)
577 def __init__(
578 self, exported_task: ExportedTask = None, *args, **kwargs
579 ) -> None:
580 """
581 Must support parameter-free construction, not least for
582 :func:`merge_db`.
583 """
584 super().__init__(*args, **kwargs)
585 self.exported_task = exported_task
587 self._hl7_msg = None # type: Optional[hl7.Message]
589 @reconstructor
590 def init_on_load(self) -> None:
591 """
592 Called when SQLAlchemy recreates an object; see
593 https://docs.sqlalchemy.org/en/latest/orm/constructors.html.
594 """
595 self._hl7_msg = None
597 @staticmethod
598 def task_acceptable_for_hl7(
599 recipient: ExportRecipient, task: "Task"
600 ) -> bool:
601 """
602 Is the task valid for HL7 export. (For example, anonymous tasks and
603 tasks missing key ID information may not be.)
605 Args:
606 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient`
607 task: a :class:`camcops_server.cc_modules.cc_task.Task` object
609 Returns:
610 bool: valid?
612 """ # noqa
613 if not task:
614 return False
615 if task.is_anonymous:
616 return False # Cannot send anonymous tasks via HL7
617 patient = task.patient
618 if not patient:
619 return False
620 if not recipient.primary_idnum:
621 return False # required for HL7
622 if not patient.has_idnum_type(recipient.primary_idnum):
623 return False
624 return True
626 def valid(self) -> bool:
627 """
628 Checks for internal validity; returns a bool.
629 """
630 exported_task = self.exported_task
631 task = exported_task.task
632 recipient = exported_task.recipient
633 return self.task_acceptable_for_hl7(recipient, task)
635 def succeed(self, now: Pendulum = None) -> None:
636 """
637 Record that we succeeded, and so did our associated task export.
638 """
639 now = now or get_now_utc_datetime()
640 self.success = True
641 self.sent_at_utc = now
642 self.exported_task.succeed()
644 def abort(self, msg: str, diverted_not_sent: bool = False) -> None:
645 """
646 Record that we failed, and so did our associated task export.
648 (Called ``abort`` not ``fail`` because PyCharm has a bug relating to
649 functions named ``fail``:
650 https://stackoverflow.com/questions/21954959/pycharm-unreachable-code.)
652 Args:
653 msg: reason for failure
654 diverted_not_sent: deliberately diverted (and not counted as sent)
655 rather than a sending failure?
656 """
657 self.success = False
658 self.failure_reason = msg
659 self.exported_task.abort(
660 "HL7 message deliberately not sent; diverted to file"
661 if diverted_not_sent
662 else "HL7 sending failed"
663 )
665 def export_task(self, req: "CamcopsRequest") -> None:
666 """
667 Exports the task itself to an HL7 message.
669 Args:
670 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
671 """
672 if not self.valid():
673 self.abort(
674 "Unsuitable for HL7; should have been filtered out earlier"
675 )
676 return
677 self.make_hl7_message(req)
678 recipient = self.exported_task.recipient
679 if recipient.hl7_debug_divert_to_file:
680 self.divert_to_file(req)
681 else:
682 # Proper HL7 message
683 self.transmit_hl7()
685 def divert_to_file(self, req: "CamcopsRequest") -> None:
686 """
687 Write an HL7 message to a file. For debugging.
689 Args:
690 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
691 """
692 exported_task = self.exported_task
693 recipient = exported_task.recipient
694 filename = recipient.get_filename(
695 req, exported_task.task, override_task_format="hl7"
696 )
697 now_utc = get_now_utc_pendulum()
698 log.info("Diverting HL7 message to file {!r}", filename)
699 written = exported_task.export_file(
700 filename=filename, text=str(self._hl7_msg)
701 )
702 if not written:
703 return
705 if recipient.hl7_debug_treat_diverted_as_sent:
706 self.sent_at_utc = now_utc
707 self.succeed(now_utc)
708 else:
709 self.abort(
710 "Exported to file as requested but not sent via HL7",
711 diverted_not_sent=True,
712 )
714 def make_hl7_message(self, req: "CamcopsRequest") -> None:
715 """
716 Makes an HL7 message and stores it in ``self._hl7_msg``.
718 May also store it in ``self.message`` (which is saved to the database),
719 if we're saving HL7 messages.
721 See
723 - https://python-hl7.readthedocs.org/en/latest/index.html
724 """
725 task = self.exported_task.task
726 recipient = self.exported_task.recipient
728 # ---------------------------------------------------------------------
729 # Parts
730 # ---------------------------------------------------------------------
731 msh_segment = make_msh_segment(
732 message_datetime=req.now, message_control_id=str(self.id)
733 )
734 pid_segment = task.get_patient_hl7_pid_segment(req, recipient)
735 other_segments = task.get_hl7_data_segments(req, recipient)
737 # ---------------------------------------------------------------------
738 # Whole message
739 # ---------------------------------------------------------------------
740 segments = [msh_segment, pid_segment] + other_segments
741 self._hl7_msg = hl7.Message(SEGMENT_SEPARATOR, segments)
742 if recipient.hl7_keep_message:
743 self.message = str(self._hl7_msg)
745 def transmit_hl7(self) -> None:
746 """
747 Sends the HL7 message over TCP/IP.
749 - Default MLLP/HL7 port is 2575
750 - MLLP = minimum lower layer protocol
752 - https://www.cleo.com/support/byproduct/lexicom/usersguide/mllp_configuration.htm
753 - https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=hl7
754 - Essentially just a TCP socket with a minimal wrapper:
755 https://stackoverflow.com/questions/11126918
757 - https://python-hl7.readthedocs.org/en/latest/api.html; however,
758 we've modified that
759 """ # noqa
760 recipient = self.exported_task.recipient
762 if recipient.hl7_ping_first:
763 pinged = self.ping_hl7_server(recipient)
764 if not pinged:
765 self.abort("Could not ping HL7 host")
766 return
768 try:
769 log.info(
770 "Sending HL7 message to {}:{}",
771 recipient.hl7_host,
772 recipient.hl7_port,
773 )
774 with MLLPTimeoutClient(
775 recipient.hl7_host,
776 recipient.hl7_port,
777 recipient.hl7_network_timeout_ms,
778 ) as client:
779 server_replied, reply = client.send_message(self._hl7_msg)
780 except socket.timeout:
781 self.abort("Failed to send message via MLLP: timeout")
782 return
783 except Exception as e:
784 self.abort(f"Failed to send message via MLLP: {e}")
785 return
787 if not server_replied:
788 self.abort("No response from server")
789 return
791 self.reply_at_utc = get_now_utc_datetime()
792 if recipient.hl7_keep_reply:
793 self.reply = reply
795 try:
796 replymsg = hl7.parse(reply)
797 except Exception as e:
798 self.abort(f"Malformed reply: {e}")
799 return
801 success, failure_reason = msg_is_successful_ack(replymsg)
802 if success:
803 self.succeed()
804 else:
805 self.abort(failure_reason)
807 @staticmethod
808 def ping_hl7_server(recipient: ExportRecipient) -> bool:
809 # noinspection HttpUrlsUsage
810 """
811 Performs a TCP/IP ping on our HL7 server; returns success. If we've
812 already pinged successfully during this run, don't bother doing it
813 again.
815 (No HL7 PING method yet. Proposal is
816 http://hl7tsc.org/wiki/index.php?title=FTSD-ConCalls-20081028
817 So use TCP/IP ping.)
819 Args:
820 recipient: an :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient`
822 Returns:
823 bool: success
825 """ # noqa
826 timeout_s = min(recipient.hl7_network_timeout_ms // 1000, 1)
827 if ping(hostname=recipient.hl7_host, timeout_s=timeout_s):
828 return True
829 else:
830 log.error("Failed to ping {!r}", recipient.hl7_host)
831 return False
834# =============================================================================
835# File export
836# =============================================================================
839class ExportedTaskFileGroup(Base):
840 """
841 Represents a small set of files exported in relation to a single task.
842 """
844 __tablename__ = "_exported_task_filegroup"
846 id = Column(
847 "id",
848 BigInteger,
849 primary_key=True,
850 autoincrement=True,
851 comment="Arbitrary primary key",
852 )
853 exported_task_id = Column(
854 "exported_task_id",
855 BigInteger,
856 ForeignKey(ExportedTask.id),
857 nullable=False,
858 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}",
859 )
860 filenames = Column(
861 "filenames", StringListType, comment="List of filenames exported"
862 )
863 script_called = Column(
864 "script_called",
865 Boolean,
866 default=False,
867 nullable=False,
868 comment=(
869 f"Was the {ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} "
870 f"script called?"
871 ),
872 )
873 script_retcode = Column(
874 "script_retcode",
875 Integer,
876 comment=(
877 f"Return code from the "
878 f"{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} script"
879 ),
880 )
881 script_stdout = Column(
882 "script_stdout",
883 UnicodeText,
884 comment=(
885 f"stdout from the "
886 f"{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} script"
887 ),
888 )
889 script_stderr = Column(
890 "script_stderr",
891 UnicodeText,
892 comment=(
893 f"stderr from the "
894 f"{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} script"
895 ),
896 )
898 exported_task = relationship(ExportedTask)
900 def __init__(self, exported_task: ExportedTask = None) -> None:
901 """
902 Args:
903 exported_task: :class:`ExportedTask` object
904 """
905 self.exported_task = exported_task
907 def export_file(
908 self,
909 filename: str,
910 text: str = None,
911 binary: bytes = None,
912 text_encoding: str = UTF8,
913 ) -> False:
914 """
915 Exports the file.
917 Args:
918 filename:
919 text: text contents (specify this XOR ``binary``)
920 binary: binary contents (specify this XOR ``text``)
921 text_encoding: encoding to use when writing text
923 Returns:
924 bool: was it exported?
925 """
926 assert bool(text) != bool(binary), "Specify text XOR binary"
927 exported_task = self.exported_task
928 filename = os.path.abspath(filename)
929 directory = os.path.dirname(filename)
930 recipient = exported_task.recipient
932 if not recipient.file_overwrite_files and os.path.isfile(filename):
933 self.abort(f"File already exists: {filename!r}")
934 return False
936 if recipient.file_make_directory:
937 try:
938 mkdir_p(directory)
939 except Exception as e:
940 self.abort(f"Couldn't make directory {directory!r}: {e}")
941 return False
943 try:
944 log.debug("Writing to {!r}", filename)
945 if text:
946 with open(filename, mode="w", encoding=text_encoding) as f:
947 f.write(text)
948 else:
949 with open(filename, mode="wb") as f:
950 f.write(binary)
951 except Exception as e:
952 self.abort(f"Failed to open or write file {filename!r}: {e}")
953 return False
955 self.note_exported_file(filename)
956 return True
958 def note_exported_file(self, *filenames: str) -> None:
959 """
960 Records a filename that has been exported, or several.
962 Args:
963 *filenames: filenames
964 """
965 if self.filenames is None:
966 self.filenames = list(filenames)
967 else:
968 # See ExportedTask._add_failure_reason() above:
969 # noinspection PyAugmentAssignment,PyTypeChecker
970 self.filenames = self.filenames + list(filenames)
972 def export_task(self, req: "CamcopsRequest") -> None:
973 """
974 Exports the task itself to a file.
976 Args:
977 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
978 """
979 exported_task = self.exported_task
980 task = exported_task.task
981 recipient = exported_task.recipient
982 task_format = recipient.task_format
983 task_filename = recipient.get_filename(req, task)
984 rio_metadata_filename = change_filename_ext(
985 task_filename, ".metadata"
986 ).replace(" ", "")
987 # ... in case we use it. No spaces in its filename.
989 # Before we calculate the PDF, etc., we can pre-check for existing
990 # files.
991 if not recipient.file_overwrite_files:
992 target_filenames = [task_filename]
993 if recipient.file_export_rio_metadata:
994 target_filenames.append(rio_metadata_filename)
995 for fname in target_filenames:
996 if os.path.isfile(os.path.abspath(fname)):
997 self.abort(f"File already exists: {fname!r}")
998 return
1000 # Export task
1001 if task_format == FileType.PDF:
1002 binary = task.get_pdf(req)
1003 text = None
1004 elif task_format == FileType.HTML:
1005 binary = None
1006 text = task.get_html(req)
1007 elif task_format == FileType.XML:
1008 binary = None
1009 text = task.get_xml(req)
1010 else:
1011 raise AssertionError("Unknown task_format")
1012 written = self.export_file(
1013 task_filename, text=text, binary=binary, text_encoding=UTF8
1014 )
1015 if not written:
1016 return
1018 # RiO metadata too?
1019 if recipient.file_export_rio_metadata:
1021 metadata = task.get_rio_metadata(
1022 req,
1023 recipient.rio_idnum,
1024 recipient.rio_uploading_user,
1025 recipient.rio_document_type,
1026 )
1027 # We're going to write in binary mode, to get the newlines right.
1028 # One way is:
1029 # with codecs.open(filename, mode="w", encoding="ascii") as f:
1030 # f.write(metadata.replace("\n", DOS_NEWLINE))
1031 # Here's another.
1032 metadata = metadata.replace("\n", DOS_NEWLINE)
1033 # ... Servelec say CR = "\r", but DOS is \r\n.
1034 metadata_binary = metadata.encode("ascii")
1035 # UTF-8 is NOT supported by RiO for metadata.
1036 written_metadata = self.export_file(
1037 rio_metadata_filename, binary=metadata_binary
1038 )
1039 if not written_metadata:
1040 return
1042 self.finish_run_script_if_necessary()
1044 def succeed(self) -> None:
1045 """
1046 Register success.
1047 """
1048 self.exported_task.succeed()
1050 def abort(self, msg: str) -> None:
1051 """
1052 Record failure, and why.
1054 (Called ``abort`` not ``fail`` because PyCharm has a bug relating to
1055 functions named ``fail``:
1056 https://stackoverflow.com/questions/21954959/pycharm-unreachable-code.)
1058 Args:
1059 msg: why
1060 """
1061 self.exported_task.abort(msg)
1063 def finish_run_script_if_necessary(self) -> None:
1064 """
1065 Completes the file export by running the external script, if required.
1066 """
1067 recipient = self.exported_task.recipient
1068 if self.filenames and recipient.file_script_after_export:
1069 args = [recipient.file_script_after_export] + self.filenames
1070 try:
1071 encoding = sys.getdefaultencoding()
1072 p = subprocess.Popen(
1073 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
1074 )
1075 out, err = p.communicate()
1076 self.script_called = True
1077 self.script_stdout = out.decode(encoding)
1078 self.script_stderr = err.decode(encoding)
1079 self.script_retcode = p.returncode
1080 except Exception as e:
1081 self.script_called = False
1082 self.script_stdout = ""
1083 self.script_stderr = str(e)
1084 self.abort("Failed to run script")
1085 return
1086 self.succeed()
1089# =============================================================================
1090# E-mail export
1091# =============================================================================
1094class ExportedTaskEmail(Base):
1095 """
1096 Represents an individual email export.
1097 """
1099 __tablename__ = "_exported_task_email"
1101 id = Column(
1102 "id",
1103 BigInteger,
1104 primary_key=True,
1105 autoincrement=True,
1106 comment="Arbitrary primary key",
1107 )
1108 exported_task_id = Column(
1109 "exported_task_id",
1110 BigInteger,
1111 ForeignKey(ExportedTask.id),
1112 nullable=False,
1113 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}",
1114 )
1115 email_id = Column(
1116 "email_id",
1117 BigInteger,
1118 ForeignKey(Email.id),
1119 comment=f"FK to {Email.__tablename__}.{Email.id.name}",
1120 )
1122 exported_task = relationship(ExportedTask)
1123 email = relationship(Email)
1125 def __init__(self, exported_task: ExportedTask = None) -> None:
1126 """
1127 Args:
1128 exported_task: :class:`ExportedTask` object
1129 """
1130 self.exported_task = exported_task
1132 def export_task(self, req: "CamcopsRequest") -> None:
1133 """
1134 Exports the task itself to an email.
1136 Args:
1137 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
1138 """
1139 exported_task = self.exported_task
1140 task = exported_task.task
1141 recipient = exported_task.recipient
1142 task_format = recipient.task_format
1143 task_filename = os.path.basename(recipient.get_filename(req, task))
1144 # ... we don't want a full path for e-mail!
1145 encoding = "utf8"
1147 # Export task
1148 attachments = [] # type: List[Tuple[str, bytes]]
1149 if task_format == FileType.PDF:
1150 binary = task.get_pdf(req)
1151 elif task_format == FileType.HTML:
1152 binary = task.get_html(req).encode(encoding)
1153 elif task_format == FileType.XML:
1154 binary = task.get_xml(req).encode(encoding)
1155 else:
1156 raise AssertionError("Unknown task_format")
1157 attachments.append((task_filename, binary))
1159 self.email = Email(
1160 from_addr=recipient.email_from,
1161 # date: automatic
1162 sender=recipient.email_sender,
1163 reply_to=recipient.email_reply_to,
1164 to=recipient.email_to,
1165 cc=recipient.email_cc,
1166 bcc=recipient.email_bcc,
1167 subject=recipient.get_email_subject(req, task),
1168 body=recipient.get_email_body(req, task),
1169 content_type=(
1170 CONTENT_TYPE_HTML
1171 if recipient.email_body_as_html
1172 else CONTENT_TYPE_TEXT
1173 ),
1174 charset=encoding,
1175 attachments_binary=attachments,
1176 save_msg_string=recipient.email_keep_message,
1177 )
1178 self.email.send(
1179 host=recipient.email_host,
1180 username=recipient.email_host_username,
1181 password=recipient.email_host_password,
1182 port=recipient.email_port,
1183 use_tls=recipient.email_use_tls,
1184 )
1185 if self.email.sent:
1186 exported_task.succeed()
1187 else:
1188 exported_task.abort("Failed to send e-mail")
1191# =============================================================================
1192# REDCap export
1193# =============================================================================
1196class ExportedTaskRedcap(Base):
1197 """
1198 Represents an individual REDCap export.
1199 """
1201 __tablename__ = "_exported_task_redcap"
1203 id = Column(
1204 "id",
1205 Integer,
1206 primary_key=True,
1207 autoincrement=True,
1208 comment="Arbitrary primary key",
1209 )
1210 exported_task_id = Column(
1211 "exported_task_id",
1212 BigInteger,
1213 ForeignKey(ExportedTask.id),
1214 nullable=False,
1215 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}",
1216 )
1218 exported_task = relationship(ExportedTask)
1220 # We store these just as an audit trail
1221 redcap_record_id = Column(
1222 "redcap_record_id",
1223 UnicodeText,
1224 comment=(
1225 "ID of the (patient) record on the REDCap instance where "
1226 "this task has been exported"
1227 ),
1228 )
1230 redcap_instrument_name = Column(
1231 "redcap_instrument_name",
1232 UnicodeText,
1233 comment=(
1234 "The name of the REDCap instrument name (form) where this "
1235 "task has been exported"
1236 ),
1237 )
1239 redcap_instance_id = Column(
1240 "redcap_instance_id",
1241 Integer,
1242 comment=(
1243 "1-based index of this particular task within the patient "
1244 "record. Increments on every repeat attempt."
1245 ),
1246 )
1248 def __init__(self, exported_task: ExportedTask = None) -> None:
1249 """
1250 Args:
1251 exported_task: :class:`ExportedTask` object
1252 """
1253 self.exported_task = exported_task
1255 def export_task(self, req: "CamcopsRequest") -> None:
1256 """
1257 Exports the task to REDCap.
1259 Args:
1260 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
1261 """
1262 exported_task = self.exported_task
1263 exporter = RedcapTaskExporter()
1265 try:
1266 exporter.export_task(req, self)
1267 exported_task.succeed()
1268 except RedcapExportException as e:
1269 exported_task.abort(str(e))
1272# =============================================================================
1273# FHIR export
1274# =============================================================================
1277class ExportedTaskFhir(Base):
1278 """
1279 Represents an individual FHIR export.
1280 """
1282 __tablename__ = "_exported_task_fhir"
1284 id = Column(
1285 "id",
1286 Integer,
1287 primary_key=True,
1288 autoincrement=True,
1289 comment="Arbitrary primary key",
1290 )
1292 exported_task_id = Column(
1293 "exported_task_id",
1294 BigInteger,
1295 ForeignKey(ExportedTask.id),
1296 nullable=False,
1297 comment=f"FK to {ExportedTask.__tablename__}.{ExportedTask.id.name}",
1298 )
1300 exported_task = relationship(ExportedTask)
1302 entries = relationship("ExportedTaskFhirEntry")
1304 def __init__(self, exported_task: ExportedTask = None) -> None:
1305 """
1306 Args:
1307 exported_task: :class:`ExportedTask` object
1308 """
1309 self.exported_task = exported_task
1311 def export_task(self, req: "CamcopsRequest") -> None:
1312 """
1313 Exports the task to FHIR.
1315 Args:
1316 req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
1317 """
1318 exported_task = self.exported_task
1320 try:
1321 exporter = FhirTaskExporter(req, self)
1322 exporter.export_task()
1323 exported_task.succeed()
1324 except FhirExportException as e:
1325 exported_task.abort(str(e))
1328class ExportedTaskFhirEntry(Base):
1329 """
1330 Details of Patients, Questionnaires, QuestionnaireResponses exported to
1331 a FHIR server for a single task.
1332 """
1334 __tablename__ = "_exported_task_fhir_entry"
1336 id = Column(
1337 "id",
1338 Integer,
1339 primary_key=True,
1340 autoincrement=True,
1341 comment="Arbitrary primary key",
1342 )
1344 exported_task_fhir_id = Column(
1345 "exported_task_fhir_id",
1346 Integer,
1347 ForeignKey(ExportedTaskFhir.id),
1348 nullable=False,
1349 comment="FK to {}.{}".format(
1350 ExportedTaskFhir.__tablename__, ExportedTaskFhir.id.name
1351 ),
1352 )
1354 etag = Column(
1355 "etag", UnicodeText, comment="The ETag for the resource (if relevant)"
1356 )
1358 last_modified = Column(
1359 "last_modified", DateTime, comment="Server's date/time modified."
1360 )
1362 location = Column(
1363 "location",
1364 UnicodeText,
1365 comment="The location (if the operation returns a location).",
1366 )
1368 status = Column(
1369 "status", UnicodeText, comment="Status response code (text optional)."
1370 )
1372 # TODO: outcome?
1374 exported_task_fhir = relationship(ExportedTaskFhir)
1376 @property
1377 def location_url(self) -> str:
1378 """
1379 Puts the FHIR server API URL together with the returned location, so
1380 we can hyperlink to the resource.
1381 """
1382 if not self.location:
1383 return ""
1384 try:
1385 api_url = (
1386 self.exported_task_fhir.exported_task.recipient.fhir_api_url
1387 )
1388 except AttributeError:
1389 return ""
1390 # Avoid urllib.parse.urljoin; it does complex (and for our purposes
1391 # wrong) things. See
1392 # https://stackoverflow.com/questions/10893374/python-confusions-with-urljoin
1393 return posixpath.join(api_url, self.location)