Coverage for cc_modules/cc_config.py: 63%
453 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
1#!/usr/bin/env python
3# noinspection HttpUrlsUsage
4"""
5camcops_server/cc_modules/cc_config.py
7===============================================================================
9 Copyright (C) 2012, University of Cambridge, Department of Psychiatry.
10 Created by Rudolf Cardinal (rnc1001@cam.ac.uk).
12 This file is part of CamCOPS.
14 CamCOPS is free software: you can redistribute it and/or modify
15 it under the terms of the GNU General Public License as published by
16 the Free Software Foundation, either version 3 of the License, or
17 (at your option) any later version.
19 CamCOPS is distributed in the hope that it will be useful,
20 but WITHOUT ANY WARRANTY; without even the implied warranty of
21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 GNU General Public License for more details.
24 You should have received a copy of the GNU General Public License
25 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
27===============================================================================
29**Read and represent a CamCOPS config file.**
31Also contains various types of demonstration config file (CamCOPS, but also
32``supervisord``, Apache, etc.) and demonstration helper scripts (e.g. MySQL).
34There are CONDITIONAL AND IN-FUNCTION IMPORTS HERE; see below. This is to
35minimize the number of modules loaded when this is used in the context of the
36client-side database script, rather than the webview.
38Moreover, it should not use SQLAlchemy objects directly; see ``celery.py``.
40In particular, I tried hard to use a "database-unaware" (unbound) SQLAlchemy
41ExportRecipient object. However, when the backend re-calls the config to get
42its recipients, we get errors like:
44.. code-block:: none
46 [2018-12-25 00:56:00,118: ERROR/ForkPoolWorker-7] Task camcops_server.cc_modules.celery_tasks.export_to_recipient_backend[ab2e2691-c2fa-4821-b8cd-2cbeb86ddc8f] raised unexpected: DetachedInstanceError('Instance <ExportRecipient at 0x7febbeeea7b8> is not bound to a Session; attribute refresh operation cannot proceed',)
47 Traceback (most recent call last):
48 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/app/trace.py", line 382, in trace_task
49 R = retval = fun(*args, **kwargs)
50 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/app/trace.py", line 641, in __protected_call__
51 return self.run(*args, **kwargs)
52 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/celery_tasks.py", line 103, in export_to_recipient_backend
53 schedule_via_backend=False)
54 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_export.py", line 255, in export
55 req, recipient_names=recipient_names, all_recipients=all_recipients)
56 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_config.py", line 1460, in get_export_recipients
57 valid_names = set(r.recipient_name for r in recipients)
58 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_config.py", line 1460, in <genexpr>
59 valid_names = set(r.recipient_name for r in recipients)
60 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 242, in __get__
61 return self.impl.get(instance_state(instance), dict_)
62 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 594, in get
63 value = state._load_expired(state, passive)
64 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/state.py", line 608, in _load_expired
65 self.manager.deferred_scalar_loader(self, toload)
66 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/loading.py", line 813, in load_scalar_attributes
67 (state_str(state)))
68 sqlalchemy.orm.exc.DetachedInstanceError: Instance <ExportRecipient at 0x7febbeeea7b8> is not bound to a Session; attribute refresh operation cannot proceed (Background on this error at: http://sqlalche.me/e/bhk3)
70""" # noqa
72import codecs
73import collections
74import configparser
75import contextlib
76import datetime
77import os
78import logging
79import re
80from subprocess import run, PIPE
81from typing import Any, Dict, Generator, List, Optional, Union
83from cardinal_pythonlib.classes import class_attribute_values
84from cardinal_pythonlib.configfiles import (
85 get_config_parameter,
86 get_config_parameter_boolean,
87 get_config_parameter_loglevel,
88 get_config_parameter_multiline,
89)
90from cardinal_pythonlib.docker import running_under_docker
91from cardinal_pythonlib.fileops import relative_filename_within_dir
92from cardinal_pythonlib.logs import BraceStyleAdapter
93from cardinal_pythonlib.randomness import create_base64encoded_randomness
94from cardinal_pythonlib.reprfunc import auto_repr
95from cardinal_pythonlib.sqlalchemy.alembic_func import (
96 get_current_and_head_revision,
97)
98from cardinal_pythonlib.sqlalchemy.engine_func import (
99 is_sqlserver,
100 is_sqlserver_2008_or_later,
101)
102from cardinal_pythonlib.sqlalchemy.logs import (
103 pre_disable_sqlalchemy_extra_echo_log,
104)
105from cardinal_pythonlib.sqlalchemy.schema import get_table_names
106from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_engine
107from cardinal_pythonlib.wsgi.reverse_proxied_mw import ReverseProxiedMiddleware
108import celery.schedules
109from sqlalchemy.engine import create_engine
110from sqlalchemy.engine.base import Engine
111from sqlalchemy.orm import sessionmaker
112from sqlalchemy.orm import Session as SqlASession
114from camcops_server.cc_modules.cc_baseconstants import (
115 ALEMBIC_BASE_DIR,
116 ALEMBIC_CONFIG_FILENAME,
117 ALEMBIC_VERSION_TABLE,
118 ENVVAR_CONFIG_FILE,
119 LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR,
120 ON_READTHEDOCS,
121)
122from camcops_server.cc_modules.cc_cache import cache_region_static, fkg
123from camcops_server.cc_modules.cc_constants import (
124 CONFIG_FILE_EXPORT_SECTION,
125 CONFIG_FILE_SERVER_SECTION,
126 CONFIG_FILE_SITE_SECTION,
127 CONFIG_FILE_SMS_BACKEND_PREFIX,
128 ConfigDefaults,
129 ConfigParamExportGeneral,
130 ConfigParamExportRecipient,
131 ConfigParamServer,
132 ConfigParamSite,
133 DockerConstants,
134 MfaMethod,
135 SmsBackendNames,
136)
137from camcops_server.cc_modules.cc_exportrecipientinfo import (
138 ExportRecipientInfo,
139)
140from camcops_server.cc_modules.cc_exception import raise_runtime_error
141from camcops_server.cc_modules.cc_filename import PatientSpecElementForFilename
142from camcops_server.cc_modules.cc_language import POSSIBLE_LOCALES
143from camcops_server.cc_modules.cc_pyramid import MASTER_ROUTE_CLIENT_API
144from camcops_server.cc_modules.cc_sms import (
145 get_sms_backend,
146 KapowSmsBackend,
147 TwilioSmsBackend,
148)
149from camcops_server.cc_modules.cc_snomed import (
150 get_all_task_snomed_concepts,
151 get_icd9_snomed_concepts_from_xml,
152 get_icd10_snomed_concepts_from_xml,
153 SnomedConcept,
154)
155from camcops_server.cc_modules.cc_validators import (
156 validate_export_recipient_name,
157 validate_group_name,
158)
159from camcops_server.cc_modules.cc_version_string import (
160 CAMCOPS_SERVER_VERSION_STRING,
161)
163log = BraceStyleAdapter(logging.getLogger(__name__))
165pre_disable_sqlalchemy_extra_echo_log()
168# =============================================================================
169# Constants
170# =============================================================================
172VALID_RECIPIENT_NAME_REGEX = r"^[\w_-]+$"
173# ... because we'll use them for filenames, amongst other things
174# https://stackoverflow.com/questions/10944438/
175# https://regexr.com/
177# Windows paths: irrelevant, as Windows doesn't run supervisord
178DEFAULT_LINUX_CAMCOPS_CONFIG = "/etc/camcops/camcops.conf"
179DEFAULT_LINUX_CAMCOPS_BASE_DIR = "/usr/share/camcops"
180DEFAULT_LINUX_CAMCOPS_VENV_DIR = os.path.join(
181 DEFAULT_LINUX_CAMCOPS_BASE_DIR, "venv"
182)
183DEFAULT_LINUX_CAMCOPS_VENV_BIN_DIR = os.path.join(
184 DEFAULT_LINUX_CAMCOPS_VENV_DIR, "bin"
185)
186DEFAULT_LINUX_CAMCOPS_EXECUTABLE = os.path.join(
187 DEFAULT_LINUX_CAMCOPS_VENV_BIN_DIR, "camcops_server"
188)
189DEFAULT_LINUX_CAMCOPS_STATIC_DIR = os.path.join(
190 DEFAULT_LINUX_CAMCOPS_VENV_DIR,
191 "lib",
192 "python3.8",
193 "site-packages",
194 "camcops_server",
195 "static",
196)
197DEFAULT_LINUX_LOGDIR = "/var/log/supervisor"
198DEFAULT_LINUX_USER = "www-data" # Ubuntu default
201# =============================================================================
202# Helper functions
203# =============================================================================
206def warn_if_not_within_docker_dir(
207 param_name: str,
208 filespec: str,
209 permit_cfg: bool = False,
210 permit_venv: bool = False,
211 permit_tmp: bool = False,
212 param_contains_not_is: bool = False,
213) -> None:
214 """
215 If the specified filename isn't within a relevant directory that will be
216 used by CamCOPS when operating within a Docker Compose application, warn
217 the user.
219 Args:
220 param_name:
221 Name of the parameter in the CamCOPS config file.
222 filespec:
223 Filename (or filename-like thing) to check.
224 permit_cfg:
225 Permit the file to be in the configuration directory.
226 permit_venv:
227 Permit the file to be in the virtual environment directory.
228 permit_tmp:
229 Permit the file to be in the shared temporary space.
230 param_contains_not_is:
231 The parameter "contains", not "is", the filename.
232 """
233 if not filespec:
234 return
235 is_phrase = "contains" if param_contains_not_is else "is"
236 permitted_dirs = [] # type: List[str]
237 if permit_cfg:
238 permitted_dirs.append(DockerConstants.CONFIG_DIR)
239 if permit_venv:
240 permitted_dirs.append(DockerConstants.VENV_DIR)
241 if permit_tmp:
242 permitted_dirs.append(DockerConstants.TMP_DIR)
243 ok = any(relative_filename_within_dir(filespec, d) for d in permitted_dirs)
244 if not ok:
245 log.warning(
246 f"Config parameter {param_name} {is_phrase} {filespec!r}, "
247 f"which is not within the permitted Docker directories "
248 f"{permitted_dirs!r}"
249 )
252def warn_if_not_docker_value(
253 param_name: str, actual_value: Any, required_value: Any
254) -> None:
255 """
256 Warn the user if a parameter does not match the specific value required
257 when operating under Docker.
259 Args:
260 param_name:
261 Name of the parameter in the CamCOPS config file.
262 actual_value:
263 Value in the config file.
264 required_value:
265 Value that should be used.
266 """
267 if actual_value != required_value:
268 log.warning(
269 f"Config parameter {param_name} is {actual_value!r}, "
270 f"but should be {required_value!r} when running inside "
271 f"Docker"
272 )
275def warn_if_not_present(param_name: str, value: Any) -> None:
276 """
277 Warn the user if a parameter is not set (None, or an empty string), for
278 when operating under Docker.
280 Args:
281 param_name:
282 Name of the parameter in the CamCOPS config file.
283 value:
284 Value in the config file.
285 """
286 if value is None or value == "":
287 log.warning(
288 f"Config parameter {param_name} is not specified, "
289 f"but should be specified when running inside Docker"
290 )
293def list_to_multiline_string(values: List[Any]) -> str:
294 """
295 Converts a Python list to a multiline string suitable for use as a config
296 file default (in a pretty way).
297 """
298 spacer = "\n "
299 gen_values = (str(x) for x in values)
300 if len(values) <= 1:
301 return spacer.join(gen_values)
302 else:
303 return spacer + spacer.join(gen_values)
306# =============================================================================
307# Demo config
308# =============================================================================
310# Cosmetic demonstration constants:
311DEFAULT_DB_READONLY_USER = "QQQ_USERNAME_REPLACE_ME"
312DEFAULT_DB_READONLY_PASSWORD = "PPP_PASSWORD_REPLACE_ME"
313DUMMY_INSTITUTION_URL = "https://www.mydomain/"
316def get_demo_config(for_docker: bool = False) -> str:
317 """
318 Returns a demonstration config file based on the specified parameters.
320 Args:
321 for_docker:
322 Adjust defaults for the Docker environment.
323 """
324 # ...
325 # http://www.debian.org/doc/debian-policy/ch-opersys.html#s-writing-init
326 # https://people.canonical.com/~cjwatson/ubuntu-policy/policy.html/ch-opersys.html # noqa
327 session_cookie_secret = create_base64encoded_randomness(num_bytes=64)
329 cd = ConfigDefaults(docker=for_docker)
330 return f"""
331# Demonstration CamCOPS server configuration file.
332#
333# Created by CamCOPS server version {CAMCOPS_SERVER_VERSION_STRING}.
334# See help at https://camcops.readthedocs.io/.
335#
336# Using defaults for Docker environment: {for_docker}
338# =============================================================================
339# CamCOPS site
340# =============================================================================
342[{CONFIG_FILE_SITE_SECTION}]
344# -----------------------------------------------------------------------------
345# Database connection
346# -----------------------------------------------------------------------------
348{ConfigParamSite.DB_URL} = {cd.demo_db_url}
349{ConfigParamSite.DB_ECHO} = {cd.DB_ECHO}
351# -----------------------------------------------------------------------------
352# URLs and paths
353# -----------------------------------------------------------------------------
355{ConfigParamSite.LOCAL_INSTITUTION_URL} = {DUMMY_INSTITUTION_URL}
356{ConfigParamSite.LOCAL_LOGO_FILE_ABSOLUTE} = {cd.LOCAL_LOGO_FILE_ABSOLUTE}
357{ConfigParamSite.CAMCOPS_LOGO_FILE_ABSOLUTE} = {cd.CAMCOPS_LOGO_FILE_ABSOLUTE}
359{ConfigParamSite.EXTRA_STRING_FILES} = {cd.EXTRA_STRING_FILES}
360{ConfigParamSite.RESTRICTED_TASKS} =
361{ConfigParamSite.LANGUAGE} = {cd.LANGUAGE}
363{ConfigParamSite.SNOMED_TASK_XML_FILENAME} =
364{ConfigParamSite.SNOMED_ICD9_XML_FILENAME} =
365{ConfigParamSite.SNOMED_ICD10_XML_FILENAME} =
367{ConfigParamSite.WKHTMLTOPDF_FILENAME} =
369# -----------------------------------------------------------------------------
370# Server geographical location
371# -----------------------------------------------------------------------------
373{ConfigParamSite.REGION_CODE} = {cd.REGION_CODE}
375# -----------------------------------------------------------------------------
376# Login and session configuration
377# -----------------------------------------------------------------------------
379{ConfigParamSite.MFA_METHODS} = {list_to_multiline_string(cd.MFA_METHODS)}
380{ConfigParamSite.MFA_TIMEOUT_S} = {cd.MFA_TIMEOUT_S}
381{ConfigParamSite.SESSION_COOKIE_SECRET} = camcops_autogenerated_secret_{session_cookie_secret}
382{ConfigParamSite.SESSION_TIMEOUT_MINUTES} = {cd.SESSION_TIMEOUT_MINUTES}
383{ConfigParamSite.SESSION_CHECK_USER_IP} = {cd.SESSION_CHECK_USER_IP}
384{ConfigParamSite.PASSWORD_CHANGE_FREQUENCY_DAYS} = {cd.PASSWORD_CHANGE_FREQUENCY_DAYS}
385{ConfigParamSite.LOCKOUT_THRESHOLD} = {cd.LOCKOUT_THRESHOLD}
386{ConfigParamSite.LOCKOUT_DURATION_INCREMENT_MINUTES} = {cd.LOCKOUT_DURATION_INCREMENT_MINUTES}
387{ConfigParamSite.DISABLE_PASSWORD_AUTOCOMPLETE} = {cd.DISABLE_PASSWORD_AUTOCOMPLETE}
389# -----------------------------------------------------------------------------
390# Suggested filenames for saving PDFs from the web view
391# -----------------------------------------------------------------------------
393{ConfigParamSite.PATIENT_SPEC_IF_ANONYMOUS} = {cd.PATIENT_SPEC_IF_ANONYMOUS}
394{ConfigParamSite.PATIENT_SPEC} = {{{PatientSpecElementForFilename.SURNAME}}}_{{{PatientSpecElementForFilename.FORENAME}}}_{{{PatientSpecElementForFilename.ALLIDNUMS}}}
396{ConfigParamSite.TASK_FILENAME_SPEC} = CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}}
397{ConfigParamSite.TRACKER_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_tracker.{{filetype}}
398{ConfigParamSite.CTV_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_clinicaltextview.{{filetype}}
400# -----------------------------------------------------------------------------
401# E-mail options
402# -----------------------------------------------------------------------------
404{ConfigParamSite.EMAIL_HOST} = mysmtpserver.mydomain
405{ConfigParamSite.EMAIL_PORT} = {cd.EMAIL_PORT}
406{ConfigParamSite.EMAIL_USE_TLS} = {cd.EMAIL_USE_TLS}
407{ConfigParamSite.EMAIL_HOST_USERNAME} = myusername
408{ConfigParamSite.EMAIL_HOST_PASSWORD} = mypassword
409{ConfigParamSite.EMAIL_FROM} = CamCOPS computer <noreply@myinstitution.mydomain>
410{ConfigParamSite.EMAIL_SENDER} =
411{ConfigParamSite.EMAIL_REPLY_TO} = CamCOPS clinical administrator <admin@myinstitution.mydomain>
413# -----------------------------------------------------------------------------
414# SMS options
415# -----------------------------------------------------------------------------
417{ConfigParamSite.SMS_BACKEND} = {cd.SMS_BACKEND}
419# -----------------------------------------------------------------------------
420# User download options
421# -----------------------------------------------------------------------------
423{ConfigParamSite.PERMIT_IMMEDIATE_DOWNLOADS} = {cd.PERMIT_IMMEDIATE_DOWNLOADS}
424{ConfigParamSite.USER_DOWNLOAD_DIR} = {cd.USER_DOWNLOAD_DIR}
425{ConfigParamSite.USER_DOWNLOAD_FILE_LIFETIME_MIN} = {cd.USER_DOWNLOAD_FILE_LIFETIME_MIN}
426{ConfigParamSite.USER_DOWNLOAD_MAX_SPACE_MB} = {cd.USER_DOWNLOAD_MAX_SPACE_MB}
428# -----------------------------------------------------------------------------
429# Debugging options
430# -----------------------------------------------------------------------------
432{ConfigParamSite.WEBVIEW_LOGLEVEL} = {cd.WEBVIEW_LOGLEVEL_TEXTFORMAT}
433{ConfigParamSite.CLIENT_API_LOGLEVEL} = {cd.CLIENT_API_LOGLEVEL_TEXTFORMAT}
434{ConfigParamSite.ALLOW_INSECURE_COOKIES} = {cd.ALLOW_INSECURE_COOKIES}
437# =============================================================================
438# Web server options
439# =============================================================================
441[{CONFIG_FILE_SERVER_SECTION}]
443# -----------------------------------------------------------------------------
444# Common web server options
445# -----------------------------------------------------------------------------
447{ConfigParamServer.HOST} = {cd.HOST}
448{ConfigParamServer.PORT} = {cd.PORT}
449{ConfigParamServer.UNIX_DOMAIN_SOCKET} =
451# If you host CamCOPS behind Apache, it’s likely that you’ll want Apache to
452# handle HTTPS and CamCOPS to operate unencrypted behind a reverse proxy, in
453# which case don’t set SSL_CERTIFICATE or SSL_PRIVATE_KEY.
454{ConfigParamServer.SSL_CERTIFICATE} = {cd.SSL_CERTIFICATE}
455{ConfigParamServer.SSL_PRIVATE_KEY} = {cd.SSL_PRIVATE_KEY}
456{ConfigParamServer.STATIC_CACHE_DURATION_S} = {cd.STATIC_CACHE_DURATION_S}
458# -----------------------------------------------------------------------------
459# WSGI options
460# -----------------------------------------------------------------------------
462{ConfigParamServer.DEBUG_REVERSE_PROXY} = {cd.DEBUG_REVERSE_PROXY}
463{ConfigParamServer.DEBUG_TOOLBAR} = {cd.DEBUG_TOOLBAR}
464{ConfigParamServer.SHOW_REQUESTS} = {cd.SHOW_REQUESTS}
465{ConfigParamServer.SHOW_REQUEST_IMMEDIATELY} = {cd.SHOW_REQUEST_IMMEDIATELY}
466{ConfigParamServer.SHOW_RESPONSE} = {cd.SHOW_RESPONSE}
467{ConfigParamServer.SHOW_TIMING} = {cd.SHOW_TIMING}
468{ConfigParamServer.PROXY_HTTP_HOST} =
469{ConfigParamServer.PROXY_REMOTE_ADDR} =
470{ConfigParamServer.PROXY_REWRITE_PATH_INFO} = {cd.PROXY_REWRITE_PATH_INFO}
471{ConfigParamServer.PROXY_SCRIPT_NAME} =
472{ConfigParamServer.PROXY_SERVER_NAME} =
473{ConfigParamServer.PROXY_SERVER_PORT} =
474{ConfigParamServer.PROXY_URL_SCHEME} =
475{ConfigParamServer.TRUSTED_PROXY_HEADERS} =
476 HTTP_X_FORWARDED_HOST
477 HTTP_X_FORWARDED_SERVER
478 HTTP_X_FORWARDED_PORT
479 HTTP_X_FORWARDED_PROTO
480 HTTP_X_FORWARDED_FOR
481 HTTP_X_SCRIPT_NAME
483# -----------------------------------------------------------------------------
484# Determining the externally accessible CamCOPS URL for back-end work
485# -----------------------------------------------------------------------------
487{ConfigParamServer.EXTERNAL_URL_SCHEME} =
488{ConfigParamServer.EXTERNAL_SERVER_NAME} =
489{ConfigParamServer.EXTERNAL_SERVER_PORT} =
490{ConfigParamServer.EXTERNAL_SCRIPT_NAME} =
492# -----------------------------------------------------------------------------
493# CherryPy options
494# -----------------------------------------------------------------------------
496{ConfigParamServer.CHERRYPY_SERVER_NAME} = {cd.CHERRYPY_SERVER_NAME}
497{ConfigParamServer.CHERRYPY_THREADS_START} = {cd.CHERRYPY_THREADS_START}
498{ConfigParamServer.CHERRYPY_THREADS_MAX} = {cd.CHERRYPY_THREADS_MAX}
499{ConfigParamServer.CHERRYPY_LOG_SCREEN} = {cd.CHERRYPY_LOG_SCREEN}
500{ConfigParamServer.CHERRYPY_ROOT_PATH} = {cd.CHERRYPY_ROOT_PATH}
502# -----------------------------------------------------------------------------
503# Gunicorn options
504# -----------------------------------------------------------------------------
506{ConfigParamServer.GUNICORN_NUM_WORKERS} = {cd.GUNICORN_NUM_WORKERS}
507{ConfigParamServer.GUNICORN_DEBUG_RELOAD} = {cd.GUNICORN_DEBUG_RELOAD}
508{ConfigParamServer.GUNICORN_TIMEOUT_S} = {cd.GUNICORN_TIMEOUT_S}
509{ConfigParamServer.DEBUG_SHOW_GUNICORN_OPTIONS} = {cd.DEBUG_SHOW_GUNICORN_OPTIONS}
512# =============================================================================
513# Export options
514# =============================================================================
516[{CONFIG_FILE_EXPORT_SECTION}]
518{ConfigParamExportGeneral.CELERY_BEAT_EXTRA_ARGS} =
519{ConfigParamExportGeneral.CELERY_BEAT_SCHEDULE_DATABASE} = {cd.CELERY_BEAT_SCHEDULE_DATABASE}
520{ConfigParamExportGeneral.CELERY_BROKER_URL} = {cd.CELERY_BROKER_URL}
521{ConfigParamExportGeneral.CELERY_WORKER_EXTRA_ARGS} =
522 --max-tasks-per-child=1000
523{ConfigParamExportGeneral.CELERY_EXPORT_TASK_RATE_LIMIT} = 100/m
524{ConfigParamExportGeneral.EXPORT_LOCKDIR} = {cd.EXPORT_LOCKDIR}
526{ConfigParamExportGeneral.RECIPIENTS} =
528{ConfigParamExportGeneral.SCHEDULE_TIMEZONE} = {cd.SCHEDULE_TIMEZONE}
529{ConfigParamExportGeneral.SCHEDULE} =
532# =============================================================================
533# Details for each export recipient
534# =============================================================================
536# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
537# Example recipient
538# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
539 # Example (disabled because it's not in the {ConfigParamExportGeneral.RECIPIENTS} list above)
541[recipient:recipient_A]
543 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
544 # How to export
545 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
547{ConfigParamExportRecipient.TRANSMISSION_METHOD} = hl7
548{ConfigParamExportRecipient.PUSH} = true
549{ConfigParamExportRecipient.TASK_FORMAT} = pdf
550{ConfigParamExportRecipient.XML_FIELD_COMMENTS} = {cd.XML_FIELD_COMMENTS}
552 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
553 # What to export
554 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
556{ConfigParamExportRecipient.ALL_GROUPS} = false
557{ConfigParamExportRecipient.GROUPS} =
558 myfirstgroup
559 mysecondgroup
560{ConfigParamExportRecipient.TASKS} =
562{ConfigParamExportRecipient.START_DATETIME_UTC} =
563{ConfigParamExportRecipient.END_DATETIME_UTC} =
564{ConfigParamExportRecipient.FINALIZED_ONLY} = {cd.FINALIZED_ONLY}
565{ConfigParamExportRecipient.INCLUDE_ANONYMOUS} = {cd.INCLUDE_ANONYMOUS}
566{ConfigParamExportRecipient.PRIMARY_IDNUM} = 1
567{ConfigParamExportRecipient.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY} = {cd.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY}
569 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
570 # Options applicable to database exports
571 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
573{ConfigParamExportRecipient.DB_URL} = some_sqlalchemy_url
574{ConfigParamExportRecipient.DB_ECHO} = {cd.DB_ECHO}
575{ConfigParamExportRecipient.DB_INCLUDE_BLOBS} = {cd.DB_INCLUDE_BLOBS}
576{ConfigParamExportRecipient.DB_ADD_SUMMARIES} = {cd.DB_ADD_SUMMARIES}
577{ConfigParamExportRecipient.DB_PATIENT_ID_PER_ROW} = {cd.DB_PATIENT_ID_PER_ROW}
579 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
580 # Options applicable to e-mail exports
581 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
583{ConfigParamExportRecipient.EMAIL_TO} =
584 Perinatal Psychiatry Admin <perinatal@myinstitution.mydomain>
586{ConfigParamExportRecipient.EMAIL_CC} =
587 Dr Alice Bradford <alice.bradford@myinstitution.mydomain>
588 Dr Charles Dogfoot <charles.dogfoot@myinstitution.mydomain>
590{ConfigParamExportRecipient.EMAIL_BCC} =
591 superuser <root@myinstitution.mydomain>
593{ConfigParamExportRecipient.EMAIL_PATIENT_SPEC_IF_ANONYMOUS} = anonymous
594{ConfigParamExportRecipient.EMAIL_PATIENT_SPEC} = {{{PatientSpecElementForFilename.SURNAME}}}, {{{PatientSpecElementForFilename.FORENAME}}}, {{{PatientSpecElementForFilename.ALLIDNUMS}}}
595{ConfigParamExportRecipient.EMAIL_SUBJECT} = CamCOPS task for {{patient}}, created {{created}}: {{tasktype}}, PK {{serverpk}}
596{ConfigParamExportRecipient.EMAIL_BODY_IS_HTML} = false
597{ConfigParamExportRecipient.EMAIL_BODY} =
598 Please find attached a new CamCOPS task for manual filing to the electronic
599 patient record of
601 {{patient}}
603 Task type: {{tasktype}}
604 Created: {{created}}
605 CamCOPS server primary key: {{serverpk}}
607 Yours faithfully,
609 The CamCOPS computer.
611{ConfigParamExportRecipient.EMAIL_KEEP_MESSAGE} = {cd.HL7_KEEP_MESSAGE}
613 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
614 # Options applicable to FHIR
615 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
617{ConfigParamExportRecipient.FHIR_API_URL} = https://my.fhir.server/api
618{ConfigParamExportRecipient.FHIR_APP_ID} = {cd.FHIR_APP_ID}
619{ConfigParamExportRecipient.FHIR_APP_SECRET} = my_fhir_secret_abc
620{ConfigParamExportRecipient.FHIR_LAUNCH_TOKEN} =
621{ConfigParamExportRecipient.FHIR_CONCURRENT} =
623 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
624 # Options applicable to HL7 (v2) exports
625 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
627{ConfigParamExportRecipient.HL7_HOST} = myhl7server.mydomain
628{ConfigParamExportRecipient.HL7_PORT} = {cd.HL7_PORT}
629{ConfigParamExportRecipient.HL7_PING_FIRST} = {cd.HL7_PING_FIRST}
630{ConfigParamExportRecipient.HL7_NETWORK_TIMEOUT_MS} = {cd.HL7_NETWORK_TIMEOUT_MS}
631{ConfigParamExportRecipient.HL7_KEEP_MESSAGE} = {cd.HL7_KEEP_MESSAGE}
632{ConfigParamExportRecipient.HL7_KEEP_REPLY} = {cd.HL7_KEEP_REPLY}
633{ConfigParamExportRecipient.HL7_DEBUG_DIVERT_TO_FILE} = {cd.HL7_DEBUG_DIVERT_TO_FILE}
634{ConfigParamExportRecipient.HL7_DEBUG_TREAT_DIVERTED_AS_SENT} = {cd.HL7_DEBUG_TREAT_DIVERTED_AS_SENT}
636 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
637 # Options applicable to file transfers/attachments
638 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
640{ConfigParamExportRecipient.FILE_PATIENT_SPEC} = {{surname}}_{{forename}}_{{idshortdesc1}}{{idnum1}}
641{ConfigParamExportRecipient.FILE_PATIENT_SPEC_IF_ANONYMOUS} = {cd.FILE_PATIENT_SPEC_IF_ANONYMOUS}
642{ConfigParamExportRecipient.FILE_FILENAME_SPEC} = /my_nfs_mount/mypath/CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}}
643{ConfigParamExportRecipient.FILE_MAKE_DIRECTORY} = {cd.FILE_MAKE_DIRECTORY}
644{ConfigParamExportRecipient.FILE_OVERWRITE_FILES} = {cd.FILE_OVERWRITE_FILES}
645{ConfigParamExportRecipient.FILE_EXPORT_RIO_METADATA} = {cd.FILE_EXPORT_RIO_METADATA}
646{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} =
648 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
649 # Extra options for RiO metadata for file-based export
650 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
652{ConfigParamExportRecipient.RIO_IDNUM} = 2
653{ConfigParamExportRecipient.RIO_UPLOADING_USER} = CamCOPS
654{ConfigParamExportRecipient.RIO_DOCUMENT_TYPE} = CC
656 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
657 # Extra options for REDCap export
658 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
660{ConfigParamExportRecipient.REDCAP_API_URL} = https://domain.of.redcap.server/api/
661{ConfigParamExportRecipient.REDCAP_API_KEY} = myapikey
662{ConfigParamExportRecipient.REDCAP_FIELDMAP_FILENAME} = /location/of/fieldmap.xml
664# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
665# Example SMS Backends. No configuration needed for '{SmsBackendNames.CONSOLE}' (testing only).
666# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
668[{CONFIG_FILE_SMS_BACKEND_PREFIX}:{SmsBackendNames.KAPOW}]
670{KapowSmsBackend.PARAM_USERNAME} = myusername
671{KapowSmsBackend.PARAM_PASSWORD} = mypassword
673[{CONFIG_FILE_SMS_BACKEND_PREFIX}:{SmsBackendNames.TWILIO}]
675{TwilioSmsBackend.PARAM_SID.upper()} = mysid
676{TwilioSmsBackend.PARAM_TOKEN.upper()} = mytoken
677{TwilioSmsBackend.PARAM_FROM_PHONE_NUMBER.upper()} = myphonenumber
679 """.strip() # noqa
682# =============================================================================
683# Demo configuration files, other than the CamCOPS config file itself
684# =============================================================================
686DEFAULT_SOCKET_FILENAME = "/run/camcops/camcops.socket"
689def get_demo_supervisor_config() -> str:
690 """
691 Returns a demonstration ``supervisord`` config file based on the
692 specified parameters.
693 """
694 redirect_stderr = "true"
695 autostart = "true"
696 autorestart = "true"
697 startsecs = "30"
698 stopwaitsecs = "60"
699 return f"""
700# =============================================================================
701# Demonstration 'supervisor' (supervisord) config file for CamCOPS.
702# Created by CamCOPS version {CAMCOPS_SERVER_VERSION_STRING}.
703# =============================================================================
704# See https://camcops.readthedocs.io/en/latest/administrator/server_configuration.html#start-camcops
706[program:camcops_server]
708command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} serve_gunicorn
709 --config {DEFAULT_LINUX_CAMCOPS_CONFIG}
711directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR}
712environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"
713user = {DEFAULT_LINUX_USER}
714stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_server.log
715redirect_stderr = {redirect_stderr}
716autostart = {autostart}
717autorestart = {autorestart}
718startsecs = {startsecs}
719stopwaitsecs = {stopwaitsecs}
721[program:camcops_workers]
723command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} launch_workers
724 --config {DEFAULT_LINUX_CAMCOPS_CONFIG}
726directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR}
727environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"
728user = {DEFAULT_LINUX_USER}
729stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_workers.log
730redirect_stderr = {redirect_stderr}
731autostart = {autostart}
732autorestart = {autorestart}
733startsecs = {startsecs}
734stopwaitsecs = {stopwaitsecs}
736[program:camcops_scheduler]
738command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} launch_scheduler
739 --config {DEFAULT_LINUX_CAMCOPS_CONFIG}
741directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR}
742environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"
743user = {DEFAULT_LINUX_USER}
744stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_scheduler.log
745redirect_stderr = {redirect_stderr}
746autostart = {autostart}
747autorestart = {autorestart}
748startsecs = {startsecs}
749stopwaitsecs = {stopwaitsecs}
751[group:camcops]
753programs = camcops_server, camcops_workers, camcops_scheduler
755 """.strip() # noqa
758def get_demo_apache_config(
759 rootpath: str = "", # no slash
760 specimen_internal_port: int = None,
761 specimen_socket_file: str = DEFAULT_SOCKET_FILENAME,
762) -> str:
763 """
764 Returns a demo Apache HTTPD config file section applicable to CamCOPS.
765 """
766 cd = ConfigDefaults()
767 specimen_internal_port = specimen_internal_port or cd.PORT
768 indent_8 = " " * 8
770 if rootpath:
771 urlbase = f"/{rootpath}"
772 urlbaseslash = f"{urlbase}/"
773 api_path = f"{urlbase}{MASTER_ROUTE_CLIENT_API}"
774 trailing_slash_notes = f"""{indent_8}#
775 # - Don't specify trailing slashes for the ProxyPass and
776 # ProxyPassReverse directives.
777 # If you do, http://camcops.example.com{urlbase} will fail though
778 # http://camcops.example.com{urlbaseslash} will succeed.
779 #
780 # - An alternative fix is to enable mod_rewrite (e.g. sudo a2enmod
781 # rewrite), then add these commands:
782 #
783 # RewriteEngine on
784 # RewriteRule ^/{rootpath}$ {rootpath}/ [L,R=301]
785 #
786 # which will redirect requests without the trailing slash to a
787 # version with the trailing slash.
788 #"""
790 x_script_name = (
791 f"{indent_8}RequestHeader set X-Script-Name {urlbase}\n"
792 )
793 else:
794 urlbase = "/"
795 urlbaseslash = "/"
796 api_path = MASTER_ROUTE_CLIENT_API
797 trailing_slash_notes = " " * 8 + "#"
798 x_script_name = ""
800 # noinspection HttpUrlsUsage
801 return f"""
802# Demonstration Apache config file section for CamCOPS.
803# Created by CamCOPS version {CAMCOPS_SERVER_VERSION_STRING}.
804#
805# Under Ubuntu, the Apache config will be somewhere in /etc/apache2/
806# Under CentOS, the Apache config will be somewhere in /etc/httpd/
807#
808# This section should go within the <VirtualHost> directive for the secure
809# (SSL, HTTPS) part of the web site.
811<VirtualHost *:443>
812 # ...
814 # =========================================================================
815 # CamCOPS
816 # =========================================================================
817 # Apache operates on the principle that the first match wins. So, if we
818 # want to serve CamCOPS but then override some of its URLs to serve static
819 # files faster, we define the static stuff first.
821 # ---------------------------------------------------------------------
822 # 1. Serve static files
823 # ---------------------------------------------------------------------
824 # a) offer them at the appropriate URL
825 # b) provide permission
826 # c) disable ProxyPass for static files
828 # CHANGE THIS: aim the alias at your own institutional logo.
830 Alias {urlbaseslash}static/logo_local.png {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}/logo_local.png
832 # We move from more specific to less specific aliases; the first match
833 # takes precedence. (Apache will warn about conflicting aliases if
834 # specified in a wrong, less-to-more-specific, order.)
836 Alias {urlbaseslash}static/ {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}/
838 <Directory {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}>
839 Require all granted
841 # ... for old Apache versions (e.g. 2.2), use instead:
842 # Order allow,deny
843 # Allow from all
844 </Directory>
846 # Don't ProxyPass the static files; we'll serve them via Apache.
848 ProxyPassMatch ^{urlbaseslash}static/ !
850 # ---------------------------------------------------------------------
851 # 2. Proxy requests to the CamCOPS web server and back; allow access
852 # ---------------------------------------------------------------------
853 # ... either via an internal TCP/IP port (e.g. 1024 or higher, and NOT
854 # accessible to users);
855 # ... or, better, via a Unix socket, e.g. {specimen_socket_file}
856 #
857 # NOTES
858 #
859 # - When you ProxyPass {urlbase}, you should browse to (e.g.)
860 #
861 # https://camcops.example.com{urlbase}
862 #
863 # and point your tablet devices to
864 #
865 # https://camcops.example.com{api_path}
866{trailing_slash_notes}
867 # - Ensure that you put the CORRECT PROTOCOL (http, https) in the rules
868 # below.
869 #
870 # - For ProxyPass options, see https://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass
871 #
872 # - Include "retry=0" to stop Apache disabling the connection for
873 # while on failure.
874 # - Consider adding a "timeout=<seconds>" option if the back-end is
875 # slow and causing timeouts.
876 #
877 # - CamCOPS MUST BE TOLD about its location and protocol, because that
878 # information is critical for synthesizing URLs, but is stripped out
879 # by the reverse proxy system. There are two ways:
880 #
881 # (i) specifying headers or WSGI environment variables, such as
882 # the HTTP(S) headers X-Forwarded-Proto and X-Script-Name below
883 # (and telling CamCOPS to trust them via its
884 # TRUSTED_PROXY_HEADERS setting);
885 #
886 # (ii) specifying other options to "camcops_server", including
887 # PROXY_SCRIPT_NAME, PROXY_URL_SCHEME; see the help for the
888 # CamCOPS config.
889 #
890 # So:
891 #
892 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
893 # (a) Reverse proxy
894 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
895 #
896 # #####################################################################
897 # PORT METHOD
898 # #####################################################################
899 # Note the use of "http" (reflecting the backend), not https (like the
900 # front end).
902 # ProxyPass {urlbase} http://127.0.0.1:{specimen_internal_port} retry=0 timeout=300
903 # ProxyPassReverse {urlbase} http://127.0.0.1:{specimen_internal_port}
905 # #####################################################################
906 # UNIX SOCKET METHOD (Apache 2.4.9 and higher)
907 # #####################################################################
908 # This requires Apache 2.4.9, and passes after the '|' character a URL
909 # that determines the Host: value of the request; see
910 # ://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypass
911 #
912 # The general syntax is:
913 #
914 # ProxyPass /URL_USER_SEES unix:SOCKETFILE|PROTOCOL://HOST/EXTRA_URL_FOR_BACKEND retry=0
915 #
916 # Note that:
917 #
918 # - the protocol should be http, not https (Apache deals with the
919 # HTTPS part and passes HTTP on)
920 # - the EXTRA_URL_FOR_BACKEND needs to be (a) unique for each
921 # instance or Apache will use a single worker for multiple
922 # instances, and (b) blank for the backend's benefit. Since those
923 # two conflict when there's >1 instance, there's a problem.
924 # - Normally, HOST is given as localhost. It may be that this problem
925 # is solved by using a dummy unique value for HOST:
926 # https://bz.apache.org/bugzilla/show_bug.cgi?id=54101#c1
927 #
928 # If your Apache version is too old, you will get the error
929 #
930 # "AH00526: Syntax error on line 56 of /etc/apache2/sites-enabled/SOMETHING:
931 # ProxyPass URL must be absolute!"
932 #
933 # If you get this error:
934 #
935 # AH01146: Ignoring parameter 'retry=0' for worker 'unix:/tmp/.camcops_gunicorn.sock|https://localhost' because of worker sharing
936 # https://wiki.apache.org/httpd/ListOfErrors
937 #
938 # ... then your URLs are overlapping and should be redone or sorted;
939 # see http://httpd.apache.org/docs/2.4/mod/mod_proxy.html#workers
940 #
941 # The part that must be unique for each instance, with no part a
942 # leading substring of any other, is THIS_BIT in:
943 #
944 # ProxyPass /URL_USER_SEES unix:SOCKETFILE|http://localhost/THIS_BIT retry=0
945 #
946 # If you get an error like this:
947 #
948 # AH01144: No protocol handler was valid for the URL /SOMEWHERE. If you are using a DSO version of mod_proxy, make sure the proxy submodules are included in the configuration using LoadModule.
949 #
950 # Then do this:
951 #
952 # sudo a2enmod proxy proxy_http
953 # sudo apache2ctl restart
954 #
955 # If you get an error like this:
956 #
957 # ... [proxy_http:error] [pid 32747] (103)Software caused connection abort: [client 109.151.49.173:56898] AH01102: error reading status line from remote server httpd-UDS:0
958 # [proxy:error] [pid 32747] [client 109.151.49.173:56898] AH00898: Error reading from remote server returned by /camcops_bruhl/webview
959 #
960 # then check you are specifying http://, not https://, in the ProxyPass
961 #
962 # Other information sources:
963 #
964 # - https://emptyhammock.com/projects/info/pyweb/webconfig.html
966 ProxyPass {urlbase} unix:{specimen_socket_file}|http://dummy1 retry=0 timeout=300
967 ProxyPassReverse {urlbase} unix:{specimen_socket_file}|http://dummy1
969 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
970 # (b) Allow proxy over SSL.
971 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
972 # Without this, you will get errors like:
973 # ... SSL Proxy requested for wombat:443 but not enabled [Hint: SSLProxyEngine]
974 # ... failed to enable ssl support for 0.0.0.0:0 (httpd-UDS)
976 SSLProxyEngine on
978 <Location {urlbase}>
980 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
981 # (c) Allow access
982 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
984 Require all granted
986 # ... for old Apache versions (e.g. 2.2), use instead:
987 #
988 # Order allow,deny
989 # Allow from all
991 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
992 # (d) Tell the proxied application that we are using HTTPS, and
993 # where the application is installed
994 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
995 # ... https://stackoverflow.com/questions/16042647
996 #
997 # Enable mod_headers (e.g. "sudo a2enmod headers") and set:
999 RequestHeader set X-Forwarded-Proto https
1000{x_script_name}
1001 # ... then ensure the TRUSTED_PROXY_HEADERS setting in the CamCOPS
1002 # config file includes:
1003 #
1004 # HTTP_X_FORWARDED_HOST
1005 # HTTP_X_FORWARDED_SERVER
1006 # HTTP_X_FORWARDED_PORT
1007 # HTTP_X_FORWARDED_PROTO
1008 # HTTP_X_SCRIPT_NAME
1009 #
1010 # (X-Forwarded-For, X-Forwarded-Host, and X-Forwarded-Server are
1011 # supplied by Apache automatically.)
1013 </Location>
1015 #==========================================================================
1016 # SSL security (for HTTPS)
1017 #==========================================================================
1019 # You will also need to install your SSL certificate; see the
1020 # instructions that came with it. You get a certificate by creating a
1021 # certificate signing request (CSR). You enter some details about your
1022 # site, and a software tool makes (1) a private key, which you keep
1023 # utterly private, and (2) a CSR, which you send to a Certificate
1024 # Authority (CA) for signing. They send back a signed certificate, and
1025 # a chain of certificates leading from yours to a trusted root CA.
1026 #
1027 # You can create your own (a 'snake-oil' certificate), but your tablets
1028 # and browsers will not trust it, so this is a bad idea.
1029 #
1030 # Once you have your certificate: edit and uncomment these lines:
1032 # SSLEngine on
1034 # SSLCertificateKeyFile /etc/ssl/private/my.private.key
1036 # ... a private file that you made before creating the certificate
1037 # request, and NEVER GAVE TO ANYBODY, and NEVER WILL (or your
1038 # security is broken and you need a new certificate).
1040 # SSLCertificateFile /etc/ssl/certs/my.public.cert
1042 # ... signed and supplied to you by the certificate authority (CA),
1043 # from the public certificate you sent to them.
1045 # SSLCertificateChainFile /etc/ssl/certs/my-institution.ca-bundle
1047 # ... made from additional certificates in a chain, supplied to you by
1048 # the CA. For example, mine is univcam.ca-bundle, made with the
1049 # command:
1050 #
1051 # cat TERENASSLCA.crt UTNAddTrustServer_CA.crt AddTrustExternalCARoot.crt > univcam.ca-bundle
1053</VirtualHost>
1055 """.strip() # noqa
1058# =============================================================================
1059# Helper functions
1060# =============================================================================
1063def raise_missing(section: str, parameter: str) -> None:
1064 msg = (
1065 f"Config file: missing/blank parameter {parameter} "
1066 f"in section [{section}]"
1067 )
1068 raise_runtime_error(msg)
1071# =============================================================================
1072# CrontabEntry
1073# =============================================================================
1076class CrontabEntry(object):
1077 """
1078 Class to represent a ``crontab``-style entry.
1079 """
1081 def __init__(
1082 self,
1083 line: str = None,
1084 minute: Union[str, int, List[int]] = "*",
1085 hour: Union[str, int, List[int]] = "*",
1086 day_of_week: Union[str, int, List[int]] = "*",
1087 day_of_month: Union[str, int, List[int]] = "*",
1088 month_of_year: Union[str, int, List[int]] = "*",
1089 content: str = None,
1090 ) -> None:
1091 """
1092 Args:
1093 line:
1094 line of the form ``m h dow dom moy content content content``.
1095 minute:
1096 crontab "minute" entry
1097 hour:
1098 crontab "hour" entry
1099 day_of_week:
1100 crontab "day_of_week" entry
1101 day_of_month:
1102 crontab "day_of_month" entry
1103 month_of_year:
1104 crontab "month_of_year" entry
1105 content:
1106 crontab "thing to run" entry
1108 If ``line`` is specified, it is used. Otherwise, the components are
1109 used; the default for each of them is ``"*"``, meaning "all". Thus, for
1110 example, you can specify ``minute="*/5"`` and that is sufficient to
1111 mean "every 5 minutes".
1112 """
1113 has_line = line is not None
1114 has_components = bool(
1115 minute
1116 and hour
1117 and day_of_week
1118 and day_of_month
1119 and month_of_year
1120 and content
1121 )
1122 assert (
1123 has_line or has_components
1124 ), "Specify either a crontab line or all the time components"
1125 if has_line:
1126 line = line.split("#")[0].strip() # everything before a '#'
1127 components = line.split() # split on whitespace
1128 assert (
1129 len(components) >= 6
1130 ), "Must specify 5 time components and then contents"
1131 (
1132 minute,
1133 hour,
1134 day_of_week,
1135 day_of_month,
1136 month_of_year,
1137 ) = components[0:5]
1138 content = " ".join(components[5:])
1140 self.minute = minute
1141 self.hour = hour
1142 self.day_of_week = day_of_week
1143 self.day_of_month = day_of_month
1144 self.month_of_year = month_of_year
1145 self.content = content
1147 def __repr__(self) -> str:
1148 return auto_repr(self, sort_attrs=False)
1150 def __str__(self) -> str:
1151 return (
1152 f"{self.minute} {self.hour} {self.day_of_week} "
1153 f"{self.day_of_month} {self.month_of_year} {self.content}"
1154 )
1156 def get_celery_schedule(self) -> celery.schedules.crontab:
1157 """
1158 Returns the corresponding Celery schedule.
1160 Returns:
1161 a :class:`celery.schedules.crontab`
1163 Raises:
1164 :exc:`celery.schedules.ParseException` if the input can't be parsed
1165 """
1166 return celery.schedules.crontab(
1167 minute=self.minute,
1168 hour=self.hour,
1169 day_of_week=self.day_of_week,
1170 day_of_month=self.day_of_month,
1171 month_of_year=self.month_of_year,
1172 )
1175# =============================================================================
1176# Configuration class. (It gets cached on a per-process basis.)
1177# =============================================================================
1180class CamcopsConfig(object):
1181 """
1182 Class representing the CamCOPS configuration.
1183 """
1185 def __init__(self, config_filename: str, config_text: str = None) -> None:
1186 """
1187 Initialize by reading the config file.
1189 Args:
1190 config_filename:
1191 Filename of the config file (usual method)
1192 config_text:
1193 Text contents of the config file (alternative method for
1194 special circumstances); overrides ``config_filename``
1195 """
1197 def _get_str(
1198 section: str,
1199 paramname: str,
1200 default: str = None,
1201 replace_empty_strings_with_default: bool = True,
1202 ) -> Optional[str]:
1203 p = get_config_parameter(parser, section, paramname, str, default)
1204 if p == "" and replace_empty_strings_with_default:
1205 return default
1206 return p
1208 def _get_bool(section: str, paramname: str, default: bool) -> bool:
1209 return get_config_parameter_boolean(
1210 parser, section, paramname, default
1211 )
1213 def _get_int(
1214 section: str, paramname: str, default: int = None
1215 ) -> Optional[int]:
1216 return get_config_parameter(
1217 parser, section, paramname, int, default
1218 )
1220 def _get_multiline(section: str, paramname: str) -> List[str]:
1221 # http://stackoverflow.com/questions/335695/lists-in-configparser
1222 return get_config_parameter_multiline(
1223 parser, section, paramname, []
1224 )
1226 def _get_multiline_ignoring_comments(
1227 section: str, paramname: str
1228 ) -> List[str]:
1229 # Returns lines with any trailing comments removed, and any
1230 # comment-only lines removed.
1231 lines = _get_multiline(section, paramname)
1232 return list(
1233 filter(None, (x.split("#")[0].strip() for x in lines if x))
1234 )
1236 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1237 # Learn something about our environment
1238 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1239 self.running_under_docker = running_under_docker()
1241 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1242 # Open config file
1243 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1244 self.camcops_config_filename = config_filename
1245 parser = configparser.ConfigParser()
1247 if config_text:
1248 log.info("Reading config from supplied string")
1249 parser.read_string(config_text)
1250 else:
1251 if not config_filename:
1252 raise AssertionError(
1253 f"Environment variable {ENVVAR_CONFIG_FILE} not specified "
1254 f"(and no command-line alternative given)"
1255 )
1256 log.info("Reading from config file: {}", config_filename)
1257 with codecs.open(config_filename, "r", "utf8") as file:
1258 parser.read_file(file)
1260 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1261 # Main section (in alphabetical order)
1262 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1263 s = CONFIG_FILE_SITE_SECTION
1264 cs = ConfigParamSite
1265 cd = ConfigDefaults()
1267 self.allow_insecure_cookies = _get_bool(
1268 s, cs.ALLOW_INSECURE_COOKIES, cd.ALLOW_INSECURE_COOKIES
1269 )
1271 self.camcops_logo_file_absolute = _get_str(
1272 s, cs.CAMCOPS_LOGO_FILE_ABSOLUTE, cd.CAMCOPS_LOGO_FILE_ABSOLUTE
1273 )
1274 self.ctv_filename_spec = _get_str(s, cs.CTV_FILENAME_SPEC)
1276 self.db_url = parser.get(s, cs.DB_URL)
1277 # ... no default: will fail if not provided
1278 self.db_echo = _get_bool(s, cs.DB_ECHO, cd.DB_ECHO)
1279 self.client_api_loglevel = get_config_parameter_loglevel(
1280 parser, s, cs.CLIENT_API_LOGLEVEL, cd.CLIENT_API_LOGLEVEL
1281 )
1282 logging.getLogger("camcops_server.cc_modules.client_api").setLevel(
1283 self.client_api_loglevel
1284 )
1285 # ... MUTABLE GLOBAL STATE (if relatively unimportant); todo: fix
1287 self.disable_password_autocomplete = _get_bool(
1288 s,
1289 cs.DISABLE_PASSWORD_AUTOCOMPLETE,
1290 cd.DISABLE_PASSWORD_AUTOCOMPLETE,
1291 )
1293 self.email_host = _get_str(s, cs.EMAIL_HOST, "")
1294 self.email_port = _get_int(s, cs.EMAIL_PORT, cd.EMAIL_PORT)
1295 self.email_use_tls = _get_bool(s, cs.EMAIL_USE_TLS, cd.EMAIL_USE_TLS)
1296 self.email_host_username = _get_str(s, cs.EMAIL_HOST_USERNAME, "")
1297 self.email_host_password = _get_str(s, cs.EMAIL_HOST_PASSWORD, "")
1299 # Development only: Read password from safe using GNU Pass
1300 gnu_pass_lookup = _get_str(
1301 s, cs.EMAIL_HOST_PASSWORD_GNU_PASS_LOOKUP, ""
1302 )
1303 if gnu_pass_lookup:
1304 output = run(["pass", gnu_pass_lookup], stdout=PIPE)
1305 self.email_host_password = output.stdout.decode("utf-8").split()[0]
1307 self.email_from = _get_str(s, cs.EMAIL_FROM, "")
1308 self.email_sender = _get_str(s, cs.EMAIL_SENDER, "")
1309 self.email_reply_to = _get_str(s, cs.EMAIL_REPLY_TO, "")
1311 self.extra_string_files = _get_multiline(s, cs.EXTRA_STRING_FILES)
1313 self.language = _get_str(s, cs.LANGUAGE, cd.LANGUAGE)
1314 if self.language not in POSSIBLE_LOCALES:
1315 log.warning(
1316 f"Invalid language {self.language!r}, "
1317 f"switching to {cd.LANGUAGE!r}"
1318 )
1319 self.language = cd.LANGUAGE
1320 self.local_institution_url = _get_str(
1321 s, cs.LOCAL_INSTITUTION_URL, cd.LOCAL_INSTITUTION_URL
1322 )
1323 self.local_logo_file_absolute = _get_str(
1324 s, cs.LOCAL_LOGO_FILE_ABSOLUTE, cd.LOCAL_LOGO_FILE_ABSOLUTE
1325 )
1326 self.lockout_threshold = _get_int(
1327 s, cs.LOCKOUT_THRESHOLD, cd.LOCKOUT_THRESHOLD
1328 )
1329 self.lockout_duration_increment_minutes = _get_int(
1330 s,
1331 cs.LOCKOUT_DURATION_INCREMENT_MINUTES,
1332 cd.LOCKOUT_DURATION_INCREMENT_MINUTES,
1333 )
1335 self.mfa_methods = _get_multiline(s, cs.MFA_METHODS)
1336 if not self.mfa_methods:
1337 self.mfa_methods = cd.MFA_METHODS
1338 log.warning(f"MFA_METHODS not specified. Using {self.mfa_methods}")
1339 self.mfa_methods = [x.lower() for x in self.mfa_methods]
1340 assert self.mfa_methods, "Bug: missing MFA_METHODS"
1341 _valid_mfa_methods = class_attribute_values(MfaMethod)
1342 for _mfa_method in self.mfa_methods:
1343 if _mfa_method not in _valid_mfa_methods:
1344 raise ValueError(f"Bad MFA_METHOD item: {_mfa_method!r}")
1346 self.mfa_timeout_s = _get_int(s, cs.MFA_TIMEOUT_S, cd.MFA_TIMEOUT_S)
1348 self.password_change_frequency_days = _get_int(
1349 s,
1350 cs.PASSWORD_CHANGE_FREQUENCY_DAYS,
1351 cd.PASSWORD_CHANGE_FREQUENCY_DAYS,
1352 )
1353 self.patient_spec_if_anonymous = _get_str(
1354 s, cs.PATIENT_SPEC_IF_ANONYMOUS, cd.PATIENT_SPEC_IF_ANONYMOUS
1355 )
1356 self.patient_spec = _get_str(s, cs.PATIENT_SPEC)
1357 self.permit_immediate_downloads = _get_bool(
1358 s, cs.PERMIT_IMMEDIATE_DOWNLOADS, cd.PERMIT_IMMEDIATE_DOWNLOADS
1359 )
1360 # currently not configurable, but easy to add in the future:
1361 self.plot_fontsize = cd.PLOT_FONTSIZE
1363 self.region_code = _get_str(s, cs.REGION_CODE, cd.REGION_CODE)
1364 self.restricted_tasks = {} # type: Dict[str, List[str]]
1365 # ... maps XML task names to lists of authorized group names
1366 restricted_tasks = _get_multiline(s, cs.RESTRICTED_TASKS)
1367 for rt_line in restricted_tasks:
1368 rt_line = rt_line.split("#")[0].strip()
1369 # ... everything before a '#'
1370 if not rt_line: # comment line
1371 continue
1372 try:
1373 xml_taskname, groupnames = rt_line.split(":")
1374 except ValueError:
1375 raise ValueError(
1376 f"Restricted tasks line not in the format "
1377 f"'xml_taskname: groupname1, groupname2, ...'. Line was:\n"
1378 f"{rt_line!r}"
1379 )
1380 xml_taskname = xml_taskname.strip()
1381 if xml_taskname in self.restricted_tasks:
1382 raise ValueError(
1383 f"Duplicate restricted task specification "
1384 f"for {xml_taskname!r}"
1385 )
1386 groupnames = [x.strip() for x in groupnames.split(",")]
1387 for gn in groupnames:
1388 validate_group_name(gn)
1389 self.restricted_tasks[xml_taskname] = groupnames
1391 self.session_timeout_minutes = _get_int(
1392 s, cs.SESSION_TIMEOUT_MINUTES, cd.SESSION_TIMEOUT_MINUTES
1393 )
1394 self.session_cookie_secret = _get_str(s, cs.SESSION_COOKIE_SECRET)
1395 self.session_timeout = datetime.timedelta(
1396 minutes=self.session_timeout_minutes
1397 )
1398 self.session_check_user_ip = _get_bool(
1399 s, cs.SESSION_CHECK_USER_IP, cd.SESSION_CHECK_USER_IP
1400 )
1401 sms_label = _get_str(s, cs.SMS_BACKEND, cd.SMS_BACKEND)
1402 sms_config = self._read_sms_config(parser, sms_label)
1403 self.sms_backend = get_sms_backend(sms_label, sms_config)
1404 self.snomed_task_xml_filename = _get_str(
1405 s, cs.SNOMED_TASK_XML_FILENAME
1406 )
1407 self.snomed_icd9_xml_filename = _get_str(
1408 s, cs.SNOMED_ICD9_XML_FILENAME
1409 )
1410 self.snomed_icd10_xml_filename = _get_str(
1411 s, cs.SNOMED_ICD10_XML_FILENAME
1412 )
1414 self.task_filename_spec = _get_str(s, cs.TASK_FILENAME_SPEC)
1415 self.tracker_filename_spec = _get_str(s, cs.TRACKER_FILENAME_SPEC)
1417 self.user_download_dir = _get_str(s, cs.USER_DOWNLOAD_DIR, "")
1418 self.user_download_file_lifetime_min = _get_int(
1419 s,
1420 cs.USER_DOWNLOAD_FILE_LIFETIME_MIN,
1421 cd.USER_DOWNLOAD_FILE_LIFETIME_MIN,
1422 )
1423 self.user_download_max_space_mb = _get_int(
1424 s, cs.USER_DOWNLOAD_MAX_SPACE_MB, cd.USER_DOWNLOAD_MAX_SPACE_MB
1425 )
1427 self.webview_loglevel = get_config_parameter_loglevel(
1428 parser, s, cs.WEBVIEW_LOGLEVEL, cd.WEBVIEW_LOGLEVEL
1429 )
1430 logging.getLogger().setLevel(self.webview_loglevel) # root logger
1431 # ... MUTABLE GLOBAL STATE (if relatively unimportant); todo: fix
1432 self.wkhtmltopdf_filename = _get_str(s, cs.WKHTMLTOPDF_FILENAME)
1434 # More validity checks for the main section:
1435 if not self.patient_spec_if_anonymous:
1436 raise_missing(s, cs.PATIENT_SPEC_IF_ANONYMOUS)
1437 if not self.patient_spec:
1438 raise_missing(s, cs.PATIENT_SPEC)
1439 if not self.session_cookie_secret:
1440 raise_missing(s, cs.SESSION_COOKIE_SECRET)
1441 if not self.task_filename_spec:
1442 raise_missing(s, cs.TASK_FILENAME_SPEC)
1443 if not self.tracker_filename_spec:
1444 raise_missing(s, cs.TRACKER_FILENAME_SPEC)
1445 if not self.ctv_filename_spec:
1446 raise_missing(s, cs.CTV_FILENAME_SPEC)
1448 # To prevent errors:
1449 del s
1450 del cs
1452 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1453 # Web server/WSGI section
1454 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1455 ws = CONFIG_FILE_SERVER_SECTION
1456 cw = ConfigParamServer
1458 self.cherrypy_log_screen = _get_bool(
1459 ws, cw.CHERRYPY_LOG_SCREEN, cd.CHERRYPY_LOG_SCREEN
1460 )
1461 self.cherrypy_root_path = _get_str(
1462 ws, cw.CHERRYPY_ROOT_PATH, cd.CHERRYPY_ROOT_PATH
1463 )
1464 self.cherrypy_server_name = _get_str(
1465 ws, cw.CHERRYPY_SERVER_NAME, cd.CHERRYPY_SERVER_NAME
1466 )
1467 self.cherrypy_threads_max = _get_int(
1468 ws, cw.CHERRYPY_THREADS_MAX, cd.CHERRYPY_THREADS_MAX
1469 )
1470 self.cherrypy_threads_start = _get_int(
1471 ws, cw.CHERRYPY_THREADS_START, cd.CHERRYPY_THREADS_START
1472 )
1473 self.debug_reverse_proxy = _get_bool(
1474 ws, cw.DEBUG_REVERSE_PROXY, cd.DEBUG_REVERSE_PROXY
1475 )
1476 self.debug_show_gunicorn_options = _get_bool(
1477 ws, cw.DEBUG_SHOW_GUNICORN_OPTIONS, cd.DEBUG_SHOW_GUNICORN_OPTIONS
1478 )
1479 self.debug_toolbar = _get_bool(ws, cw.DEBUG_TOOLBAR, cd.DEBUG_TOOLBAR)
1480 self.gunicorn_debug_reload = _get_bool(
1481 ws, cw.GUNICORN_DEBUG_RELOAD, cd.GUNICORN_DEBUG_RELOAD
1482 )
1483 self.gunicorn_num_workers = _get_int(
1484 ws, cw.GUNICORN_NUM_WORKERS, cd.GUNICORN_NUM_WORKERS
1485 )
1486 self.gunicorn_timeout_s = _get_int(
1487 ws, cw.GUNICORN_TIMEOUT_S, cd.GUNICORN_TIMEOUT_S
1488 )
1489 self.host = _get_str(ws, cw.HOST, cd.HOST)
1490 self.port = _get_int(ws, cw.PORT, cd.PORT)
1491 self.proxy_http_host = _get_str(ws, cw.PROXY_HTTP_HOST)
1492 self.proxy_remote_addr = _get_str(ws, cw.PROXY_REMOTE_ADDR)
1493 self.proxy_rewrite_path_info = _get_bool(
1494 ws, cw.PROXY_REWRITE_PATH_INFO, cd.PROXY_REWRITE_PATH_INFO
1495 )
1496 self.proxy_script_name = _get_str(ws, cw.PROXY_SCRIPT_NAME)
1497 self.proxy_server_name = _get_str(ws, cw.PROXY_SERVER_NAME)
1498 self.proxy_server_port = _get_int(ws, cw.PROXY_SERVER_PORT)
1499 self.proxy_url_scheme = _get_str(ws, cw.PROXY_URL_SCHEME)
1500 self.show_request_immediately = _get_bool(
1501 ws, cw.SHOW_REQUEST_IMMEDIATELY, cd.SHOW_REQUEST_IMMEDIATELY
1502 )
1503 self.show_requests = _get_bool(ws, cw.SHOW_REQUESTS, cd.SHOW_REQUESTS)
1504 self.show_response = _get_bool(ws, cw.SHOW_RESPONSE, cd.SHOW_RESPONSE)
1505 self.show_timing = _get_bool(ws, cw.SHOW_TIMING, cd.SHOW_TIMING)
1506 self.ssl_certificate = _get_str(ws, cw.SSL_CERTIFICATE)
1507 self.ssl_private_key = _get_str(ws, cw.SSL_PRIVATE_KEY)
1508 self.static_cache_duration_s = _get_int(
1509 ws, cw.STATIC_CACHE_DURATION_S, cd.STATIC_CACHE_DURATION_S
1510 )
1511 self.trusted_proxy_headers = _get_multiline(
1512 ws, cw.TRUSTED_PROXY_HEADERS
1513 )
1514 self.unix_domain_socket = _get_str(ws, cw.UNIX_DOMAIN_SOCKET)
1516 # The defaults here depend on values above:
1517 self.external_url_scheme = _get_str(
1518 ws, cw.EXTERNAL_URL_SCHEME, cd.EXTERNAL_URL_SCHEME
1519 )
1520 self.external_server_name = _get_str(
1521 ws, cw.EXTERNAL_SERVER_NAME, self.host
1522 )
1523 self.external_server_port = _get_int(
1524 ws, cw.EXTERNAL_SERVER_PORT, self.port
1525 )
1526 self.external_script_name = _get_str(ws, cw.EXTERNAL_SCRIPT_NAME, "")
1528 for tph in self.trusted_proxy_headers:
1529 if tph not in ReverseProxiedMiddleware.ALL_CANDIDATES:
1530 raise ValueError(
1531 f"Invalid {cw.TRUSTED_PROXY_HEADERS} value specified: "
1532 f"was {tph!r}, options are "
1533 f"{ReverseProxiedMiddleware.ALL_CANDIDATES}"
1534 )
1536 del ws
1537 del cw
1539 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1540 # Export section
1541 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1542 es = CONFIG_FILE_EXPORT_SECTION
1543 ce = ConfigParamExportGeneral
1545 self.celery_beat_extra_args = _get_multiline(
1546 es, ce.CELERY_BEAT_EXTRA_ARGS
1547 )
1548 self.celery_beat_schedule_database = _get_str(
1549 es, ce.CELERY_BEAT_SCHEDULE_DATABASE
1550 )
1551 if not self.celery_beat_schedule_database:
1552 raise_missing(es, ce.CELERY_BEAT_SCHEDULE_DATABASE)
1553 self.celery_broker_url = _get_str(
1554 es, ce.CELERY_BROKER_URL, cd.CELERY_BROKER_URL
1555 )
1556 self.celery_worker_extra_args = _get_multiline(
1557 es, ce.CELERY_WORKER_EXTRA_ARGS
1558 )
1559 self.celery_export_task_rate_limit = _get_str(
1560 es, ce.CELERY_EXPORT_TASK_RATE_LIMIT
1561 )
1563 self.export_lockdir = _get_str(es, ce.EXPORT_LOCKDIR)
1564 if not self.export_lockdir:
1565 raise_missing(es, ConfigParamExportGeneral.EXPORT_LOCKDIR)
1567 self.export_recipient_names = _get_multiline_ignoring_comments(
1568 CONFIG_FILE_EXPORT_SECTION, ce.RECIPIENTS
1569 )
1570 duplicates = [
1571 name
1572 for name, count in collections.Counter(
1573 self.export_recipient_names
1574 ).items()
1575 if count > 1
1576 ]
1577 if duplicates:
1578 raise ValueError(
1579 f"Duplicate export recipients specified: {duplicates!r}"
1580 )
1581 for recip_name in self.export_recipient_names:
1582 if re.match(VALID_RECIPIENT_NAME_REGEX, recip_name) is None:
1583 raise ValueError(
1584 f"Recipient names must be alphanumeric or _- only; was "
1585 f"{recip_name!r}"
1586 )
1587 if len(set(self.export_recipient_names)) != len(
1588 self.export_recipient_names
1589 ):
1590 raise ValueError("Recipient names contain duplicates")
1591 self._export_recipients = [] # type: List[ExportRecipientInfo]
1592 self._read_export_recipients(parser)
1594 self.schedule_timezone = _get_str(
1595 es, ce.SCHEDULE_TIMEZONE, cd.SCHEDULE_TIMEZONE
1596 )
1598 self.crontab_entries = [] # type: List[CrontabEntry]
1599 crontab_lines = _get_multiline(es, ce.SCHEDULE)
1600 for crontab_line in crontab_lines:
1601 crontab_line = crontab_line.split("#")[0].strip()
1602 # ... everything before a '#'
1603 if not crontab_line: # comment line
1604 continue
1605 crontab_entry = CrontabEntry(line=crontab_line)
1606 if crontab_entry.content not in self.export_recipient_names:
1607 raise ValueError(
1608 f"{ce.SCHEDULE} setting exists for non-existent recipient "
1609 f"{crontab_entry.content}"
1610 )
1611 self.crontab_entries.append(crontab_entry)
1613 del es
1614 del ce
1616 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1617 # Other attributes
1618 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1619 self._sqla_engine = None
1621 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1622 # Docker checks
1623 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1624 if self.running_under_docker:
1625 log.info("Docker environment detected")
1627 # Values expected to be fixed
1628 warn_if_not_docker_value(
1629 param_name=ConfigParamExportGeneral.CELERY_BROKER_URL,
1630 actual_value=self.celery_broker_url,
1631 required_value=DockerConstants.CELERY_BROKER_URL,
1632 )
1633 warn_if_not_docker_value(
1634 param_name=ConfigParamServer.HOST,
1635 actual_value=self.host,
1636 required_value=DockerConstants.HOST,
1637 )
1639 # Values expected to be present
1640 #
1641 # - Re SSL certificates: reconsidered. People may want to run
1642 # internal plain HTTP but then an Apache front end, and they
1643 # wouldn't appreciate the warnings.
1644 #
1645 # warn_if_not_present(
1646 # param_name=ConfigParamServer.SSL_CERTIFICATE,
1647 # value=self.ssl_certificate
1648 # )
1649 # warn_if_not_present(
1650 # param_name=ConfigParamServer.SSL_PRIVATE_KEY,
1651 # value=self.ssl_private_key
1652 # )
1654 # Config-related files
1655 warn_if_not_within_docker_dir(
1656 param_name=ConfigParamServer.SSL_CERTIFICATE,
1657 filespec=self.ssl_certificate,
1658 permit_cfg=True,
1659 )
1660 warn_if_not_within_docker_dir(
1661 param_name=ConfigParamServer.SSL_PRIVATE_KEY,
1662 filespec=self.ssl_private_key,
1663 permit_cfg=True,
1664 )
1665 warn_if_not_within_docker_dir(
1666 param_name=ConfigParamSite.LOCAL_LOGO_FILE_ABSOLUTE,
1667 filespec=self.local_logo_file_absolute,
1668 permit_cfg=True,
1669 permit_venv=True,
1670 )
1671 warn_if_not_within_docker_dir(
1672 param_name=ConfigParamSite.CAMCOPS_LOGO_FILE_ABSOLUTE,
1673 filespec=self.camcops_logo_file_absolute,
1674 permit_cfg=True,
1675 permit_venv=True,
1676 )
1677 for esf in self.extra_string_files:
1678 warn_if_not_within_docker_dir(
1679 param_name=ConfigParamSite.EXTRA_STRING_FILES,
1680 filespec=esf,
1681 permit_cfg=True,
1682 permit_venv=True,
1683 param_contains_not_is=True,
1684 )
1685 warn_if_not_within_docker_dir(
1686 param_name=ConfigParamSite.SNOMED_ICD9_XML_FILENAME,
1687 filespec=self.snomed_icd9_xml_filename,
1688 permit_cfg=True,
1689 permit_venv=True,
1690 )
1691 warn_if_not_within_docker_dir(
1692 param_name=ConfigParamSite.SNOMED_ICD10_XML_FILENAME,
1693 filespec=self.snomed_icd10_xml_filename,
1694 permit_cfg=True,
1695 permit_venv=True,
1696 )
1697 warn_if_not_within_docker_dir(
1698 param_name=ConfigParamSite.SNOMED_TASK_XML_FILENAME,
1699 filespec=self.snomed_task_xml_filename,
1700 permit_cfg=True,
1701 permit_venv=True,
1702 )
1704 # Temporary/scratch space that needs to be shared between Docker
1705 # containers
1706 warn_if_not_within_docker_dir(
1707 param_name=ConfigParamSite.USER_DOWNLOAD_DIR,
1708 filespec=self.user_download_dir,
1709 permit_tmp=True,
1710 )
1711 warn_if_not_within_docker_dir(
1712 param_name=ConfigParamExportGeneral.CELERY_BEAT_SCHEDULE_DATABASE, # noqa
1713 filespec=self.celery_beat_schedule_database,
1714 permit_tmp=True,
1715 )
1716 warn_if_not_within_docker_dir(
1717 param_name=ConfigParamExportGeneral.EXPORT_LOCKDIR,
1718 filespec=self.export_lockdir,
1719 permit_tmp=True,
1720 )
1722 # -------------------------------------------------------------------------
1723 # Database functions
1724 # -------------------------------------------------------------------------
1726 def get_sqla_engine(self) -> Engine:
1727 """
1728 Returns an SQLAlchemy :class:`Engine`.
1730 I was previously misinterpreting the appropriate scope of an Engine.
1731 I thought: create one per request.
1732 But the Engine represents the connection *pool*.
1733 So if you create them all the time, you get e.g. a
1734 'Too many connections' error.
1736 "The appropriate scope is once per [database] URL per application,
1737 at the module level."
1739 - https://groups.google.com/forum/#!topic/sqlalchemy/ZtCo2DsHhS4
1740 - https://stackoverflow.com/questions/8645250/how-to-close-sqlalchemy-connection-in-mysql
1742 Now, our CamcopsConfig instance is cached, so there should be one of
1743 them overall. See get_config() below.
1745 Therefore, making the engine a member of this class should do the
1746 trick, whilst avoiding global variables.
1747 """ # noqa
1748 if self._sqla_engine is None:
1749 self._sqla_engine = create_engine(
1750 self.db_url,
1751 echo=self.db_echo,
1752 pool_pre_ping=True,
1753 # pool_size=0, # no limit (for parallel testing, which failed)
1754 )
1755 log.debug(
1756 "Created SQLAlchemy engine for URL {}",
1757 get_safe_url_from_engine(self._sqla_engine),
1758 )
1759 return self._sqla_engine
1761 @property
1762 @cache_region_static.cache_on_arguments(function_key_generator=fkg)
1763 def get_all_table_names(self) -> List[str]:
1764 """
1765 Returns all table names from the database.
1766 """
1767 log.debug("Fetching database table names")
1768 engine = self.get_sqla_engine()
1769 return get_table_names(engine=engine)
1771 def get_dbsession_raw(self) -> SqlASession:
1772 """
1773 Returns a raw SQLAlchemy Session.
1774 Avoid this -- use :func:`get_dbsession_context` instead.
1775 """
1776 engine = self.get_sqla_engine()
1777 maker = sessionmaker(bind=engine)
1778 dbsession = maker() # type: SqlASession
1779 return dbsession
1781 @contextlib.contextmanager
1782 def get_dbsession_context(self) -> Generator[SqlASession, None, None]:
1783 """
1784 Context manager to provide an SQLAlchemy session that will COMMIT
1785 once we've finished, or perform a ROLLBACK if there was an exception.
1786 """
1787 dbsession = self.get_dbsession_raw()
1788 # noinspection PyBroadException
1789 try:
1790 yield dbsession
1791 dbsession.commit()
1792 except Exception:
1793 dbsession.rollback()
1794 raise
1795 finally:
1796 dbsession.close()
1798 def _assert_valid_database_engine(self) -> None:
1799 """
1800 Assert that our backend database is a valid type.
1802 Specifically, we prohibit:
1804 - SQL Server versions before 2008: they don't support timezones
1805 and we need that.
1806 """
1807 engine = self.get_sqla_engine()
1808 if not is_sqlserver(engine):
1809 return
1810 assert is_sqlserver_2008_or_later(engine), (
1811 "If you use Microsoft SQL Server as the back-end database for a "
1812 "CamCOPS server, it must be at least SQL Server 2008. Older "
1813 "versions do not have time zone awareness."
1814 )
1816 def _assert_database_is_at_head(self) -> None:
1817 """
1818 Assert that the current database is at its head (most recent) revision,
1819 by comparing its Alembic version number (written into the Alembic
1820 version table of the database) to the most recent Alembic revision in
1821 our ``camcops_server/alembic/versions`` directory.
1822 """
1823 current, head = get_current_and_head_revision(
1824 database_url=self.db_url,
1825 alembic_config_filename=ALEMBIC_CONFIG_FILENAME,
1826 alembic_base_dir=ALEMBIC_BASE_DIR,
1827 version_table=ALEMBIC_VERSION_TABLE,
1828 )
1829 if current == head:
1830 log.debug("Database is at correct (head) revision of {}", current)
1831 else:
1832 raise_runtime_error(
1833 f"Database structure is at version {current} but should be at "
1834 f"version {head}. CamCOPS will not start. Please use the "
1835 f"'upgrade_db' command to fix this."
1836 )
1838 def assert_database_ok(self) -> None:
1839 """
1840 Asserts that our database engine is OK and our database structure is
1841 correct.
1842 """
1843 self._assert_valid_database_engine()
1844 self._assert_database_is_at_head()
1846 # -------------------------------------------------------------------------
1847 # SNOMED-CT functions
1848 # -------------------------------------------------------------------------
1850 def get_task_snomed_concepts(self) -> Dict[str, SnomedConcept]:
1851 """
1852 Returns all SNOMED-CT concepts for tasks.
1854 Returns:
1855 dict: maps lookup strings to :class:`SnomedConcept` objects
1856 """
1857 if not self.snomed_task_xml_filename:
1858 return {}
1859 return get_all_task_snomed_concepts(self.snomed_task_xml_filename)
1861 def get_icd9cm_snomed_concepts(self) -> Dict[str, List[SnomedConcept]]:
1862 """
1863 Returns all SNOMED-CT concepts for ICD-9-CM codes supported by CamCOPS.
1865 Returns:
1866 dict: maps ICD-9-CM codes to :class:`SnomedConcept` objects
1867 """
1868 if not self.snomed_icd9_xml_filename:
1869 return {}
1870 return get_icd9_snomed_concepts_from_xml(self.snomed_icd9_xml_filename)
1872 def get_icd10_snomed_concepts(self) -> Dict[str, List[SnomedConcept]]:
1873 """
1874 Returns all SNOMED-CT concepts for ICD-10-CM codes supported by
1875 CamCOPS.
1877 Returns:
1878 dict: maps ICD-10 codes to :class:`SnomedConcept` objects
1879 """
1880 if not self.snomed_icd10_xml_filename:
1881 return {}
1882 return get_icd10_snomed_concepts_from_xml(
1883 self.snomed_icd10_xml_filename
1884 )
1886 # -------------------------------------------------------------------------
1887 # Export functions
1888 # -------------------------------------------------------------------------
1890 def _read_export_recipients(
1891 self, parser: configparser.ConfigParser = None
1892 ) -> None:
1893 """
1894 Loads
1895 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo`
1896 objects from the config file. Stores them in
1897 ``self._export_recipients``.
1899 Note that these objects are **not** associated with a database session.
1901 Args:
1902 parser: optional :class:`configparser.ConfigParser` object.
1903 """
1904 self._export_recipients = [] # type: List[ExportRecipientInfo]
1905 for recip_name in self.export_recipient_names:
1906 log.debug("Loading export config for recipient {!r}", recip_name)
1907 try:
1908 validate_export_recipient_name(recip_name)
1909 except ValueError as e:
1910 raise ValueError(f"Bad recipient name {recip_name!r}: {e}")
1911 recipient = ExportRecipientInfo.read_from_config(
1912 self, parser=parser, recipient_name=recip_name
1913 )
1914 self._export_recipients.append(recipient)
1916 def get_all_export_recipient_info(self) -> List["ExportRecipientInfo"]:
1917 """
1918 Returns all export recipients (in their "database unaware" form)
1919 specified in the config.
1921 Returns:
1922 list: of
1923 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo`
1924 """ # noqa
1925 return self._export_recipients
1927 # -------------------------------------------------------------------------
1928 # File-based locks
1929 # -------------------------------------------------------------------------
1931 def get_export_lockfilename_recipient_db(self, recipient_name: str) -> str:
1932 """
1933 Returns a full path to a lockfile suitable for locking for a
1934 whole-database export to a particular export recipient.
1936 Args:
1937 recipient_name: name of the recipient
1939 Returns:
1940 a filename
1941 """
1942 filename = f"camcops_export_db_{recipient_name}"
1943 # ".lock" is appended automatically by the lockfile package
1944 return os.path.join(self.export_lockdir, filename)
1946 def get_export_lockfilename_recipient_fhir(
1947 self, recipient_name: str
1948 ) -> str:
1949 """
1950 Returns a full path to a lockfile suitable for locking for a
1951 FHIR export to a particular export recipient.
1953 (This must be different from
1954 :meth:`get_export_lockfilename_recipient_db`, because of what we assume
1955 about someone else holding the same lock.)
1957 Args:
1958 recipient_name: name of the recipient
1960 Returns:
1961 a filename
1962 """
1963 filename = f"camcops_export_fhir_{recipient_name}"
1964 # ".lock" is appended automatically by the lockfile package
1965 return os.path.join(self.export_lockdir, filename)
1967 def get_export_lockfilename_recipient_task(
1968 self, recipient_name: str, basetable: str, pk: int
1969 ) -> str:
1970 """
1971 Returns a full path to a lockfile suitable for locking for a
1972 single-task export to a particular export recipient.
1974 Args:
1975 recipient_name: name of the recipient
1976 basetable: task base table name
1977 pk: server PK of the task
1979 Returns:
1980 a filename
1981 """
1982 filename = f"camcops_export_task_{recipient_name}_{basetable}_{pk}"
1983 # ".lock" is appended automatically by the lockfile package
1984 return os.path.join(self.export_lockdir, filename)
1986 def get_master_export_recipient_lockfilename(self) -> str:
1987 """
1988 When we are modifying export recipients, we check "is this information
1989 the same as the current version in the database", and if not, we write
1990 fresh information to the database. If lots of processes do that at the
1991 same time, we have a problem (usually a database deadlock) -- hence
1992 this lock.
1994 Returns:
1995 a filename
1996 """
1997 filename = "camcops_master_export_recipient"
1998 # ".lock" is appended automatically by the lockfile package
1999 return os.path.join(self.export_lockdir, filename)
2001 def get_celery_beat_pidfilename(self) -> str:
2002 """
2003 Process ID file (pidfile) used by ``celery beat --pidfile ...``.
2004 """
2005 filename = "camcops_celerybeat.pid"
2006 return os.path.join(self.export_lockdir, filename)
2008 # -------------------------------------------------------------------------
2009 # SMS backend
2010 # -------------------------------------------------------------------------
2012 @staticmethod
2013 def _read_sms_config(
2014 parser: configparser.ConfigParser, sms_label: str
2015 ) -> Dict[str, str]:
2016 """
2017 Read a config section for a specific SMS backend.
2018 """
2019 section_name = f"{CONFIG_FILE_SMS_BACKEND_PREFIX}:{sms_label}"
2020 if not parser.has_section(section_name):
2021 return {}
2023 sms_config = {}
2024 section = parser[section_name]
2025 for key in section:
2026 sms_config[key.lower()] = section[key]
2027 return sms_config
2030# =============================================================================
2031# Get config filename from an appropriate environment (WSGI or OS)
2032# =============================================================================
2035def get_config_filename_from_os_env() -> str:
2036 """
2037 Returns the config filename to use, from our operating system environment
2038 variable.
2040 (We do NOT trust the WSGI environment for this.)
2041 """
2042 config_filename = os.environ.get(ENVVAR_CONFIG_FILE)
2043 if not config_filename:
2044 raise AssertionError(
2045 f"OS environment did not provide the required "
2046 f"environment variable {ENVVAR_CONFIG_FILE}"
2047 )
2048 return config_filename
2051# =============================================================================
2052# Cached instances
2053# =============================================================================
2056@cache_region_static.cache_on_arguments(function_key_generator=fkg)
2057def get_config(config_filename: str) -> CamcopsConfig:
2058 """
2059 Returns a :class:`camcops_server.cc_modules.cc_config.CamcopsConfig` from
2060 the specified config filename.
2062 Cached.
2063 """
2064 return CamcopsConfig(config_filename)
2067# =============================================================================
2068# Get default config
2069# =============================================================================
2072def get_default_config_from_os_env() -> CamcopsConfig:
2073 """
2074 Returns the :class:`camcops_server.cc_modules.cc_config.CamcopsConfig`
2075 representing the config filename that we read from our operating system
2076 environment variable.
2077 """
2078 if ON_READTHEDOCS:
2079 return CamcopsConfig(config_filename="", config_text=get_demo_config())
2080 else:
2081 return get_config(get_config_filename_from_os_env())