Source code for camcops_server.cc_modules.cc_config

#!/usr/bin/env python
# camcops_server/cc_modules/cc_config.py

"""
===============================================================================

    Copyright (C) 2012-2018 Rudolf Cardinal (rudolf@pobox.com).

    This file is part of CamCOPS.

    CamCOPS is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    CamCOPS is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with CamCOPS. If not, see <http://www.gnu.org/licenses/>.

===============================================================================
"""

# There are CONDITIONAL AND IN-FUNCTION IMPORTS HERE; see below. This is to
# minimize the number of modules loaded when this is used in the context of the
# client-side database script, rather than the webview.

import codecs
import configparser
import contextlib
import datetime
import operator
import os
import logging
from typing import Dict, Generator, List

from cardinal_pythonlib.configfiles import (
    get_config_parameter,
    get_config_parameter_boolean,
    get_config_parameter_loglevel,
    get_config_parameter_multiline
)
from cardinal_pythonlib.logs import BraceStyleAdapter
from cardinal_pythonlib.randomness import create_base64encoded_randomness
from cardinal_pythonlib.sqlalchemy.alembic_func import (
    get_current_and_head_revision,
)
from cardinal_pythonlib.sqlalchemy.engine_func import (
    is_sqlserver,
    is_sqlserver_2008_or_later,
)
from cardinal_pythonlib.sqlalchemy.logs import pre_disable_sqlalchemy_extra_echo_log  # noqa
from cardinal_pythonlib.sqlalchemy.schema import get_table_names
from cardinal_pythonlib.sqlalchemy.session import (
    get_safe_url_from_engine,
    make_mysql_url,
)
from pendulum import DateTime as Pendulum
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.base import Engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import Session as SqlASession

from .cc_baseconstants import (
    ALEMBIC_BASE_DIR,
    ALEMBIC_CONFIG_FILENAME,
    ALEMBIC_VERSION_TABLE,
    CAMCOPS_EXECUTABLE,
    CAMCOPS_SERVER_DIRECTORY,
    DEFAULT_EXTRA_STRINGS_DIR,
    ENVVAR_CONFIG_FILE,
    INTROSPECTABLE_EXTENSIONS,
    LINUX_DEFAULT_LOCK_DIR,
    LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR,
    STATIC_ROOT_DIR,
)
from .cc_cache import cache_region_static, fkg
from .cc_constants import (
    CONFIG_FILE_MAIN_SECTION,
    CONFIG_FILE_RECIPIENTLIST_SECTION,
    DEFAULT_CAMCOPS_LOGO_FILE,
    DEFAULT_LOCAL_INSTITUTION_URL,
    DEFAULT_LOCAL_LOGO_FILE,
    DEFAULT_LOCKOUT_DURATION_INCREMENT_MINUTES,
    DEFAULT_LOCKOUT_THRESHOLD,
    DEFAULT_PASSWORD_CHANGE_FREQUENCY_DAYS,
    DEFAULT_PLOT_FONTSIZE,
    DEFAULT_TIMEOUT_MINUTES,
    INTROSPECTION_BASE_DIRECTORY,
)
from .cc_filename import FilenameSpecElement, PatientSpecElementForFilename
from .cc_pyramid import MASTER_ROUTE_CLIENT_API
from .cc_simpleobjects import IntrospectionFileDetails
from .cc_recipdef import ConfigParamRecipient, RecipientDefinition
from .cc_version_string import CAMCOPS_SERVER_VERSION_STRING

log = BraceStyleAdapter(logging.getLogger(__name__))

pre_disable_sqlalchemy_extra_echo_log()


# =============================================================================
# Demo config
# =============================================================================

DEFAULT_DB_NAME = 'camcops'
DEFAULT_DB_USER = 'YYY_USERNAME_REPLACE_ME'
DEFAULT_DB_PASSWORD = 'ZZZ_PASSWORD_REPLACE_ME'
DEFAULT_DB_READONLY_USER = 'QQQ_USERNAME_REPLACE_ME'
DEFAULT_DB_READONLY_PASSWORD = 'PPP_PASSWORD_REPLACE_ME'
DUMMY_INSTITUTION_URL = 'http://www.mydomain/'


class ConfigParamMain(object):
    ALLOW_INSECURE_COOKIES = "ALLOW_INSECURE_COOKIES"
    CAMCOPS_LOGO_FILE_ABSOLUTE = "CAMCOPS_LOGO_FILE_ABSOLUTE"
    CLIENT_API_LOGLEVEL = "CLIENT_API_LOGLEVEL"
    CTV_FILENAME_SPEC = "CTV_FILENAME_SPEC"
    DB_URL = "DB_URL"
    DB_ECHO = "DB_ECHO"
    DISABLE_PASSWORD_AUTOCOMPLETE = "DISABLE_PASSWORD_AUTOCOMPLETE"
    EXTRA_STRING_FILES = "EXTRA_STRING_FILES"
    HL7_LOCKFILE = "HL7_LOCKFILE"
    INTROSPECTION = "INTROSPECTION"
    LOCAL_INSTITUTION_URL = "LOCAL_INSTITUTION_URL"
    LOCAL_LOGO_FILE_ABSOLUTE = "LOCAL_LOGO_FILE_ABSOLUTE"
    LOCKOUT_DURATION_INCREMENT_MINUTES = "LOCKOUT_DURATION_INCREMENT_MINUTES"
    LOCKOUT_THRESHOLD = "LOCKOUT_THRESHOLD"
    PASSWORD_CHANGE_FREQUENCY_DAYS = "PASSWORD_CHANGE_FREQUENCY_DAYS"
    PATIENT_SPEC = "PATIENT_SPEC"
    PATIENT_SPEC_IF_ANONYMOUS = "PATIENT_SPEC_IF_ANONYMOUS"
    SESSION_COOKIE_SECRET = "SESSION_COOKIE_SECRET"
    SESSION_TIMEOUT_MINUTES = "SESSION_TIMEOUT_MINUTES"
    SUMMARY_TABLES_LOCKFILE = "SUMMARY_TABLES_LOCKFILE"
    TASK_FILENAME_SPEC = "TASK_FILENAME_SPEC"
    TRACKER_FILENAME_SPEC = "TRACKER_FILENAME_SPEC"
    WEBVIEW_LOGLEVEL = "WEBVIEW_LOGLEVEL"
    WKHTMLTOPDF_FILENAME = "WKHTMLTOPDF_FILENAME"


def get_demo_config(extra_strings_dir: str = None,
                    lock_dir: str = None,
                    hl7_lockfile_stem: str = None,
                    static_dir: str = None,
                    summary_table_lock_file_stem: str = None,
                    db_url: str = None,
                    db_echo: bool = False) -> str:
    extra_strings_dir = extra_strings_dir or DEFAULT_EXTRA_STRINGS_DIR
    extra_strings_spec = os.path.join(extra_strings_dir, '*')
    lock_dir = lock_dir or LINUX_DEFAULT_LOCK_DIR
    hl7_lockfile_stem = hl7_lockfile_stem or os.path.join(
        lock_dir, 'camcops.hl7')
    static_dir = static_dir or STATIC_ROOT_DIR
    summary_table_lock_file_stem = (
        summary_table_lock_file_stem or os.path.join(
            lock_dir, 'camcops.summarytables'
        )
    )
    # ...
    # http://www.debian.org/doc/debian-policy/ch-opersys.html#s-writing-init
    # https://people.canonical.com/~cjwatson/ubuntu-policy/policy.html/ch-opersys.html  # noqa
    session_cookie_secret = create_base64encoded_randomness(num_bytes=64)

    if not db_url:
        db_url = make_mysql_url(username=DEFAULT_DB_USER,
                                password=DEFAULT_DB_PASSWORD,
                                dbname=DEFAULT_DB_NAME)
    return """
# Demonstration CamCOPS configuration file.
# Created by CamCOPS version {version} at {now}.

# =============================================================================
# Format of the CamCOPS configuration file
# =============================================================================
# COMMENTS. Hashes (#) and semicolons (;) mark out comments.
# SECTIONS. Sections are indicated with: [section]
# NAME/VALUE (KEY/VALUE) PAIRS.
# - The parser used is ConfigParser (Python).
# - It allows "name=value" or "name:value".
# - BOOLEAN OPTIONS. For Boolean options, TRUE values are any of: 1, yes, true,
#   on (case-insensitive). FALSE values are any of: 0, no, false, off.
# - UTF-8 encoding. Use this. For ConfigParser, the file is explicitly opened
#   in UTF-8 mode.
# - Avoid indentation.

# =============================================================================
# Main section: [{CONFIG_FILE_MAIN_SECTION}]
# =============================================================================

[{CONFIG_FILE_MAIN_SECTION}]

# -----------------------------------------------------------------------------
# Database connection/tools
# -----------------------------------------------------------------------------

# {cp.DB_URL}: SQLAlchemy connection URL.
# See http://docs.sqlalchemy.org/en/latest/core/engines.html
# Examples:
# - MySQL under Linux via mysqlclient
#   $$ pip install mysqlclient
#   DB_URL = mysql+mysqldb://<username>:<password>@<host>:<port>/<database>?charset=utf8
#
#   (The default MySQL port is 3306, and 'localhost' is often the right host.)
#
# - SQL Server under Windows via ODBC and username/password authentication
#   C:\> pip install pyodbc
#   DB_URL = mssql+pyodbc://<username>:<password>@<odbc_dsn_name>
#
# - ... or via Windows authentication: 
#   DB_URL = mssql+pyodbc://@<odbc_dsn_name>

{cp.DB_URL} = {db_url}

# {cp.DB_ECHO}: echo all SQL?

{cp.DB_ECHO} = {db_echo}

# -----------------------------------------------------------------------------
# URLs and paths
# -----------------------------------------------------------------------------
#
# A quick note on absolute and relative URLs, and how CamCOPS is mounted.
#
# Suppose your CamCOPS site is visible at
#       https://www.somewhere.ac.uk/camcops_smith_lab/webview
#       ^      ^^                 ^^                ^^      ^
#       +------++-----------------++----------------++------+
#       |       |                  |                 |
#       1       2                  3                 4
#
# Part 1 is the protocol, and part 2 the machine name.
# Part 3 is the mount point. The main server (e.g. Apache) knows where the
# CamCOPS script is mounted (in this case /camcops_smith_lab). It does NOT
# tell the script via the script's WSGI environment. Therefore, if the script
# sends HTML including links, the script can operate only in relative mode.
# For it to operate in absolute mode, it would need to know (3).
# Part 4 is visible to the CamCOPS script.
#
# If CamCOPS emitted URLs starting with '/', it would need to be told at least
# part (3). To use absolute URLs, it would need to know all of (1), (2), (3).
# We will follow others (e.g. http://stackoverflow.com/questions/2005079) and
# use only relative URLs.

# {cp.LOCAL_INSTITUTION_URL}: Clicking on your institution's logo in the CamCOPS
# menu will take you to this URL.
# Edit the next line to point to your institution:

{cp.LOCAL_INSTITUTION_URL} = {DUMMY_INSTITUTION_URL}

# {cp.LOCAL_LOGO_FILE_ABSOLUTE}: Specify the full path to your institution's logo
# file, e.g. /var/www/logo_local_myinstitution.png . It's used for PDF
# generation; HTML views use the fixed string "static/logo_local.png", aliased
# to your file via the Apache configuration file).
# Edit the next line to point to your local institution's logo file:

{cp.LOCAL_LOGO_FILE_ABSOLUTE} = {static_dir}/logo_local.png

# {cp.CAMCOPS_LOGO_FILE_ABSOLUTE}: similarly, but for the CamCOPS logo.
# It's fine not to specify this.

# {cp.CAMCOPS_LOGO_FILE_ABSOLUTE} = {static_dir}/logo_camcops.png

# {cp.EXTRA_STRING_FILES}: multiline list of filenames (with absolute paths), read
# by the server, and used as EXTRA STRING FILES. Should at the MINIMUM point
# to the string file camcops.xml
# May use "glob" pattern-matching (see
# https://docs.python.org/3.5/library/glob.html).

{cp.EXTRA_STRING_FILES} = {extra_strings_spec}

# {cp.INTROSPECTION}: permits the offering of CamCOPS source code files to the user,
# allowing inspection of tasks' internal calculating algorithms. Default is
# true.

{cp.INTROSPECTION} = true

# {cp.HL7_LOCKFILE}: filename stem used for process locking for HL7 message
# transmission. Default is {hl7_lockfile_stem}
# The actual lockfile will, in this case, be called
#     {hl7_lockfile_stem}.lock
# and other process-specific files will be created in the same directory (so
# the CamCOPS script must have permission from the operating system to do so).
# The installation script will create the directory {lock_dir}

{cp.HL7_LOCKFILE} = {hl7_lockfile_stem}

# {cp.SUMMARY_TABLES_LOCKFILE}: file stem used for process locking for summary table
# generation. Default is {summary_table_lock_file_stem}.
# The lockfile will, in this case, be called
#     {summary_table_lock_file_stem}.lock
# and other process-specific files will be created in the same directory (so
# the CamCOPS script must have permission from the operating system to do so).
# The installation script will create the directory {lock_dir}

{cp.SUMMARY_TABLES_LOCKFILE} = {summary_table_lock_file_stem}

# {cp.WKHTMLTOPDF_FILENAME}: for the pdfkit PDF engine, specify a filename for
# wkhtmltopdf that incorporates any need for an X Server (not the default
# /usr/bin/wkhtmltopdf). See http://stackoverflow.com/questions/9604625/ .
# A suitable one is bundled with CamCOPS, so you shouldn't have to alter this
# default. Default is None, which usually ends up calling /usr/bin/wkhtmltopdf

{cp.WKHTMLTOPDF_FILENAME} =

# -----------------------------------------------------------------------------
# Login and session configuration
# -----------------------------------------------------------------------------

# {cp.SESSION_COOKIE_SECRET}: Secret used for HTTP cookie signing via Pyramid.
# Put something random in here and keep it secret.
# (When you make a CamCOPS demo config, the value shown is fresh and random.)

{cp.SESSION_COOKIE_SECRET} = camcops_autogenerated_secret_{session_cookie_secret}

# {cp.SESSION_TIMEOUT_MINUTES}: Time after which a session will expire (default 30).

{cp.SESSION_TIMEOUT_MINUTES} = 30

# {cp.PASSWORD_CHANGE_FREQUENCY_DAYS}: Force password changes (at webview login)
# with this frequency (0 for never). Note that password expiry will not prevent
# uploads from tablets, but when the user next logs on, a password change will
# be forced before they can do anything else.

{cp.PASSWORD_CHANGE_FREQUENCY_DAYS} = 0

# {cp.LOCKOUT_THRESHOLD}: Lock user accounts after every n login failures (default
# 10).

{cp.LOCKOUT_THRESHOLD} = 10

# {cp.LOCKOUT_DURATION_INCREMENT_MINUTES}: Account lockout time increment (default
# 10).
# Suppose {cp.LOCKOUT_THRESHOLD} = 10 and {cp.LOCKOUT_DURATION_INCREMENT_MINUTES} = 20.
# After the first 10 failures, the account will be locked for 20 minutes.
# After the next 10 failures, the account will be locked for 40 minutes.
# After the next 10 failures, the account will be locked for 60 minutes, and so
# on. Time and administrators can unlock accounts.

{cp.LOCKOUT_DURATION_INCREMENT_MINUTES} = 10

# {cp.DISABLE_PASSWORD_AUTOCOMPLETE}: if true, asks browsers not to autocomplete the
# password field on the main login page. The correct setting for maximum
# security is debated (don't cache passwords, versus allow a password manager
# so that users can use better/unique passwords). Default: true.
# Note that some browsers (e.g. Chrome v34 and up) may ignore this.

{cp.DISABLE_PASSWORD_AUTOCOMPLETE} = true

# -----------------------------------------------------------------------------
# Suggested filenames for saving PDFs from the web view
# -----------------------------------------------------------------------------
# Try with Chrome, Firefox. Internet Explorer may be less obliging.

# {cp.PATIENT_SPEC_IF_ANONYMOUS}: for anonymous tasks, this fixed string is 
# used as the patient descriptor (see also {cp.PATIENT_SPEC} below).
# Typically "anonymous".

{cp.PATIENT_SPEC_IF_ANONYMOUS} = anonymous

# {cp.PATIENT_SPEC}: string, into which substitutions will be made, that defines the
# {{{fse.PATIENT}}} element available for substitution into the *_FILENAME_SPEC
# variables (see below). Possible substitutions:
#
#   {{{pse.SURNAME}}}      : patient's surname in upper case
#   {{{pse.FORENAME}}}     : patient's forename in upper case
#   {{{pse.DOB}}}          : patient's date of birth (format "%Y-%m-%d", e.g.
#                    2013-07-24)
#   {{{pse.SEX}}}          : patient's sex (M, F, X)
#
#   {{{pse.IDSHORTDESC_PREFIX}1}} : short description of the relevant ID number, if that ID
#   {{{pse.IDSHORTDESC_PREFIX}2}}   number is not blank; otherwise blank
#   ...
#
#   {{{pse.IDNUM_PREFIX}1}}       : ID numbers
#   {{{pse.IDNUM_PREFIX}2}}
#   ...
#
#   {{{pse.ALLIDNUMS}}}    : all available ID numbers in "shortdesc-value" pairs joined
#                    by "_". For example, if ID numbers 1, 4, and 5 are
#                    non-blank, this would have the format
#                    {pse.IDSHORTDESC_PREFIX}1-{pse.IDNUM_PREFIX}1_{pse.IDSHORTDESC_PREFIX}4-{pse.IDNUM_PREFIX}4_{pse.IDSHORTDESC_PREFIX}5-{pse.IDNUM_PREFIX}5

{cp.PATIENT_SPEC} = {{{pse.SURNAME}}}_{{{pse.FORENAME}}}_{{{pse.ALLIDNUMS}}}

# {cp.TASK_FILENAME_SPEC}:
# {cp.TRACKER_FILENAME_SPEC}:
# {cp.CTV_FILENAME_SPEC}:
# Strings used for suggested filenames to save from the webview, for tasks,
# trackers, and clinical text views. Substitutions will be made to determine
# the filename to be used for each file. Possible substitutions:
#
#   {{{fse.PATIENT}}}   : Patient string. If the task is anonymous, this is
#                 {cp.PATIENT_SPEC_IF_ANONYMOUS}; otherwise, it is defined by
#                 {cp.PATIENT_SPEC} above.
#   {{{fse.CREATED}}}   : Date/time of task creation.  Dates/times are of the format
#                 "%Y-%m-%dT%H%M", e.g. 2013-07-24T2004. They are expressed in
#                 the timezone of creation (but without the timezone
#                 information for filename brevity).
#   {{{fse.NOW}}}       : Time of access/download (i.e. time now), in local timezone.
#   {{{fse.TASKTYPE}}}  : Base table name of the task (e.g. "phq9"). May contain an
#                 underscore. Blank for to trackers/CTVs.
#   {{{fse.SERVERPK}}}  : Server's primary key. (In combination with tasktype, this
#                 uniquely identifies not just a task but a version of that
#                 task.) Blank for trackers/CTVs.
#   {{{fse.FILETYPE}}}  : e.g. "pdf", "html", "xml" (lower case)
#   {{{fse.ANONYMOUS}}} : evaluates to {cp.PATIENT_SPEC_IF_ANONYMOUS} if anonymous,
#                 otherwise ""
#   ... plus all those substitutions applicable to {cp.PATIENT_SPEC}
#
# After these substitutions have been made, the entire filename is then
# processed to ensure that only characters generally acceptable to filenames
# are used (see convert_string_for_filename() in the CamCOPS source code).
# Specifically:
#
#   - Unicode converted to 7-bit ASCII (will mangle, e.g. removing accents)
#   - spaces converted to underscores
#   - characters are removed unless they are one of the following: all
#     alphanumeric characters (0-9, A-Z, a-z); '-'; '_'; '.'; and the
#     operating-system-specific directory separator (Python's os.sep, a forward
#     slash '/' on UNIX or a backslash '\' under Windows).

{cp.TASK_FILENAME_SPEC} = CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}}
{cp.TRACKER_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_tracker.{{filetype}}
{cp.CTV_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_clinicaltextview.{{filetype}}

# -----------------------------------------------------------------------------
# Debugging options
# -----------------------------------------------------------------------------
# Possible log levels are (case-insensitive): "debug", "info", "warn"
# (equivalent: "warning"), "error", and "critical" (equivalent: "fatal").

# {cp.WEBVIEW_LOGLEVEL}: Set the level of detail provided from the webview to the
# Apache server log. (Loglevel option; see above.)

{cp.WEBVIEW_LOGLEVEL} = info

# {cp.CLIENT_API_LOGLEVEL}: Set the log level for the tablet client database access
# script. (Loglevel option; see above.)

{cp.CLIENT_API_LOGLEVEL} = info

# {cp.ALLOW_INSECURE_COOKIES}: DANGEROUS option that removes the requirement that
# cookies be HTTPS (SSL) only.

{cp.ALLOW_INSECURE_COOKIES} = false

# =============================================================================
# List of HL7/file recipients, and then details for each one
# =============================================================================
# This section defines a list of recipients to which Health Level Seven (HL7)
# messages or raw files will be sent. Typically, you will send them by calling
# "camcops -7 CONFIGFILE" regularly from the system's /etc/crontab or other
# scheduling system. For example, a conventional /etc/crontab file has these
# fields:
#
#   minutes, hours, day_of_month, month, day_of_week, user, command
#
# so you could add a line like this to /etc/crontab:
#
#   * * * * *  root  camcops -7 /etc/camcops/MYCONFIG.conf
#
# ... and CamCOPS would run its exports once per minute. See "man 5 crontab"
# or http://en.wikipedia.org/wiki/Cron for more options.
#
# In this section, keys are ignored; values are section headings (one per
# recipient).

[{CONFIG_FILE_RECIPIENTLIST_SECTION}]

# Examples (commented out):

# recipient=recipient_A
# recipient=recipient_B

# =============================================================================
# Individual HL7/file recipient configurations
# =============================================================================
# Dates are YYYY-MM-DD, e.g. 2013-12-31, or blank

# Example (disabled because it's not in the list above)

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# First example
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

[recipient_A]

# {cpr.TYPE}: one of the following methods.
#   hl7
#       Sends HL7 messages across a TCP/IP network.
#   file
#       Writes files to a local filesystem.

{cpr.TYPE} = hl7

# -----------------------------------------------------------------------------
# Options applicable to HL7 messages and file transfers
# -----------------------------------------------------------------------------

# {cpr.GROUP_ID}: CamCOPS group to export from.
# HL7 messages are sent from one group at a time. Which group will this
# recipient definition use? (Note that you can just duplicate a recipient
# definition to export a second or subsequent group.)
# This is an integer.

{cpr.GROUP_ID} = 1

# {cpr.PRIMARY_IDNUM}: which ID number (1-8) should be considered the "internal"
# (primary) ID number? Must be specified for HL7 messages. May be blank for
# file transmission.

{cpr.PRIMARY_IDNUM} = 1

# {cpr.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY}: defines behaviour relating to the
# primary ID number (as defined by PRIMARY_IDNUM).
# - If true, no message sending will be attempted unless the {cpr.PRIMARY_IDNUM} is a
#   mandatory part of the finalizing policy (and if {cpr.FINALIZED_ONLY} is false,
#   also of the upload policy).
# - If false, messages will be sent, but ONLY FROM TASKS FOR WHICH THE
#   {cpr.PRIMARY_IDNUM} IS PRESENT; others will be ignored.
# - For file sending only, this will be ignored if {cpr.PRIMARY_IDNUM} is blank.
# - For file sending only, this setting does not apply to anonymous tasks,
#   whose behaviour is controlled by {cpr.INCLUDE_ANONYMOUS} (see below).

{cpr.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY} = true

# {cpr.START_DATE}: earliest date for which tasks will be sent. Assessed against the
# task's "when_created" field, converted to Universal Coordinated Time (UTC) --
# that is, this date is in UTC (beware if you are in a very different time
# zone). Blank to apply no start date restriction.

{cpr.START_DATE} =

# {cpr.END_DATE}: latest date for which tasks will be sent. In UTC. Assessed against
# the task's "when_created" field (converted to UTC). Blank to apply no end
# date restriction.

{cpr.END_DATE} =

# {cpr.FINALIZED_ONLY}: if true, only send tasks that are finalized (moved off their
# originating tablet and not susceptible to later modification). If false, also
# send tasks that are uploaded but not yet finalized (they will then be sent
# again if they are modified later).

{cpr.FINALIZED_ONLY} = true

# {cpr.TASK_FORMAT}: one of: pdf, html, xml

{cpr.TASK_FORMAT} = pdf

# {cpr.XML_FIELD_COMMENTS}: if {cpr.TASK_FORMAT} is xml, then {cpr.XML_FIELD_COMMENTS} determines
# whether field comments are included. These describe the meaning of each field
# so add to space requirements but provide more information for human readers.
# (Default true.)

{cpr.XML_FIELD_COMMENTS} = true

# -----------------------------------------------------------------------------
# Options applicable to HL7 only ({cpr.TYPE} = hl7)
# -----------------------------------------------------------------------------

# {cpr.HOST}: HL7 hostname or IP address

{cpr.HOST} = myhl7server.mydomain

# {cpr.PORT}: HL7 port (default 2575)

{cpr.PORT} = 2575

# {cpr.PING_FIRST}: if true, requires a successful ping to the server prior to
# sending HL7 messages. (Note: this is a TCP/IP ping, and tests that the
# machine is up, not that it is running an HL7 server.) Default: true.

{cpr.PING_FIRST} = true

# {cpr.NETWORK_TIMEOUT_MS}: network time (in milliseconds). Default: 10000.

{cpr.NETWORK_TIMEOUT_MS} = 10000

# {cpr.KEEP_MESSAGE}: keep a copy of the entire message in the databaase. Default is
# false. WARNING: may consume significant space in the database.

{cpr.KEEP_MESSAGE} = false

# {cpr.KEEP_REPLY}: keep a copy of the reply (e.g. acknowledgement) message received
# from the server. Default is false. WARNING: may consume significant space.

{cpr.KEEP_REPLY} = false

# {cpr.DIVERT_TO_FILE}: Override HOST/PORT options and send HL7 messages to this
# (single) file instead. Each messages is appended to the file. Default is
# blank (meaning network transmission will be used). This is a debugging
# option, allowing you to redirect HL7 messages to a file and inspect them.

{cpr.DIVERT_TO_FILE} =

# {cpr.TREAT_DIVERTED_AS_SENT}: Any messages that are diverted to a file (using
# {cpr.DIVERT_TO_FILE}) are treated as having been sent (thus allowing the file to
# mimic an HL7-receiving server that's accepting messages happily). If set to
# false (the default), a diversion will allow you to preview messages for
# debugging purposes without "swallowing" them. BEWARE, though: if you have
# an automatically scheduled job (for example, to send messages every minute)
# and you divert with this flag set to false, you will end up with a great many
# message attempts!

{cpr.TREAT_DIVERTED_AS_SENT} = false

# -----------------------------------------------------------------------------
# Options applicable to file transfers only (TYPE = file)
# -----------------------------------------------------------------------------

# {cpr.INCLUDE_ANONYMOUS}: include anonymous tasks.
# - Note that anonymous tasks cannot be sent via HL7; the HL7 specification is
#   heavily tied to identification.
# - Note also that this setting operates independently of the
#   {cpr.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY} setting.

{cpr.INCLUDE_ANONYMOUS} = true

# {cpr.PATIENT_SPEC_IF_ANONYMOUS}: for anonymous tasks, this string is used as the
# patient descriptor (see also {cpr.PATIENT_SPEC}, {cpr.FILENAME_SPEC} below). Typically
# "anonymous".

{cpr.PATIENT_SPEC_IF_ANONYMOUS} = anonymous

# {cpr.PATIENT_SPEC}: string, into which substitutions will be made, that defines the
# {{patient}} element available for substitution into the {cpr.FILENAME_SPEC} (see
# below). Possible substitutions: as for {cpr.PATIENT_SPEC} in the main
# "[{CONFIG_FILE_MAIN_SECTION}]" section of the configuration file (see above).

{cpr.PATIENT_SPEC} = {{surname}}_{{forename}}_{{idshortdesc1}}{{idnum1}}

# {cpr.FILENAME_SPEC}: string into which substitutions will be made to determine the
# filename to be used for each file. Possible substitutions: as for
# {cp.PATIENT_SPEC} in the main "[{CONFIG_FILE_MAIN_SECTION}]" section of the configuration
# file (see above).

{cpr.FILENAME_SPEC} = /my_nfs_mount/mypath/CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}}

# {cpr.MAKE_DIRECTORY}: make the directory if it doesn't already exist. Default is
# false.

{cpr.MAKE_DIRECTORY} = true

# {cpr.OVERWRITE_FILES}: whether or not to attempt overwriting existing files of the
# same name (default false). There is a DANGER of inadvertent data loss if you
# set this to true. (Needing to overwrite a file suggests that your filenames
# are not task-unique; try ensuring that both the {{tasktype}} and {{serverpk}}
# attributes are used in the filename.)

{cpr.OVERWRITE_FILES} = false

# {cpr.RIO_METADATA}: whether or not to export a metadata file for Servelec's RiO
# (default false).
# Details of this file format are in cc_task.py / Task.get_rio_metadata().
# The metadata filename is that of its associated file, but with the extension
# replaced by ".metadata" (e.g. X.pdf is accompanied by X.metadata).
# If {cpr.RIO_METADATA} is true, the following options also apply:
#   {cpr.RIO_IDNUM}: which of the ID numbers (as above) is the RiO ID?
#   {cpr.RIO_UPLOADING_USER}: username for the uploading user (maximum of 10
#       characters)
#   {cpr.RIO_DOCUMENT_TYPE}: document type as defined in the receiving RiO system.
#       This is a code that maps to a human-readable document type; for
#       example, the code "APT" might map to "Appointment Letter". Typically we
#       might want a code that maps to "Clinical Correspondence", but the code
#       will be defined within the local RiO system configuration.

{cpr.RIO_METADATA} = false
{cpr.RIO_IDNUM} = 2
{cpr.RIO_UPLOADING_USER} = CamCOPS
{cpr.RIO_DOCUMENT_TYPE} = CC

# {cpr.SCRIPT_AFTER_FILE_EXPORT}: filename of a shell script or other executable to
# run after file export is complete. You might use this script, for example, to
# move the files to a different location (such as across a network). If the
# parameter is blank, no script will be run. If no files are exported, the
# script will not be run.
# - Parameters passed to the script: a list of all the filenames exported.
#   (This includes any RiO metadata filenames.)
# - WARNING: the script will execute with the same permissions as the instance
#   of CamCOPS that's doing the export (so, for example, if you run CamCOPS
#   from your /etc/crontab as root, then this script will be run as root; that
#   can pose a risk!).
# - The script executes while the export lock is still held by CamCOPS (i.e.
#   further HL7/file transfers won't be started until the script(s) is/are
#   complete).
# - If the script fails, an error message is recorded, but the file transfer is
#   still considered to have been made (CamCOPS has done all it can and the
#   responsibility now lies elsewhere).
# - Example test script: suppose this is /usr/local/bin/print_arguments:
#       #!/bin/bash
#       for f in $$@
#       do
#           echo "CamCOPS has just exported this file: $$f"
#       done
#   ... then you could set:
#       {cpr.SCRIPT_AFTER_FILE_EXPORT} = /usr/local/bin/print_arguments

{cpr.SCRIPT_AFTER_FILE_EXPORT} =

    """.format(  # noqa
        cp=ConfigParamMain,
        cpr=ConfigParamRecipient,
        CONFIG_FILE_MAIN_SECTION=CONFIG_FILE_MAIN_SECTION,
        CONFIG_FILE_RECIPIENTLIST_SECTION=CONFIG_FILE_RECIPIENTLIST_SECTION,
        db_echo=db_echo,
        db_url=db_url,
        DEFAULT_DB_NAME=DEFAULT_DB_NAME,
        DEFAULT_DB_PASSWORD=DEFAULT_DB_PASSWORD,
        DEFAULT_DB_USER=DEFAULT_DB_USER,
        extra_strings_spec=extra_strings_spec,
        hl7_lockfile_stem=hl7_lockfile_stem,
        lock_dir=lock_dir,
        static_dir=static_dir,
        summary_table_lock_file_stem=summary_table_lock_file_stem,
        DUMMY_INSTITUTION_URL=DUMMY_INSTITUTION_URL,
        fse=FilenameSpecElement,
        now=str(Pendulum.now()),
        pse=PatientSpecElementForFilename,
        session_cookie_secret=session_cookie_secret,
        version=CAMCOPS_SERVER_VERSION_STRING,
    )


# =============================================================================
# Demo configuration files, other than the CamCOPS config file itself
# =============================================================================

DEFAULT_INTERNAL_PORT = 8000
DEFAULT_SOCKET_FILENAME = "/tmp/.camcops.sock"


def get_demo_supervisor_config(
        specimen_internal_port: int = DEFAULT_INTERNAL_PORT,
        specimen_socket_file: str = DEFAULT_SOCKET_FILENAME) -> str:
    return """
# =============================================================================
# Demonstration 'supervisor' config file for CamCOPS.
# Created by CamCOPS version {version} at {now}.
# =============================================================================
# - Supervisor is a system for controlling background processes running on
#   UNIX-like operating systems. See:
#       http://supervisord.org
# - On Ubuntu systems, you would typically install supervisor with
#       sudo apt install supervisor
#   and then save this file as
#       /etc/supervisor/conf.d/camcops.conf
#
# - IF YOU EDIT THIS FILE, run:
#       sudo service supervisor restart
# - TO MONITOR SUPERVISOR, run:
#       sudo supervisorctl status
#   ... or just "sudo supervisorctl" for an interactive prompt.
#
# - TO ADD MORE CAMCOPS INSTANCES, first consider whether you wouldn't be 
#   better off just adding groups. If you decide you want a completely new
#   instance, make a copy of the [program:camcops] section, renaming the copy, 
#   and change the following:
#   - the --config switch;
#   - the port or socket;
#   - the log files.
#   Then make the main web server point to the copy as well.
#
# NOTES ON THE SUPERVISOR CONFIG FILE AND ENVIRONMENT:
# - Indented lines are treated as continuation (even in commands; no need for
#   end-of-line backslashes or similar).
# - The downside of that is that indented comment blocks can join onto your
#   commands! Beware that.
# - You can't put quotes around the directory variable
#   http://stackoverflow.com/questions/10653590
# - Python programs that are installed within a Python virtual environment 
#   automatically use the virtualenv's copy of Python via their shebang; you do
#   not need to specify that by hand, nor the PYTHONPATH.
# - The "environment" setting sets the OS environment. The "--env" parameter
#   to gunicorn, if you use it, sets the WSGI environment.

[program:camcops]

command = {CAMCOPS_EXECUTABLE}
    serve_gunicorn
    --config /etc/camcops/camcops.conf
    --unix_domain_socket {specimen_socket_file}
    --trusted_proxy_headers 
        HTTP_X_FORWARDED_HOST 
        HTTP_X_FORWARDED_SERVER 
        HTTP_X_FORWARDED_PORT 
        HTTP_X_FORWARDED_PROTO 
        HTTP_X_SCRIPT_NAME

# To run via a TCP socket, use e.g.:
#   --host 127.0.0.1 --port {specimen_internal_port}
# To run via a UNIX domain socket, use e.g.
#   --unix_domain_socket {specimen_socket_file} 

directory = {CAMCOPS_SERVER_DIRECTORY}

environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"

# MPLCONFIGDIR specifies a cache directory for matplotlib, which greatly
# speeds up its subsequent loading. 

user = www-data

# ... Ubuntu: typically www-data
# ... CentOS: typically apache

stdout_logfile = /var/log/supervisor/camcops_out.log
stderr_logfile = /var/log/supervisor/camcops_err.log

autostart = true
autorestart = true
startsecs = 30
stopwaitsecs = 60

    """.format(
        CAMCOPS_EXECUTABLE=CAMCOPS_EXECUTABLE,
        CAMCOPS_SERVER_DIRECTORY=CAMCOPS_SERVER_DIRECTORY,
        LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR=LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR,
        now=str(Pendulum.now()),
        specimen_internal_port=specimen_internal_port,
        specimen_socket_file=specimen_socket_file,
        version=CAMCOPS_SERVER_VERSION_STRING,
    )


def get_demo_apache_config(
        urlbase: str = "/camcops",
        specimen_internal_port: int = DEFAULT_INTERNAL_PORT,
        specimen_socket_file: str = DEFAULT_SOCKET_FILENAME) -> str:
    return """
    # Demonstration Apache config file section for CamCOPS.
    # Created by CamCOPS version {version} at {now}.
    #
    # Under Ubuntu, the Apache config will be somewhere in /etc/apache2/
    # Under CentOS, the Apache config will be somewhere in /etc/httpd/
    #
    # This section should go within the <VirtualHost> directive for the secure
    # (SSL, HTTPS) part of the web site. 

<VirtualHost *:443>
    # ...

    # =========================================================================
    # CamCOPS
    # =========================================================================
    # Apache operates on the principle that the first match wins. So, if we
    # want to serve CamCOPS but then override some of its URLs to serve static
    # files faster, we define the static stuff first.

        # ---------------------------------------------------------------------
        # 1. Serve static files
        # ---------------------------------------------------------------------
        # a) offer them at the appropriate URL
        # b) provide permission
        # c) disable ProxyPass for static files

        # Change this: aim the alias at your own institutional logo.
        
    Alias {urlbase}/static/logo_local.png {STATIC_ROOT_DIR}/logo_local.png
    
        # We move from more specific to less specific aliases; the first match
        # takes precedence. (Apache will warn about conflicting aliases if
        # specified in a wrong, less-to-more-specific, order.)

    Alias {urlbase}/static/ {STATIC_ROOT_DIR}/

    <Directory {STATIC_ROOT_DIR}>
        Require all granted
        
        # ... for old Apache version (e.g. 2.2), use instead:
        # Order allow,deny
        # Allow from all
    </Directory>
    
        # Don't ProxyPass the static files; we'll serve them via Apache.
        
    ProxyPassMatch ^{urlbase}/static/ !

        # ---------------------------------------------------------------------
        # 2. Proxy requests to the CamCOPS web server and back; allow access
        # ---------------------------------------------------------------------
        # ... either via an internal TCP/IP port (e.g. 1024 or higher, and NOT
        #     accessible to users);
        # ... or, better, via a Unix socket, e.g. /tmp/.camcops.sock
        #
        # NOTES
        # - When you ProxyPass {urlbase}, you should browse to
        #       https://YOURSITE{urlbase}
        #   and point your tablet devices to
        #       https://YOURSITE{urlbase}{MASTER_ROUTE_CLIENT_API}
        # - Don't specify trailing slashes for the ProxyPass and 
        #   ProxyPassReverse directives.
        #   If you do, http://host/camcops will fail though
        #              http://host/camcops/ will succeed.
        # - Ensure that you put the CORRECT PROTOCOL (http, https) in the rules
        #   below.
        # - For ProxyPass options, see https://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass
        #   ... including "retry=0" to stop Apache disabling the connection for
        #       a while on failure.
        # - Using a socket
        #   - this requires Apache 2.4.9, and passes after the '|' character a
        #     URL that determines the Host: value of the request; see
        #     https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypass
        # - CamCOPS MUST BE TOLD about its location and protocol, because that
        #   information is critical for synthesizing URLs, but is stripped out
        #   by the reverse proxy system. There are two ways:
        #   (i)  specifying headers or WSGI environment variables, such as
        #        the HTTP(S) headers X-Forwarded-Proto and X-Script-Name below
        #        (CamCOPS is aware of these);
        #   (ii) specifying options to "camcops serve", including
        #           --script_name
        #           --scheme
        #        and optionally
        #           --server
        #
        # So:
        #
        # ~~~~~~~~~~~~~~~~~
        # (a) Reverse proxy
        # ~~~~~~~~~~~~~~~~~
        #
        # PORT METHOD
        # Note the use of "http" (reflecting the backend), not https (like the
        # front end).
        
    ProxyPass {urlbase} http://127.0.0.1:{specimen_internal_port} retry=0
    ProxyPassReverse {urlbase} http://127.0.0.1:{specimen_internal_port} retry=0

        # UNIX SOCKET METHOD (Apache 2.4.9 and higher)
        #
        # The general syntax is:
        #   ProxyPass /URL_USER_SEES unix:SOCKETFILE|PROTOCOL://HOST/EXTRA_URL_FOR_BACKEND retry=0
        # Note that:
        #   - the protocol should be http, not https (Apache deals with the
        #     HTTPS part and passes HTTP on)
        #   - the EXTRA_URL_FOR_BACKEND needs to be (a) unique for each
        #     instance or Apache will use a single worker for multiple
        #     instances, and (b) blank for the backend's benefit. Since those
        #     two conflict when there's >1 instance, there's a problem.
        #   - Normally, HOST is given as localhost. It may be that this problem
        #     is solved by using a dummy unique value for HOST:
        #     https://bz.apache.org/bugzilla/show_bug.cgi?id=54101#c1
        #
        # If your Apache version is too old, you will get the error
        #   "AH00526: Syntax error on line 56 of /etc/apache2/sites-enabled/SOMETHING:
        #    ProxyPass URL must be absolute!"
        # On Ubuntu, if your Apache is too old, you could use
        #   sudo add-apt-repository ppa:ondrej/apache2
        # ... details at https://launchpad.net/~ondrej/+archive/ubuntu/apache2
        #
        # If you get this error:
        #   AH01146: Ignoring parameter 'retry=0' for worker 'unix:/tmp/.camcops_gunicorn.sock|https://localhost' because of worker sharing
        #   https://wiki.apache.org/httpd/ListOfErrors
        # ... then your URLs are overlapping and should be redone or sorted:
        #   http://httpd.apache.org/docs/2.4/mod/mod_proxy.html#workers
        # The part that must be unique for each instance, with no part a
        # leading substring of any other, is THIS_BIT in:
        #   ProxyPass /URL_USER_SEES unix:SOCKETFILE|https://localhost/THIS_BIT retry=0
        #
        # If you get an error like this:
        #   AH01144: No protocol handler was valid for the URL /SOMEWHERE. If you are using a DSO version of mod_proxy, make sure the proxy submodules are included in the configuration using LoadModule.
        # Then do this:
        #   sudo a2enmod proxy proxy_http
        #   sudo apache2ctl restart
        #
        # If you get an error like this:
        #   ... [proxy_http:error] [pid 32747] (103)Software caused connection abort: [client 109.151.49.173:56898] AH01102: error reading status line from remote server httpd-UDS:0
        #       [proxy:error] [pid 32747] [client 109.151.49.173:56898] AH00898: Error reading from remote server returned by /camcops_bruhl/webview
        # then check you are specifying http://, not https://, in the ProxyPass
        #
        # Other information sources:
        #   https://emptyhammock.com/projects/info/pyweb/webconfig.html

    # ProxyPass /camcops unix:{specimen_socket_file}|https://dummy1/ retry=0
    # ProxyPassReverse /camcops unix:{specimen_socket_file}|https://dummy1/ retry=0

        # ~~~~~~~~~~~~~~~~~~~~~~~~~
        # (b) Allow proxy over SSL.
        # ~~~~~~~~~~~~~~~~~~~~~~~~~
        # Without this, you will get errors like:
        #   ... SSL Proxy requested for wombat:443 but not enabled [Hint: SSLProxyEngine]
        #   ... failed to enable ssl support for 0.0.0.0:0 (httpd-UDS)
        
    SSLProxyEngine on

    <Location /camcops>

            # ~~~~~~~~~~~~~~~~    
            # (c) Allow access
            # ~~~~~~~~~~~~~~~~
            
        Require all granted

        # ... for old Apache version (e.g. 2.2), use instead:
        # Order allow,deny
        # Allow from all

            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            # (d) Tell the proxied application that we are using HTTPS, and
            #     where the application is installed
            # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            #     ... https://stackoverflow.com/questions/16042647
            #
            # EITHER enable mod_headers (e.g. "sudo a2enmod headers") and set:
            
        RequestHeader set X-Forwarded-Proto https
        RequestHeader set X-Script-Name {urlbase}
        
            # and call CamCOPS like:
            #
            # camcops serve_gunicorn \\
            #       --config SOMECONFIG \\
            #       --trusted_proxy_headers \\
            #           HTTP_X_FORWARDED_HOST \\
            #           HTTP_X_FORWARDED_SERVER \\
            #           HTTP_X_FORWARDED_PORT \\
            #           HTTP_X_FORWARDED_PROTO \\
            #           HTTP_X_SCRIPT_NAME
            #
            # (X-Forwarded-For, X-Forwarded-Host, and X-Forwarded-Server are
            # supplied by Apache automatically)
            #
            # ... OR specify those options by hand in the CamCOPS command.
            
    </Location>

        # ---------------------------------------------------------------------
        # 3. For additional instances
        # ---------------------------------------------------------------------
        # (a) duplicate section 1 above, editing the base URL and CamCOPS
        #     connection (socket/port);
        # (b) you will also need to create an additional CamCOPS instance,
        #     as above;
        # (c) add additional static aliases (in section 2 above).
        #
        # HOWEVER, consider adding more CamCOPS groups, rather than creating
        # additional instances; the former are *much* easier to administer!


    #==========================================================================
    # SSL security (for HTTPS)
    #==========================================================================

        # You will also need to install your SSL certificate; see the 
        # instructions that came with it. You get a certificate by creating a 
        # certificate signing request (CSR). You enter some details about your 
        # site, and a software tool makes (1) a private key, which you keep 
        # utterly private, and (2) a CSR, which you send to a Certificate 
        # Authority (CA) for signing. They send back a signed certificate, and 
        # a chain of certificates leading from yours to a trusted root CA.
        #
        # You can create your own (a 'snake-oil' certificate), but your tablets
        # and browsers will not trust it, so this is a bad idea.
        #
        # Once you have your certificate: edit and uncomment these lines:

    # SSLEngine on

    # SSLCertificateKeyFile /etc/ssl/private/my.private.key

        # ... a private file that you made before creating the certificate
        # request, and NEVER GAVE TO ANYBODY, and NEVER WILL (or your
        # security is broken and you need a new certificate).

    # SSLCertificateFile /etc/ssl/certs/my.public.cert

        # ... signed and supplied to you by the certificate authority (CA),
        # from the public certificate you sent to them.

    # SSLCertificateChainFile /etc/ssl/certs/my-institution.ca-bundle

        # ... made from additional certificates in a chain, supplied to you by
        # the CA. For example, mine is univcam.ca-bundle, made with the
        # command:
        #
        # cat TERENASSLCA.crt UTNAddTrustServer_CA.crt AddTrustExternalCARoot.crt > univcam.ca-bundle

</VirtualHost>

    """.format(  # noqa
        MASTER_ROUTE_CLIENT_API=MASTER_ROUTE_CLIENT_API,
        now=str(Pendulum.now()),
        specimen_internal_port=specimen_internal_port,
        specimen_socket_file=specimen_socket_file,
        STATIC_ROOT_DIR=STATIC_ROOT_DIR,
        urlbase=urlbase,
        version=CAMCOPS_SERVER_VERSION_STRING,
    )


def get_demo_mysql_create_db() -> str:
    return """
# First, from the Linux command line, log in to MySQL as root:

mysql --host=127.0.0.1 --port=3306 --user=root --password
# ... or the usual short form: mysql -u root -p

# Create the database:

CREATE DATABASE {DEFAULT_DB_NAME};

# Ideally, create another user that only has access to the CamCOPS database.
# You should do this, so that you don’t use the root account unnecessarily.

GRANT ALL PRIVILEGES ON {DEFAULT_DB_NAME}.* TO '{DEFAULT_DB_USER}'@'localhost' IDENTIFIED BY '{DEFAULT_DB_PASSWORD}';

# For future use: if you plan to explore your database directly for analysis,
# you may want to create a read-only user. Though it may not be ideal (check:
# are you happy the user can see the audit trail?), you can create a user with
# read-only access to the entire database like this:

GRANT SELECT {DEFAULT_DB_NAME}.* TO '{DEFAULT_DB_READONLY_USER}'@'localhost' IDENTIFIED BY '{DEFAULT_DB_READONLY_PASSWORD}';

# All done. Quit MySQL:

exit
    """.format(  # noqa
        DEFAULT_DB_NAME=DEFAULT_DB_NAME,
        DEFAULT_DB_USER=DEFAULT_DB_USER,
        DEFAULT_DB_PASSWORD=DEFAULT_DB_PASSWORD,
        DEFAULT_DB_READONLY_USER=DEFAULT_DB_READONLY_USER,
        DEFAULT_DB_READONLY_PASSWORD=DEFAULT_DB_READONLY_PASSWORD,
    )


def get_demo_mysql_dump_script() -> str:
    return """#!/bin/bash

# Minimal simple script to dump all current MySQL databases.
# This file must be READABLE ONLY BY ROOT (or equivalent, backup)!
# The password is in cleartext.
# Once you have copied this file and edited it, perform:
#     sudo chown root:root <filename>
#     sudo chmod 700 <filename>
# Then you can add it to your /etc/crontab for regular execution.

BACKUPDIR='/var/backups/mysql'
BACKUPFILE='all_my_mysql_databases.sql'
USERNAME='root'  # MySQL username
PASSWORD='PPPPPP_REPLACE_ME'  # MySQL password

# Make directory unless it exists already:

mkdir -p $BACKUPDIR

# Dump the database:

mysqldump -u $USERNAME -p$PASSWORD --all-databases --force > $BACKUPDIR/$BACKUPFILE

# Make sure the backups (which may contain sensitive information) are only
# readable by the 'backup' user group:

cd $BACKUPDIR
chown -R backup:backup *
chmod -R o-rwx *
chmod -R ug+rw *

    """  # noqa


# =============================================================================
# Configuration class. (It gets cached on a per-process basis.)
# =============================================================================

[docs]class CamcopsConfig(object): """ Class representing the config. """ def __init__(self, config_filename: str) -> None: """Initialize from config file.""" cp = ConfigParamMain # --------------------------------------------------------------------- # Open config file # --------------------------------------------------------------------- self.camcops_config_file = config_filename if not self.camcops_config_file: raise AssertionError("{} not specified".format(ENVVAR_CONFIG_FILE)) log.info("Reading from {}", self.camcops_config_file) config = configparser.ConfigParser() with codecs.open(self.camcops_config_file, "r", "utf8") as file: config.read_file(file) # --------------------------------------------------------------------- # Read from the config file: 1. Most stuff, in alphabetical order # --------------------------------------------------------------------- section = CONFIG_FILE_MAIN_SECTION self.allow_insecure_cookies = get_config_parameter_boolean( config, section, cp.ALLOW_INSECURE_COOKIES, False) self.camcops_logo_file_absolute = get_config_parameter( config, section, cp.CAMCOPS_LOGO_FILE_ABSOLUTE, str, DEFAULT_CAMCOPS_LOGO_FILE) self.ctv_filename_spec = get_config_parameter( config, section, cp.CTV_FILENAME_SPEC, str, None) self.db_url = config.get(section, cp.DB_URL) # ... no default: will fail if not provided self.db_echo = get_config_parameter_boolean( config, section, cp.DB_ECHO, False) self.client_api_loglevel = get_config_parameter_loglevel( config, section, cp.CLIENT_API_LOGLEVEL, logging.INFO) logging.getLogger("camcops_server.cc_modules.client_api")\ .setLevel(self.client_api_loglevel) # ... MUTABLE GLOBAL STATE (if relatively unimportant); *** fix self.disable_password_autocomplete = get_config_parameter_boolean( config, section, cp.DISABLE_PASSWORD_AUTOCOMPLETE, True) self.extra_string_files = get_config_parameter_multiline( config, section, cp.EXTRA_STRING_FILES, []) self.hl7_lockfile = get_config_parameter( config, section, cp.HL7_LOCKFILE, str, None) self.introspection = get_config_parameter_boolean( config, section, cp.INTROSPECTION, True) self.local_institution_url = get_config_parameter( config, section, cp.LOCAL_INSTITUTION_URL, str, DEFAULT_LOCAL_INSTITUTION_URL) self.local_logo_file_absolute = get_config_parameter( config, section, cp.LOCAL_LOGO_FILE_ABSOLUTE, str, DEFAULT_LOCAL_LOGO_FILE) self.lockout_threshold = get_config_parameter( config, section, cp.LOCKOUT_THRESHOLD, int, DEFAULT_LOCKOUT_THRESHOLD) self.lockout_duration_increment_minutes = get_config_parameter( config, section, cp.LOCKOUT_DURATION_INCREMENT_MINUTES, int, DEFAULT_LOCKOUT_DURATION_INCREMENT_MINUTES) self.password_change_frequency_days = get_config_parameter( config, section, cp.PASSWORD_CHANGE_FREQUENCY_DAYS, int, DEFAULT_PASSWORD_CHANGE_FREQUENCY_DAYS) self.patient_spec_if_anonymous = get_config_parameter( config, section, cp.PATIENT_SPEC_IF_ANONYMOUS, str, "anonymous") self.patient_spec = get_config_parameter( config, section, cp.PATIENT_SPEC, str, None) # currently not configurable, but easy to add in the future: self.plot_fontsize = DEFAULT_PLOT_FONTSIZE # self.send_analytics = get_config_parameter_boolean( # config, section, "SEND_ANALYTICS", True) session_timeout_minutes = get_config_parameter( config, section, cp.SESSION_TIMEOUT_MINUTES, int, DEFAULT_TIMEOUT_MINUTES) self.session_cookie_secret = get_config_parameter( config, section, cp.SESSION_COOKIE_SECRET, str, None) self.session_timeout = datetime.timedelta( minutes=session_timeout_minutes) self.summary_tables_lockfile = get_config_parameter( config, section, cp.SUMMARY_TABLES_LOCKFILE, str, None) self.task_filename_spec = get_config_parameter( config, section, cp.TASK_FILENAME_SPEC, str, None) self.tracker_filename_spec = get_config_parameter( config, section, cp.TRACKER_FILENAME_SPEC, str, None) self.webview_loglevel = get_config_parameter_loglevel( config, section, cp.WEBVIEW_LOGLEVEL, logging.INFO) logging.getLogger().setLevel(self.webview_loglevel) # root logger # ... MUTABLE GLOBAL STATE (if relatively unimportant) *** fix self.wkhtmltopdf_filename = get_config_parameter( config, section, cp.WKHTMLTOPDF_FILENAME, str, None) # --------------------------------------------------------------------- # Read from the config file: 2. HL7 section # --------------------------------------------------------------------- # http://stackoverflow.com/questions/335695/lists-in-configparser self.hl7_recipient_defs = [] # type: List[RecipientDefinition] try: hl7_items = config.items(CONFIG_FILE_RECIPIENTLIST_SECTION) for key, recipientdef_name in hl7_items: log.debug("HL7 config: key={}, recipientdef_name={}", key, recipientdef_name) h = RecipientDefinition(config=config, section=recipientdef_name) self.hl7_recipient_defs.append(h) except configparser.NoSectionError: log.info("No config file section [{}]", CONFIG_FILE_RECIPIENTLIST_SECTION) # --------------------------------------------------------------------- # Built from the preceding: # --------------------------------------------------------------------- self.introspection_files = [] # type: List[IntrospectionFileDetails] if self.introspection: # All introspection starts at INTROSPECTION_BASE_DIRECTORY rootdir = INTROSPECTION_BASE_DIRECTORY for dir_, subdirs, files in os.walk(rootdir): if dir_ == rootdir: pretty_dir = '' else: pretty_dir = os.path.relpath(dir_, rootdir) for filename in files: basename, ext = os.path.splitext(filename) if ext not in INTROSPECTABLE_EXTENSIONS: continue fullpath = os.path.join(dir_, filename) prettypath = os.path.join(pretty_dir, filename) self.introspection_files.append( IntrospectionFileDetails( fullpath=fullpath, prettypath=prettypath, ext=ext ) ) self.introspection_files = sorted( self.introspection_files, key=operator.attrgetter("prettypath")) # --------------------------------------------------------------------- # More validity checks # --------------------------------------------------------------------- if not self.patient_spec_if_anonymous: raise RuntimeError( "Blank PATIENT_SPEC_IF_ANONYMOUS in [server] " "section of config file") if not self.patient_spec: raise RuntimeError( "Missing/blank PATIENT_SPEC in [server] section" " of config file") if not self.session_cookie_secret: raise RuntimeError( "Invalid or missing SESSION_COOKIE_SECRET " "setting in [server] section of config file") if not self.task_filename_spec: raise RuntimeError("Missing/blank TASK_FILENAME_SPEC in " "[server] section of config file") if not self.tracker_filename_spec: raise RuntimeError("Missing/blank TRACKER_FILENAME_SPEC in " "[server] section of config file") if not self.ctv_filename_spec: raise RuntimeError("Missing/blank CTV_FILENAME_SPEC in " "[server] section of config file") # --------------------------------------------------------------------- # Other attributes # --------------------------------------------------------------------- self._sqla_engine = None
[docs] def get_sqla_engine(self) -> Engine: """ I was previously misinterpreting the appropriate scope of an Engine. I thought: create one per request. But the Engine represents the connection *pool*. So if you create them all the time, you get e.g. a 'Too many connections' error. "The appropriate scope is once per [database] URL per application, at the module level." https://groups.google.com/forum/#!topic/sqlalchemy/ZtCo2DsHhS4 https://stackoverflow.com/questions/8645250/how-to-close-sqlalchemy-connection-in-mysql Now, our CamcopsConfig instance is cached, so there should be one of them overall. See get_config() below. Therefore, making the engine a member of this class should do the trick, whilst avoiding global variables. """ if self._sqla_engine is None: self._sqla_engine = create_engine( self.db_url, echo=self.db_echo, pool_pre_ping=True, # pool_size=0, # no limit (for parallel testing, which failed) ) log.debug("Created SQLAlchemy engine for URL {}".format( get_safe_url_from_engine(self._sqla_engine))) return self._sqla_engine
@property @cache_region_static.cache_on_arguments(function_key_generator=fkg) def get_all_table_names(self) -> List[str]: engine = self.get_sqla_engine() return get_table_names(engine=engine) @contextlib.contextmanager def get_dbsession_context(self) -> Generator[SqlASession, None, None]: engine = self.get_sqla_engine() maker = sessionmaker(bind=engine) dbsession = maker() # type: SqlASession # noinspection PyBroadException try: yield dbsession dbsession.commit() except Exception: dbsession.rollback() finally: dbsession.close() def _assert_valid_database_engine(self) -> None: """ Excluding invalid backend database types. Specifically, SQL Server versions before 2008 don't support timezones and we need that. """ engine = self.get_sqla_engine() if not is_sqlserver(engine): return assert is_sqlserver_2008_or_later(engine), ( "If you use Microsoft SQL Server as the back-end database for a " "CamCOPS server, it must be at least SQL Server 2008. Older " "versions do not have time zone awareness." ) def _assert_database_is_at_head(self) -> None: current, head = get_current_and_head_revision( database_url=self.db_url, alembic_config_filename=ALEMBIC_CONFIG_FILENAME, alembic_base_dir=ALEMBIC_BASE_DIR, version_table=ALEMBIC_VERSION_TABLE, ) if current == head: log.debug("Database is at correct (head) revision of {}", current) else: msg = ( "Database structure is at version {} but should be at " "version {}. CamCOPS will not start. Please use the " "'upgrade_db' command to fix this.".format(current, head)) log.critical(msg) raise RuntimeError(msg) def assert_database_ok(self) -> None: self._assert_valid_database_engine() self._assert_database_is_at_head()
# ============================================================================= # Get config filename from an appropriate environment (WSGI or OS) # ============================================================================= def get_config_filename_from_os_env() -> str: # We do NOT trust the WSGI environment for this. config_filename = os.environ.get(ENVVAR_CONFIG_FILE) if not config_filename: raise AssertionError( "OS environment did not provide the required " "environment variable {}".format(ENVVAR_CONFIG_FILE)) return config_filename # ============================================================================= # Cached instances # ============================================================================= @cache_region_static.cache_on_arguments(function_key_generator=fkg) def get_config(config_filename: str) -> CamcopsConfig: return CamcopsConfig(config_filename) # ============================================================================= # Get default config # ============================================================================= def get_default_config_from_os_env() -> CamcopsConfig: return get_config(get_config_filename_from_os_env()) # ============================================================================= # NOTES # ============================================================================= TO_BE_IMPLEMENTED_AS_COMMAND_LINE_SWITCH = """ # ----------------------------------------------------------------------------- # Export to a staging database for CRIS, CRATE, or similar anonymisation # software (anonymisation staging database; ANONSTAG) # ----------------------------------------------------------------------------- {cp.ANONSTAG_DB_URL} = {anonstag_db_url} {cp.EXPORT_CRIS_DATA_DICTIONARY_TSV_FILE} = /tmp/camcops_cris_dd_draft.tsv *** Note that we must check that the anonymisation staging database doesn't have the same URL as the main one (or "isn't the same one" in a more robust fashion)! Because this is so critical, probably best to: - require a completely different database name - ensure no table names overlap (e.g. add a prefix) """