Coverage for cc_modules/cc_sqlalchemy.py: 61%
79 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:14 +0000
1#!/usr/bin/env python
3"""
4camcops_server/cc_modules/cc_sqlalchemy.py
6===============================================================================
8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry.
9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk).
11 This file is part of CamCOPS.
13 CamCOPS is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
18 CamCOPS is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
26===============================================================================
28**SQLAlchemy helper functions and constants.**
30We define our metadata ``Base`` here, and things like our index naming
31convention and MySQL table formats.
33A few random notes:
35- SQLAlchemy will automatically warn about clashing columns:
37 .. :code-block:: python
39 from sqlalchemy import Column, Integer
40 from sqlalchemy.ext.declarative import declarative_base
42 Base = declarative_base()
44 class Thing(Base):
45 __tablename__ = "thing"
46 a = Column("a", Integer, primary_key=True)
47 b = Column("b", Integer)
48 c = Column("b", Integer) # produces a warning:
50 .. code-block:: none
52 SAWarning: On class 'Thing', Column object 'b' named directly multiple
53 times, only one will be used: b, c. Consider using orm.synonym instead
55"""
57from abc import ABCMeta
58from io import StringIO
59import logging
60import sqlite3
61from typing import Any
63from cardinal_pythonlib.logs import BraceStyleAdapter
64from cardinal_pythonlib.sqlalchemy.dialect import (
65 get_dialect_from_name,
66 SqlaDialectName,
67)
68from cardinal_pythonlib.sqlalchemy.dump import dump_ddl
69from cardinal_pythonlib.sqlalchemy.session import (
70 make_sqlite_url,
71 SQLITE_MEMORY_URL,
72)
73from pendulum import DateTime as Pendulum
75from sqlalchemy.engine import create_engine
76from sqlalchemy.engine.base import Engine
77from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
78from sqlalchemy.ext.mutable import Mutable
79from sqlalchemy.schema import CreateTable
80from sqlalchemy.sql.schema import MetaData, Table
82from camcops_server.cc_modules.cc_cache import cache_region_static, fkg
84log = BraceStyleAdapter(logging.getLogger(__name__))
87# =============================================================================
88# Naming convention; metadata; Base
89# =============================================================================
90# https://alembic.readthedocs.org/en/latest/naming.html
91# https://docs.sqlalchemy.org/en/latest/core/constraints.html#configuring-constraint-naming-conventions # noqa
93MYSQL_MAX_IDENTIFIER_LENGTH = 64
94LONG_COLUMN_NAME_WARNING_LIMIT = 30
96NAMING_CONVENTION = {
97 # - Note that constraint names must be unique in the DATABASE, not the
98 # table;
99 # https://dev.mysql.com/doc/refman/5.6/en/create-table-foreign-keys.html
100 # - Index names only have to be unique for the table;
101 # https://stackoverflow.com/questions/30653452/do-index-names-have-to-be-unique-across-entire-database-in-mysql # noqa
102 # INDEX:
103 "ix": "ix_%(column_0_label)s",
104 # UNIQUE CONSTRAINT:
105 "uq": "uq_%(table_name)s_%(column_0_name)s",
106 # "uq": "uq_%(column_0_name)s",
107 # CHECK CONSTRAINT:
108 # "ck": "ck_%(table_name)s_%(constraint_name)s", # too long for MySQL
109 # ... https://groups.google.com/forum/#!topic/sqlalchemy/SIT4D8S9dUg
110 # "ck": "ck_%(table_name)s_%(column_0_name)s",
111 # Problem 2018-09-14:
112 # - constraints must be unique across database
113 # - MySQL only accepts 64 characters for constraint name
114 # - using "%(column_0_name)" means that explicit constrant names are
115 # ignored
116 # - using "%(constraint_name)" means that all constraints have to be named
117 # explicitly (very tedious)
118 # - so truncate?
119 # https://docs.python.org/3/library/stdtypes.html#old-string-formatting
120 # https://www.python.org/dev/peps/pep-0237/
121 # - The main problem is BOOL columns, e.g.
122 # cpft_lps_discharge.management_specialling_behavioural_disturbance
123 # - Example:
124 # longthing = "abcdefghijklmnopqrstuvwxyz"
125 # d = {"thing": longthing}
126 # "hello %(thing).10s world" % d # LEFT TRUNCATE
127 # # ... gives 'hello abcdefghij world'
128 # "ck": "ck_%(table_name).30s_%(column_0_name).30s",
129 # 3 for "ck_" leaves 61; 30 for table, 1 for "_", 30 for column
130 # ... no...
131 # "obs_contamination_bodily_waste_*"
132 "ck": "ck_%(table_name)s_%(column_0_name)s", # unique but maybe too long
133 # FOREIGN KEY:
134 # "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", # too long for MySQL sometimes! # noqa
135 "fk": "fk_%(table_name)s_%(column_0_name)s",
136 # "fk": "fk_%(column_0_name)s",
137 # PRIMARY KEY:
138 "pk": "pk_%(table_name)s",
139}
140MASTER_META = MetaData(naming_convention=NAMING_CONVENTION)
142# The base of all our model classes:
143Base = declarative_base(metadata=MASTER_META)
145# Special options:
146Base.__table_args__ = {
147 # -------------------------------------------------------------------------
148 # MySQL special options
149 # -------------------------------------------------------------------------
150 # SQLAlchemy __table_args__:
151 # https://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/table_config.html # noqa
152 # SQLAlchemy sends keyword arguments like 'mysql_keyword_name' to be
153 # rendered as KEYWORD_NAME in the CREATE TABLE statement:
154 # https://docs.sqlalchemy.org/en/latest/dialects/mysql.html
155 # Engine: InnoDB
156 "mysql_engine": "InnoDB",
157 # Barracuda: COMPRESSED or DYNAMIC
158 # https://dev.mysql.com/doc/refman/5.7/en/innodb-row-format-dynamic.html
159 # https://xenforo.com/community/threads/anyone-running-their-innodb-tables-with-row_format-compressed.99606/ # noqa
160 # We shouldn't compress everything by default; performance hit.
161 "mysql_row_format": "DYNAMIC",
162 # SEE server_troubleshooting.rst FOR BUG DISCUSSION
163 "mysql_charset": "utf8mb4 COLLATE utf8mb4_unicode_ci",
164 # Character set
165 # REPLACED # 'mysql_charset': 'utf8mb4',
166 # https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html
167 # Collation
168 # Which collation for MySQL? See
169 # - https://stackoverflow.com/questions/766809/whats-the-difference-between-utf8-general-ci-and-utf8-unicode-ci # noqa
170 # REPLACED # 'mysql_collate': 'utf8mb4_unicode_ci'
171 # Note that COLLATION rules are, from least to greatest precedence:
172 # Server collation
173 # Connection-specific collation
174 # Database collation
175 # Table collation
176 # Column collation
177 # Query collation (using CAST or CONVERT)
178 # - https://stackoverflow.com/questions/24356090/difference-between-database-table-column-collation # noqa
179 # Therefore, we can set the table collation for all our tables, and not
180 # worry about the column collation, e.g. Text(collation=...).
181 #
182 # To check a MySQL database, and connection/server settings:
183 # SHOW VARIABLES LIKE '%character%';
184 # SHOW VARIABLES LIKE '%collation%';
185 # To check tables:
186 # SHOW TABLE STATUS WHERE NAME LIKE 'my_tablename'\G
187 # ... note use of \G to produce long-form output!
188 # To check columns:
189 # SHOW FULL COLUMNS FROM my_tablename;
190 #
191 # ONE THING IN PARTICULAR TO BEWARE: utf8mb4_unicode_ci produces
192 # CASE-INSENSITIVE COMPARISON. For example:
193 # SELECT 'a' = 'A'; -- produces 1
194 # SELECT 'a' = 'B'; -- produces 0
195 # SELECT BINARY 'a' = BINARY 'A'; -- produces 0
196 # This is a PROBLEM FOR PASSWORD FIELDS IF WE INTEND TO DO DATABASE-LEVEL
197 # COMPARISONS WITH THEM. In that case we must ensure a different collation
198 # is set; specifically, use
199 #
200 # utf8mb4_bin
201 #
202 # and see also
203 # SHOW COLLATION WHERE `Collation` LIKE 'utf8mb4%';
204 # and
205 # https://dev.mysql.com/doc/refman/5.6/en/charset-binary-collations.html
206 #
207 # To check, run
208 # SHOW FULL COLUMNS FROM _security_users;
209}
211# MySQL things we can't set via SQLAlchemy, but would like to be set:
212# - max_allowed_packet: should be at least 32M
213# - innodb_strict_mode: should be 1, but less of a concern with SQLAlchemy
215# MySQL things we don't care about too much:
216# - innodb_file_per_table: desirable, but up to the user.
219class DeclarativeAndABCMeta(DeclarativeMeta, ABCMeta):
220 """
221 Metaclass for classes that want to inherit from Base and also ABC:
222 """
224 pass
227# =============================================================================
228# Convenience functions
229# =============================================================================
232def make_memory_sqlite_engine(echo: bool = False) -> Engine:
233 """
234 Create an SQLAlchemy :class:`Engine` for an in-memory SQLite database.
235 """
236 return create_engine(SQLITE_MEMORY_URL, echo=echo)
239def make_file_sqlite_engine(filename: str, echo: bool = False) -> Engine:
240 """
241 Create an SQLAlchemy :class:`Engine` for an on-disk SQLite database.
242 """
243 return create_engine(make_sqlite_url(filename), echo=echo)
246def sql_from_sqlite_database(connection: sqlite3.Connection) -> str:
247 """
248 Returns SQL to describe an SQLite database.
250 Args:
251 connection: connection to SQLite database via ``sqlite3`` module
253 Returns:
254 the SQL
256 """
257 with StringIO() as f:
258 # noinspection PyTypeChecker
259 for line in connection.iterdump():
260 f.write(line + "\n")
261 f.flush()
262 return f.getvalue()
265@cache_region_static.cache_on_arguments(function_key_generator=fkg)
266def get_all_ddl(dialect_name: str = SqlaDialectName.MYSQL) -> str:
267 """
268 Returns the DDL (data definition language; SQL ``CREATE TABLE`` commands)
269 for our SQLAlchemy metadata.
271 Args:
272 dialect_name: SQLAlchemy dialect name
273 """
274 metadata = Base.metadata # type: MetaData
275 with StringIO() as f:
276 dump_ddl(metadata, dialect_name=dialect_name, fileobj=f)
277 f.flush()
278 text = f.getvalue()
279 return text
282def log_all_ddl(dialect_name: str = SqlaDialectName.MYSQL) -> None:
283 """
284 Send the DDL for our SQLAlchemy metadata to the Python log.
286 Args:
287 dialect_name: SQLAlchemy dialect name
288 """
289 text = get_all_ddl(dialect_name)
290 log.info(text)
291 log.info("DDL length: {} characters", len(text))
294@cache_region_static.cache_on_arguments(function_key_generator=fkg)
295def get_table_ddl(
296 table: Table, dialect_name: str = SqlaDialectName.MYSQL
297) -> str:
298 """
299 Returns the DDL (data definition language; SQL ``CREATE TABLE`` commands)
300 for a specific table.
302 Args:
303 table:
304 Table to dump.
305 dialect_name:
306 SQLAlchemy dialect name.
308 https://stackoverflow.com/questions/2128717/sqlalchemy-printing-raw-sql-from-create
309 """ # noqa
310 dialect = get_dialect_from_name(dialect_name)
311 return str(CreateTable(table).compile(dialect=dialect))
314def assert_constraint_name_ok(table_name: str, column_name: str) -> None:
315 """
316 Checks that the automatically generated name of a constraint isn't too long
317 for specific databases.
319 Args:
320 table_name: table name
321 column_name: column name
323 Raises:
324 AssertionError, if something will break
325 """
326 d = {"table_name": table_name, "column_0_name": column_name}
327 anticipated_name = NAMING_CONVENTION["ck"] % d
328 if len(anticipated_name) > MYSQL_MAX_IDENTIFIER_LENGTH:
329 raise AssertionError(
330 f"Constraint name too long for table {table_name!r}, column "
331 f"{column_name!r}; will be {anticipated_name!r} "
332 f"of length {len(anticipated_name)}"
333 )
336# =============================================================================
337# Database engine hacks
338# =============================================================================
341def hack_pendulum_into_pymysql() -> None:
342 """
343 Hack in support for :class:`pendulum.DateTime` into the ``pymysql``
344 database interface.
346 See https://pendulum.eustace.io/docs/#limitations.
347 """
348 try:
349 # noinspection PyUnresolvedReferences
350 from pymysql.converters import encoders, escape_datetime
352 encoders[Pendulum] = escape_datetime
353 except ImportError:
354 pass
357hack_pendulum_into_pymysql()
360class MutableDict(Mutable, dict):
361 """
362 Source:
363 https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html
364 """
366 @classmethod
367 def coerce(cls, key: str, value: Any) -> Any:
368 """
369 Convert plain dictionaries to MutableDict.
370 """
372 if not isinstance(value, MutableDict):
373 if isinstance(value, dict):
374 return MutableDict(value)
376 # this call will raise ValueError
377 return Mutable.coerce(key, value)
378 else:
379 return value
381 def __setitem__(self, key: str, value: Any) -> None:
382 """
383 Detect dictionary set events and emit change events.
384 """
386 dict.__setitem__(self, key, value)
387 self.changed()
389 def __delitem__(self, key: str) -> None:
390 """
391 Detect dictionary del events and emit change events.
392 """
394 dict.__delitem__(self, key)
395 self.changed()