Coverage for cc_modules/cc_sqlalchemy.py: 61%

79 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-08 23:14 +0000

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_sqlalchemy.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry. 

9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**SQLAlchemy helper functions and constants.** 

29 

30We define our metadata ``Base`` here, and things like our index naming 

31convention and MySQL table formats. 

32 

33A few random notes: 

34 

35- SQLAlchemy will automatically warn about clashing columns: 

36 

37 .. :code-block:: python 

38 

39 from sqlalchemy import Column, Integer 

40 from sqlalchemy.ext.declarative import declarative_base 

41 

42 Base = declarative_base() 

43 

44 class Thing(Base): 

45 __tablename__ = "thing" 

46 a = Column("a", Integer, primary_key=True) 

47 b = Column("b", Integer) 

48 c = Column("b", Integer) # produces a warning: 

49 

50 .. code-block:: none 

51 

52 SAWarning: On class 'Thing', Column object 'b' named directly multiple 

53 times, only one will be used: b, c. Consider using orm.synonym instead 

54 

55""" 

56 

57from abc import ABCMeta 

58from io import StringIO 

59import logging 

60import sqlite3 

61from typing import Any 

62 

63from cardinal_pythonlib.logs import BraceStyleAdapter 

64from cardinal_pythonlib.sqlalchemy.dialect import ( 

65 get_dialect_from_name, 

66 SqlaDialectName, 

67) 

68from cardinal_pythonlib.sqlalchemy.dump import dump_ddl 

69from cardinal_pythonlib.sqlalchemy.session import ( 

70 make_sqlite_url, 

71 SQLITE_MEMORY_URL, 

72) 

73from pendulum import DateTime as Pendulum 

74 

75from sqlalchemy.engine import create_engine 

76from sqlalchemy.engine.base import Engine 

77from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta 

78from sqlalchemy.ext.mutable import Mutable 

79from sqlalchemy.schema import CreateTable 

80from sqlalchemy.sql.schema import MetaData, Table 

81 

82from camcops_server.cc_modules.cc_cache import cache_region_static, fkg 

83 

84log = BraceStyleAdapter(logging.getLogger(__name__)) 

85 

86 

87# ============================================================================= 

88# Naming convention; metadata; Base 

89# ============================================================================= 

90# https://alembic.readthedocs.org/en/latest/naming.html 

91# https://docs.sqlalchemy.org/en/latest/core/constraints.html#configuring-constraint-naming-conventions # noqa 

92 

93MYSQL_MAX_IDENTIFIER_LENGTH = 64 

94LONG_COLUMN_NAME_WARNING_LIMIT = 30 

95 

96NAMING_CONVENTION = { 

97 # - Note that constraint names must be unique in the DATABASE, not the 

98 # table; 

99 # https://dev.mysql.com/doc/refman/5.6/en/create-table-foreign-keys.html 

100 # - Index names only have to be unique for the table; 

101 # https://stackoverflow.com/questions/30653452/do-index-names-have-to-be-unique-across-entire-database-in-mysql # noqa 

102 # INDEX: 

103 "ix": "ix_%(column_0_label)s", 

104 # UNIQUE CONSTRAINT: 

105 "uq": "uq_%(table_name)s_%(column_0_name)s", 

106 # "uq": "uq_%(column_0_name)s", 

107 # CHECK CONSTRAINT: 

108 # "ck": "ck_%(table_name)s_%(constraint_name)s", # too long for MySQL 

109 # ... https://groups.google.com/forum/#!topic/sqlalchemy/SIT4D8S9dUg 

110 # "ck": "ck_%(table_name)s_%(column_0_name)s", 

111 # Problem 2018-09-14: 

112 # - constraints must be unique across database 

113 # - MySQL only accepts 64 characters for constraint name 

114 # - using "%(column_0_name)" means that explicit constrant names are 

115 # ignored 

116 # - using "%(constraint_name)" means that all constraints have to be named 

117 # explicitly (very tedious) 

118 # - so truncate? 

119 # https://docs.python.org/3/library/stdtypes.html#old-string-formatting 

120 # https://www.python.org/dev/peps/pep-0237/ 

121 # - The main problem is BOOL columns, e.g. 

122 # cpft_lps_discharge.management_specialling_behavioural_disturbance 

123 # - Example: 

124 # longthing = "abcdefghijklmnopqrstuvwxyz" 

125 # d = {"thing": longthing} 

126 # "hello %(thing).10s world" % d # LEFT TRUNCATE 

127 # # ... gives 'hello abcdefghij world' 

128 # "ck": "ck_%(table_name).30s_%(column_0_name).30s", 

129 # 3 for "ck_" leaves 61; 30 for table, 1 for "_", 30 for column 

130 # ... no... 

131 # "obs_contamination_bodily_waste_*" 

132 "ck": "ck_%(table_name)s_%(column_0_name)s", # unique but maybe too long 

133 # FOREIGN KEY: 

134 # "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", # too long for MySQL sometimes! # noqa 

135 "fk": "fk_%(table_name)s_%(column_0_name)s", 

136 # "fk": "fk_%(column_0_name)s", 

137 # PRIMARY KEY: 

138 "pk": "pk_%(table_name)s", 

139} 

140MASTER_META = MetaData(naming_convention=NAMING_CONVENTION) 

141 

142# The base of all our model classes: 

143Base = declarative_base(metadata=MASTER_META) 

144 

145# Special options: 

146Base.__table_args__ = { 

147 # ------------------------------------------------------------------------- 

148 # MySQL special options 

149 # ------------------------------------------------------------------------- 

150 # SQLAlchemy __table_args__: 

151 # https://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/table_config.html # noqa 

152 # SQLAlchemy sends keyword arguments like 'mysql_keyword_name' to be 

153 # rendered as KEYWORD_NAME in the CREATE TABLE statement: 

154 # https://docs.sqlalchemy.org/en/latest/dialects/mysql.html 

155 # Engine: InnoDB 

156 "mysql_engine": "InnoDB", 

157 # Barracuda: COMPRESSED or DYNAMIC 

158 # https://dev.mysql.com/doc/refman/5.7/en/innodb-row-format-dynamic.html 

159 # https://xenforo.com/community/threads/anyone-running-their-innodb-tables-with-row_format-compressed.99606/ # noqa 

160 # We shouldn't compress everything by default; performance hit. 

161 "mysql_row_format": "DYNAMIC", 

162 # SEE server_troubleshooting.rst FOR BUG DISCUSSION 

163 "mysql_charset": "utf8mb4 COLLATE utf8mb4_unicode_ci", 

164 # Character set 

165 # REPLACED # 'mysql_charset': 'utf8mb4', 

166 # https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html 

167 # Collation 

168 # Which collation for MySQL? See 

169 # - https://stackoverflow.com/questions/766809/whats-the-difference-between-utf8-general-ci-and-utf8-unicode-ci # noqa 

170 # REPLACED # 'mysql_collate': 'utf8mb4_unicode_ci' 

171 # Note that COLLATION rules are, from least to greatest precedence: 

172 # Server collation 

173 # Connection-specific collation 

174 # Database collation 

175 # Table collation 

176 # Column collation 

177 # Query collation (using CAST or CONVERT) 

178 # - https://stackoverflow.com/questions/24356090/difference-between-database-table-column-collation # noqa 

179 # Therefore, we can set the table collation for all our tables, and not 

180 # worry about the column collation, e.g. Text(collation=...). 

181 # 

182 # To check a MySQL database, and connection/server settings: 

183 # SHOW VARIABLES LIKE '%character%'; 

184 # SHOW VARIABLES LIKE '%collation%'; 

185 # To check tables: 

186 # SHOW TABLE STATUS WHERE NAME LIKE 'my_tablename'\G 

187 # ... note use of \G to produce long-form output! 

188 # To check columns: 

189 # SHOW FULL COLUMNS FROM my_tablename; 

190 # 

191 # ONE THING IN PARTICULAR TO BEWARE: utf8mb4_unicode_ci produces 

192 # CASE-INSENSITIVE COMPARISON. For example: 

193 # SELECT 'a' = 'A'; -- produces 1 

194 # SELECT 'a' = 'B'; -- produces 0 

195 # SELECT BINARY 'a' = BINARY 'A'; -- produces 0 

196 # This is a PROBLEM FOR PASSWORD FIELDS IF WE INTEND TO DO DATABASE-LEVEL 

197 # COMPARISONS WITH THEM. In that case we must ensure a different collation 

198 # is set; specifically, use 

199 # 

200 # utf8mb4_bin 

201 # 

202 # and see also 

203 # SHOW COLLATION WHERE `Collation` LIKE 'utf8mb4%'; 

204 # and 

205 # https://dev.mysql.com/doc/refman/5.6/en/charset-binary-collations.html 

206 # 

207 # To check, run 

208 # SHOW FULL COLUMNS FROM _security_users; 

209} 

210 

211# MySQL things we can't set via SQLAlchemy, but would like to be set: 

212# - max_allowed_packet: should be at least 32M 

213# - innodb_strict_mode: should be 1, but less of a concern with SQLAlchemy 

214 

215# MySQL things we don't care about too much: 

216# - innodb_file_per_table: desirable, but up to the user. 

217 

218 

219class DeclarativeAndABCMeta(DeclarativeMeta, ABCMeta): 

220 """ 

221 Metaclass for classes that want to inherit from Base and also ABC: 

222 """ 

223 

224 pass 

225 

226 

227# ============================================================================= 

228# Convenience functions 

229# ============================================================================= 

230 

231 

232def make_memory_sqlite_engine(echo: bool = False) -> Engine: 

233 """ 

234 Create an SQLAlchemy :class:`Engine` for an in-memory SQLite database. 

235 """ 

236 return create_engine(SQLITE_MEMORY_URL, echo=echo) 

237 

238 

239def make_file_sqlite_engine(filename: str, echo: bool = False) -> Engine: 

240 """ 

241 Create an SQLAlchemy :class:`Engine` for an on-disk SQLite database. 

242 """ 

243 return create_engine(make_sqlite_url(filename), echo=echo) 

244 

245 

246def sql_from_sqlite_database(connection: sqlite3.Connection) -> str: 

247 """ 

248 Returns SQL to describe an SQLite database. 

249 

250 Args: 

251 connection: connection to SQLite database via ``sqlite3`` module 

252 

253 Returns: 

254 the SQL 

255 

256 """ 

257 with StringIO() as f: 

258 # noinspection PyTypeChecker 

259 for line in connection.iterdump(): 

260 f.write(line + "\n") 

261 f.flush() 

262 return f.getvalue() 

263 

264 

265@cache_region_static.cache_on_arguments(function_key_generator=fkg) 

266def get_all_ddl(dialect_name: str = SqlaDialectName.MYSQL) -> str: 

267 """ 

268 Returns the DDL (data definition language; SQL ``CREATE TABLE`` commands) 

269 for our SQLAlchemy metadata. 

270 

271 Args: 

272 dialect_name: SQLAlchemy dialect name 

273 """ 

274 metadata = Base.metadata # type: MetaData 

275 with StringIO() as f: 

276 dump_ddl(metadata, dialect_name=dialect_name, fileobj=f) 

277 f.flush() 

278 text = f.getvalue() 

279 return text 

280 

281 

282def log_all_ddl(dialect_name: str = SqlaDialectName.MYSQL) -> None: 

283 """ 

284 Send the DDL for our SQLAlchemy metadata to the Python log. 

285 

286 Args: 

287 dialect_name: SQLAlchemy dialect name 

288 """ 

289 text = get_all_ddl(dialect_name) 

290 log.info(text) 

291 log.info("DDL length: {} characters", len(text)) 

292 

293 

294@cache_region_static.cache_on_arguments(function_key_generator=fkg) 

295def get_table_ddl( 

296 table: Table, dialect_name: str = SqlaDialectName.MYSQL 

297) -> str: 

298 """ 

299 Returns the DDL (data definition language; SQL ``CREATE TABLE`` commands) 

300 for a specific table. 

301 

302 Args: 

303 table: 

304 Table to dump. 

305 dialect_name: 

306 SQLAlchemy dialect name. 

307 

308 https://stackoverflow.com/questions/2128717/sqlalchemy-printing-raw-sql-from-create 

309 """ # noqa 

310 dialect = get_dialect_from_name(dialect_name) 

311 return str(CreateTable(table).compile(dialect=dialect)) 

312 

313 

314def assert_constraint_name_ok(table_name: str, column_name: str) -> None: 

315 """ 

316 Checks that the automatically generated name of a constraint isn't too long 

317 for specific databases. 

318 

319 Args: 

320 table_name: table name 

321 column_name: column name 

322 

323 Raises: 

324 AssertionError, if something will break 

325 """ 

326 d = {"table_name": table_name, "column_0_name": column_name} 

327 anticipated_name = NAMING_CONVENTION["ck"] % d 

328 if len(anticipated_name) > MYSQL_MAX_IDENTIFIER_LENGTH: 

329 raise AssertionError( 

330 f"Constraint name too long for table {table_name!r}, column " 

331 f"{column_name!r}; will be {anticipated_name!r} " 

332 f"of length {len(anticipated_name)}" 

333 ) 

334 

335 

336# ============================================================================= 

337# Database engine hacks 

338# ============================================================================= 

339 

340 

341def hack_pendulum_into_pymysql() -> None: 

342 """ 

343 Hack in support for :class:`pendulum.DateTime` into the ``pymysql`` 

344 database interface. 

345 

346 See https://pendulum.eustace.io/docs/#limitations. 

347 """ 

348 try: 

349 # noinspection PyUnresolvedReferences 

350 from pymysql.converters import encoders, escape_datetime 

351 

352 encoders[Pendulum] = escape_datetime 

353 except ImportError: 

354 pass 

355 

356 

357hack_pendulum_into_pymysql() 

358 

359 

360class MutableDict(Mutable, dict): 

361 """ 

362 Source: 

363 https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html 

364 """ 

365 

366 @classmethod 

367 def coerce(cls, key: str, value: Any) -> Any: 

368 """ 

369 Convert plain dictionaries to MutableDict. 

370 """ 

371 

372 if not isinstance(value, MutableDict): 

373 if isinstance(value, dict): 

374 return MutableDict(value) 

375 

376 # this call will raise ValueError 

377 return Mutable.coerce(key, value) 

378 else: 

379 return value 

380 

381 def __setitem__(self, key: str, value: Any) -> None: 

382 """ 

383 Detect dictionary set events and emit change events. 

384 """ 

385 

386 dict.__setitem__(self, key, value) 

387 self.changed() 

388 

389 def __delitem__(self, key: str) -> None: 

390 """ 

391 Detect dictionary del events and emit change events. 

392 """ 

393 

394 dict.__delitem__(self, key) 

395 self.changed()