Coverage for cc_modules/cc_request.py: 42%

672 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-08 23:14 +0000

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_request.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012, University of Cambridge, Department of Psychiatry. 

9 Created by Rudolf Cardinal (rnc1001@cam.ac.uk). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**Implements a Pyramid Request object customized for CamCOPS.** 

29 

30""" 

31 

32import collections 

33from contextlib import contextmanager 

34import datetime 

35import gettext 

36import logging 

37import os 

38import re 

39import secrets 

40from typing import ( 

41 Any, 

42 Dict, 

43 Generator, 

44 List, 

45 Optional, 

46 Set, 

47 Tuple, 

48 TYPE_CHECKING, 

49 Union, 

50) 

51import urllib.parse 

52 

53from cardinal_pythonlib.datetimefunc import ( 

54 coerce_to_pendulum, 

55 coerce_to_pendulum_date, 

56 convert_datetime_to_utc, 

57 format_datetime, 

58 pendulum_to_utc_datetime_without_tz, 

59) 

60from cardinal_pythonlib.fileops import get_directory_contents_size, mkdir_p 

61from cardinal_pythonlib.httpconst import HttpMethod 

62from cardinal_pythonlib.logs import BraceStyleAdapter 

63from cardinal_pythonlib.plot import ( 

64 png_img_html_from_pyplot_figure, 

65 svg_html_from_pyplot_figure, 

66) 

67import cardinal_pythonlib.rnc_web as ws 

68from cardinal_pythonlib.wsgi.constants import WsgiEnvVar 

69import lockfile 

70from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas 

71from matplotlib.figure import Figure 

72from matplotlib.font_manager import FontProperties 

73from pendulum import Date, DateTime as Pendulum, Duration 

74from pendulum.parsing.exceptions import ParserError 

75from pyramid.config import Configurator 

76from pyramid.decorator import reify 

77from pyramid.httpexceptions import HTTPBadRequest, HTTPException 

78from pyramid.interfaces import ISession 

79from pyramid.request import Request 

80from pyramid.response import Response 

81from pyramid.testing import DummyRequest 

82from sqlalchemy.engine.base import Engine 

83from sqlalchemy.orm import sessionmaker 

84from sqlalchemy.orm import Session as SqlASession 

85from webob.multidict import MultiDict 

86 

87# Note: everything uder the sun imports this file, so keep the intra-package 

88# imports as minimal as possible. 

89from camcops_server.cc_modules.cc_baseconstants import ( 

90 DOCUMENTATION_URL, 

91 TRANSLATIONS_DIR, 

92) 

93from camcops_server.cc_modules.cc_config import ( 

94 CamcopsConfig, 

95 get_config, 

96 get_config_filename_from_os_env, 

97) 

98from camcops_server.cc_modules.cc_constants import ( 

99 CSS_PAGED_MEDIA, 

100 DateFormat, 

101 PlotDefaults, 

102 USE_SVG_IN_HTML, 

103) 

104from camcops_server.cc_modules.cc_idnumdef import ( 

105 get_idnum_definitions, 

106 IdNumDefinition, 

107 validate_id_number, 

108) 

109from camcops_server.cc_modules.cc_language import ( 

110 DEFAULT_LOCALE, 

111 GETTEXT_DOMAIN, 

112 POSSIBLE_LOCALES, 

113) 

114 

115# noinspection PyUnresolvedReferences 

116import camcops_server.cc_modules.cc_plot # import side effects (configure matplotlib) # noqa 

117from camcops_server.cc_modules.cc_pyramid import ( 

118 camcops_add_mako_renderer, 

119 CamcopsAuthenticationPolicy, 

120 CamcopsAuthorizationPolicy, 

121 CookieKey, 

122 get_session_factory, 

123 icon_html, 

124 icon_text, 

125 icons_text, 

126 Permission, 

127 RouteCollection, 

128 Routes, 

129 STATIC_CAMCOPS_PACKAGE_PATH, 

130) 

131from camcops_server.cc_modules.cc_response import camcops_response_factory 

132from camcops_server.cc_modules.cc_serversettings import ( 

133 get_server_settings, 

134 ServerSettings, 

135) 

136from camcops_server.cc_modules.cc_string import ( 

137 all_extra_strings_as_dicts, 

138 APPSTRING_TASKNAME, 

139 MISSING_LOCALE, 

140) 

141from camcops_server.cc_modules.cc_tabletsession import TabletSession 

142from camcops_server.cc_modules.cc_text import SS, server_string 

143from camcops_server.cc_modules.cc_user import User 

144from camcops_server.cc_modules.cc_validators import ( 

145 STRING_VALIDATOR_TYPE, 

146 validate_alphanum_underscore, 

147 validate_redirect_url, 

148) 

149 

150if TYPE_CHECKING: 

151 from matplotlib.axis import Axis 

152 from matplotlib.axes import Axes 

153 from matplotlib.text import Text 

154 from camcops_server.cc_modules.cc_exportrecipient import ExportRecipient 

155 from camcops_server.cc_modules.cc_exportrecipientinfo import ( 

156 ExportRecipientInfo, 

157 ) 

158 from camcops_server.cc_modules.cc_session import CamcopsSession 

159 from camcops_server.cc_modules.cc_snomed import SnomedConcept 

160 

161log = BraceStyleAdapter(logging.getLogger(__name__)) 

162 

163 

164# ============================================================================= 

165# Debugging options 

166# ============================================================================= 

167 

168DEBUG_ADD_ROUTES = False 

169DEBUG_AUTHORIZATION = False 

170DEBUG_CAMCOPS_SESSION = False 

171DEBUG_DBSESSION_MANAGEMENT = False 

172DEBUG_GETTEXT = False 

173DEBUG_REQUEST_CREATION = False 

174DEBUG_TABLET_SESSION = False 

175 

176if any( 

177 [ 

178 DEBUG_ADD_ROUTES, 

179 DEBUG_AUTHORIZATION, 

180 DEBUG_CAMCOPS_SESSION, 

181 DEBUG_DBSESSION_MANAGEMENT, 

182 DEBUG_GETTEXT, 

183 DEBUG_REQUEST_CREATION, 

184 DEBUG_TABLET_SESSION, 

185 ] 

186): 

187 log.warning("Debugging options enabled!") 

188 

189 

190# ============================================================================= 

191# Constants 

192# ============================================================================= 

193 

194TRUE_STRINGS_LOWER_CASE = ["true", "t", "1", "yes", "y"] 

195FALSE_STRINGS_LOWER_CASE = ["false", "f", "0", "no", "n"] 

196 

197 

198# ============================================================================= 

199# Modified Request interface, for type checking 

200# ============================================================================= 

201# https://docs.pylonsproject.org/projects/pyramid_cookbook/en/latest/auth/user_object.html 

202# https://rollbar.com/blog/using-pyramid-request-factory-to-write-less-code/ 

203# 

204# ... everything with reify=True is cached, so if we ask for something 

205# more than once, we keep getting the same thing 

206# ... https://docs.pylonsproject.org/projects/pyramid/en/latest/api/request.html#pyramid.request.Request.set_property # noqa 

207 

208 

209class CamcopsRequest(Request): 

210 """ 

211 The CamcopsRequest is an object central to all HTTP requests. It is the 

212 main thing passed all around the server, and embodies what we need to know 

213 about the client request -- including user information, ways of accessing 

214 the database, and so on. 

215 

216 It reads its config (on first demand) from the config file specified in 

217 ``os.environ[ENVVAR_CONFIG_FILE]``. 

218 

219 """ 

220 

221 def __init__(self, *args, **kwargs): 

222 """ 

223 This is called as the Pyramid request factory; see 

224 ``config.set_request_factory(CamcopsRequest)`` 

225 

226 What's the best way of handling the database client? 

227 

228 - With Titanium, we were constrained not to use cookies. With Qt, we 

229 have the option. 

230 - But are cookies a good idea? 

231 Probably not; they are somewhat overcomplicated for this. 

232 See also 

233 

234 - https://softwareengineering.stackexchange.com/questions/141019/ 

235 - https://stackoverflow.com/questions/6068113/do-sessions-really-violate-restfulness 

236 

237 - Let's continue to avoid cookies. 

238 - We don't have to cache any information (we still send username/ 

239 password details with each request, and that is RESTful) but it 

240 does save authentication time to do so on calls after the first. 

241 - What we could try to do is: 

242 

243 - look up a session here, at Request creation time; 

244 - add a new session if there wasn't one; 

245 - but allow the database API code to replace that session (BEFORE 

246 it's saved to the database and gains its PK) with another, 

247 determined by the content. 

248 - This gives one more database hit, but avoids the bcrypt time. 

249 

250 """ # noqa 

251 super().__init__(*args, **kwargs) 

252 self.use_svg = False # use SVG (not just PNG) for graphics 

253 self.provide_png_fallback_for_svg = ( 

254 True # for SVG: provide PNG fallback image? 

255 ) 

256 self.add_response_callback(complete_request_add_cookies) 

257 self._camcops_session = None # type: Optional[CamcopsSession] 

258 self._debugging_db_session = ( 

259 None 

260 ) # type: Optional[SqlASession] # for unit testing only 

261 self._debugging_user = ( 

262 None 

263 ) # type: Optional[User] # for unit testing only 

264 self._pending_export_push_requests = ( 

265 [] 

266 ) # type: List[Tuple[str, str, int]] 

267 self._cached_sstring = {} # type: Dict[SS, str] 

268 # Don't make the _camcops_session yet; it will want a Registry, and 

269 # we may not have one yet; see command_line_request(). 

270 if DEBUG_REQUEST_CREATION: 

271 log.debug( 

272 "CamcopsRequest.__init__: args={!r}, kwargs={!r}", args, kwargs 

273 ) 

274 

275 # ------------------------------------------------------------------------- 

276 # HTTP nonce 

277 # ------------------------------------------------------------------------- 

278 

279 @reify 

280 def nonce(self) -> str: 

281 """ 

282 Return a nonce that is generated at random for each request, but 

283 remains constant for that request (because we use ``@reify``). 

284 

285 See https://content-security-policy.com/examples/allow-inline-style/. 

286 

287 And for how to make one: 

288 https://stackoverflow.com/questions/5590170/what-is-the-standard-method-for-generating-a-nonce-in-python 

289 """ # noqa 

290 return secrets.token_urlsafe() 

291 

292 # ------------------------------------------------------------------------- 

293 # CamcopsSession 

294 # ------------------------------------------------------------------------- 

295 

296 @property 

297 def camcops_session(self) -> "CamcopsSession": 

298 """ 

299 Returns the 

300 :class:`camcops_server.cc_modules.cc_session.CamcopsSession` for this 

301 request (q.v.). 

302 

303 Contrast: 

304 

305 .. code-block:: none 

306 

307 ccsession = request.camcops_session # type: CamcopsSession 

308 pyramid_session = request.session # type: ISession 

309 """ 

310 if self._camcops_session is None: 

311 from camcops_server.cc_modules.cc_session import ( 

312 CamcopsSession, 

313 ) # delayed import 

314 

315 self._camcops_session = CamcopsSession.get_session_using_cookies( 

316 self 

317 ) 

318 if DEBUG_CAMCOPS_SESSION: 

319 log.debug("{!r}", self._camcops_session) 

320 return self._camcops_session 

321 

322 def replace_camcops_session(self, ccsession: "CamcopsSession") -> None: 

323 """ 

324 Replaces any existing 

325 :class:`camcops_server.cc_modules.cc_session.CamcopsSession` with a new 

326 one. 

327 

328 Rationale: 

329 

330 We may have created a new HTTP session because the request had no 

331 cookies (added to the DB session but not yet saved), but we might 

332 then enter the database/tablet upload API and find session details, 

333 not from the cookies, but from the POST data. At that point, we 

334 want to replace the session in the Request, without committing the 

335 first one to disk. 

336 """ 

337 if self._camcops_session is not None: 

338 self.dbsession.expunge(self._camcops_session) 

339 self._camcops_session = ccsession 

340 

341 def complete_request_add_cookies(self) -> None: 

342 """ 

343 Finializes the response by adding session cookies. 

344 We do this late so that we can hot-swap the session if we're using the 

345 database/tablet API rather than a human web browser. 

346 

347 Response callbacks are called in the order first-to-most-recently-added. 

348 See :class:`pyramid.request.CallbackMethodsMixin`. 

349 

350 That looks like we can add a callback in the process of running a 

351 callback. And when we add a cookie to a Pyramid session, that sets a 

352 callback. Let's give it a go... 

353 """ # noqa: E501 

354 # 2019-03-21: If we've not used a CamcopsSession (e.g. for serving 

355 # a static view), do we care? 

356 if self._camcops_session is None: 

357 return 

358 

359 dbsession = self.dbsession 

360 dbsession.flush() # sets the PK for ccsession, if it wasn't set 

361 # Write the details back to the Pyramid session (will be persisted 

362 # via the Response automatically): 

363 pyramid_session = self.session # type: ISession 

364 ccsession = self.camcops_session 

365 pyramid_session[CookieKey.SESSION_ID] = str(ccsession.id) 

366 pyramid_session[CookieKey.SESSION_TOKEN] = ccsession.token 

367 # ... should cause the ISession to add a callback to add cookies, 

368 # which will be called immediately after this one. 

369 

370 # ------------------------------------------------------------------------- 

371 # Config 

372 # ------------------------------------------------------------------------- 

373 

374 @reify 

375 def config_filename(self) -> str: 

376 """ 

377 Gets the CamCOPS config filename in use, from the config file specified 

378 in ``os.environ[ENVVAR_CONFIG_FILE]``. 

379 """ 

380 return get_config_filename_from_os_env() 

381 

382 @reify 

383 def config(self) -> CamcopsConfig: 

384 """ 

385 Return an instance of 

386 :class:`camcops_server/cc_modules/cc_config.CamcopsConfig` for the 

387 request. 

388 

389 Access it as ``request.config``, with no brackets. 

390 """ 

391 config = get_config(config_filename=self.config_filename) 

392 return config 

393 

394 # ------------------------------------------------------------------------- 

395 # Database 

396 # ------------------------------------------------------------------------- 

397 

398 @reify 

399 def engine(self) -> Engine: 

400 """ 

401 Returns the SQLAlchemy :class:`Engine` for the request. 

402 """ 

403 cfg = self.config 

404 return cfg.get_sqla_engine() 

405 

406 @reify 

407 def dbsession(self) -> SqlASession: 

408 """ 

409 Return an SQLAlchemy session for the relevant request. 

410 

411 The use of ``@reify`` makes this elegant. If and only if a view wants a 

412 database, it can say 

413 

414 .. code-block:: python 

415 

416 dbsession = request.dbsession 

417 

418 and if it requests that, the cleanup callbacks (COMMIT or ROLLBACK) get 

419 installed. 

420 """ 

421 # log.debug("CamcopsRequest.dbsession: caller stack:\n{}", 

422 # "\n".join(get_caller_stack_info())) 

423 _dbsession = self.get_bare_dbsession() 

424 

425 def end_sqlalchemy_session(req: Request) -> None: 

426 # noinspection PyProtectedMember 

427 req._finish_dbsession() 

428 

429 # - For command-line pseudo-requests, add_finished_callback is no use, 

430 # because that's called by the Pyramid routing framework. 

431 # - So how do we autocommit a command-line session? 

432 # - Hooking into CamcopsRequest.__del__ did not work: called, yes, but 

433 # object state (e.g. newly inserted User objects) went wrong (e.g. 

434 # the objects had been blanked somehow, or that's what the INSERT 

435 # statements looked like). 

436 # - Use a context manager instead; see below. 

437 self.add_finished_callback(end_sqlalchemy_session) 

438 

439 if DEBUG_DBSESSION_MANAGEMENT: 

440 log.debug( 

441 "Returning SQLAlchemy session as " "CamcopsRequest.dbsession" 

442 ) 

443 

444 return _dbsession 

445 

446 def _finish_dbsession(self) -> None: 

447 """ 

448 A database session has finished. COMMIT or ROLLBACK, depending on how 

449 things went. 

450 """ 

451 # Do NOT roll back "if req.exception is not None"; that includes 

452 # all sorts of exceptions like HTTPFound, HTTPForbidden, etc. 

453 # See also 

454 # - https://docs.pylonsproject.org/projects/pyramid_cookbook/en/latest/pylons/exceptions.html # noqa 

455 # But they are neatly subclasses of HTTPException, and isinstance() 

456 # deals with None, so: 

457 session = self.dbsession 

458 if self.exception is not None and not isinstance( 

459 self.exception, HTTPException 

460 ): 

461 log.critical( 

462 "Request raised exception that wasn't an " 

463 "HTTPException; rolling back; exception was: {!r}", 

464 self.exception, 

465 ) 

466 session.rollback() 

467 else: 

468 if DEBUG_DBSESSION_MANAGEMENT: 

469 log.debug("Committing to database") 

470 session.commit() 

471 if self._pending_export_push_requests: 

472 self._process_pending_export_push_requests() 

473 if DEBUG_DBSESSION_MANAGEMENT: 

474 log.debug("Closing SQLAlchemy session") 

475 session.close() 

476 

477 def get_bare_dbsession(self) -> SqlASession: 

478 """ 

479 Returns a bare SQLAlchemy session for the request. 

480 

481 See :func:`dbsession`, the more commonly used wrapper function. 

482 """ 

483 if self._debugging_db_session: 

484 log.debug("Request is using debugging SQLAlchemy session") 

485 return self._debugging_db_session 

486 if DEBUG_DBSESSION_MANAGEMENT: 

487 log.debug("Making SQLAlchemy session") 

488 engine = self.engine 

489 maker = sessionmaker(bind=engine) 

490 session = maker() # type: SqlASession 

491 return session 

492 

493 # ------------------------------------------------------------------------- 

494 # TabletSession 

495 # ------------------------------------------------------------------------- 

496 

497 @reify 

498 def tabletsession(self) -> TabletSession: 

499 """ 

500 Request a 

501 :class:`camcops_server.cc_modules.cc_tabletsession.TabletSession`, 

502 which is an information structure geared to client (tablet) database 

503 accesses. 

504 

505 If we're using this interface, we also want to ensure we're using 

506 the :class:`camcops_server.cc_modules.cc_session.CamcopsSession` for 

507 the information provided by the tablet in the POST request, not 

508 anything already loaded/reset via cookies. 

509 """ 

510 from camcops_server.cc_modules.cc_session import ( 

511 CamcopsSession, 

512 ) # delayed import 

513 

514 ts = TabletSession(self) 

515 new_cc_session = CamcopsSession.get_session_for_tablet(ts) 

516 # ... does login 

517 self.replace_camcops_session(new_cc_session) 

518 ts.set_session_id_token(new_cc_session.id, new_cc_session.token) 

519 if DEBUG_TABLET_SESSION: 

520 log.debug("CamcopsRequest: {!r}", self) 

521 log.debug("CamcopsRequest.tabletsession: {!r}", ts) 

522 log.debug( 

523 "CamcopsRequest.camcops_session: {!r}", self._camcops_session 

524 ) 

525 return ts 

526 

527 # ------------------------------------------------------------------------- 

528 # Date/time 

529 # ------------------------------------------------------------------------- 

530 

531 @reify 

532 def now(self) -> Pendulum: 

533 """ 

534 Returns the time of the request as an Pendulum object. 

535 

536 (Reified, so a request only ever has one time.) 

537 Exposed as a property. 

538 """ 

539 return Pendulum.now() 

540 

541 @reify 

542 def now_utc(self) -> Pendulum: 

543 """ 

544 Returns the time of the request as a UTC Pendulum. 

545 """ 

546 p = self.now # type: Pendulum 

547 return convert_datetime_to_utc(p) 

548 

549 @reify 

550 def now_utc_no_tzinfo(self) -> datetime.datetime: 

551 """ 

552 Returns the time of the request as a datetime in UTC with no timezone 

553 information attached. For when you want to compare to something similar 

554 without getting the error "TypeError: can't compare offset-naive and 

555 offset-aware datetimes". 

556 """ 

557 p = self.now # type: Pendulum 

558 return pendulum_to_utc_datetime_without_tz(p) 

559 

560 @reify 

561 def now_era_format(self) -> str: 

562 """ 

563 Returns the request time in an ISO-8601 format suitable for use as a 

564 CamCOPS ``era``. 

565 """ 

566 return format_datetime(self.now_utc, DateFormat.ERA) 

567 

568 @property 

569 def today(self) -> Date: 

570 """ 

571 Returns today's date. 

572 """ 

573 # noinspection PyTypeChecker 

574 return self.now.date() 

575 

576 # ------------------------------------------------------------------------- 

577 # Logos, static files, and other institution-specific stuff 

578 # ------------------------------------------------------------------------- 

579 

580 @property 

581 def url_local_institution(self) -> str: 

582 """ 

583 Returns the local institution's home URL. 

584 """ 

585 return self.config.local_institution_url 

586 

587 @property 

588 def url_camcops_favicon(self) -> str: 

589 """ 

590 Returns a URL to the favicon (see 

591 https://en.wikipedia.org/wiki/Favicon) from within the CamCOPS static 

592 files. 

593 """ 

594 # Cope with reverse proxies, etc. 

595 # https://docs.pylonsproject.org/projects/pyramid/en/latest/api/request.html#pyramid.request.Request.static_url # noqa 

596 return self.static_url( 

597 STATIC_CAMCOPS_PACKAGE_PATH + "favicon_camcops.png" 

598 ) 

599 

600 @property 

601 def url_camcops_logo(self) -> str: 

602 """ 

603 Returns a URL to the CamCOPS logo from within our static files. 

604 Returns: 

605 

606 """ 

607 return self.static_url( 

608 STATIC_CAMCOPS_PACKAGE_PATH + "logo_camcops.png" 

609 ) 

610 

611 @property 

612 def url_local_logo(self) -> str: 

613 """ 

614 Returns a URL to the local institution's logo, from somewhere on our 

615 server. 

616 """ 

617 return self.static_url(STATIC_CAMCOPS_PACKAGE_PATH + "logo_local.png") 

618 

619 @property 

620 def url_camcops_docs(self) -> str: 

621 """ 

622 Returns the URL to the CamCOPS documentation. 

623 """ 

624 return DOCUMENTATION_URL 

625 

626 # ------------------------------------------------------------------------- 

627 # Icons 

628 # ------------------------------------------------------------------------- 

629 

630 @staticmethod 

631 def icon( 

632 icon: str, 

633 alt: str, 

634 url: str = None, 

635 extra_classes: List[str] = None, 

636 extra_styles: List[str] = None, 

637 escape_alt: bool = True, 

638 ) -> str: 

639 """ 

640 Instantiates a Bootstrap icon, usually with a hyperlink. Returns 

641 rendered HTML. 

642 

643 Args: 

644 icon: 

645 Icon name, without ".svg" extension (or "bi-" prefix!). 

646 alt: 

647 Alternative text for image. 

648 url: 

649 Optional URL of hyperlink. 

650 extra_classes: 

651 Optional extra CSS classes for the icon. 

652 extra_styles: 

653 Optional extra CSS styles for the icon (each looks like: 

654 "color: blue"). 

655 escape_alt: 

656 HTML-escape the alt text? Default is True. 

657 """ 

658 return icon_html( 

659 icon=icon, 

660 alt=alt, 

661 url=url, 

662 extra_classes=extra_classes, 

663 extra_styles=extra_styles, 

664 escape_alt=escape_alt, 

665 ) 

666 

667 @staticmethod 

668 def icon_text( 

669 icon: str, 

670 text: str, 

671 url: str = None, 

672 alt: str = None, 

673 extra_icon_classes: List[str] = None, 

674 extra_icon_styles: List[str] = None, 

675 extra_a_classes: List[str] = None, 

676 extra_a_styles: List[str] = None, 

677 escape_alt: bool = True, 

678 escape_text: bool = True, 

679 hyperlink_together: bool = False, 

680 ) -> str: 

681 """ 

682 Provide an icon and accompanying text. Usually, both are hyperlinked 

683 (to the same destination URL). Returns rendered HTML. 

684 

685 Args: 

686 icon: 

687 Icon name, without ".svg" extension. 

688 url: 

689 Optional URL of hyperlink. 

690 alt: 

691 Alternative text for image. Will default to the main text. 

692 text: 

693 Main text to display. 

694 extra_icon_classes: 

695 Optional extra CSS classes for the icon. 

696 extra_icon_styles: 

697 Optional extra CSS styles for the icon (each looks like: 

698 "color: blue"). 

699 extra_a_classes: 

700 Optional extra CSS classes for the <a> element. 

701 extra_a_styles: 

702 Optional extra CSS styles for the <a> element. 

703 escape_alt: 

704 HTML-escape the alt text? 

705 escape_text: 

706 HTML-escape the main text? 

707 hyperlink_together: 

708 Hyperlink the image and text as one (rather than separately and 

709 adjacent to each other)? 

710 """ 

711 return icon_text( 

712 icon=icon, 

713 text=text, 

714 url=url, 

715 alt=alt, 

716 extra_icon_classes=extra_icon_classes, 

717 extra_icon_styles=extra_icon_styles, 

718 extra_a_classes=extra_a_classes, 

719 extra_a_styles=extra_a_styles, 

720 escape_alt=escape_alt, 

721 escape_text=escape_text, 

722 hyperlink_together=hyperlink_together, 

723 ) 

724 

725 @staticmethod 

726 def icons_text( 

727 icons: List[str], 

728 text: str, 

729 url: str = None, 

730 alt: str = None, 

731 extra_icon_classes: List[str] = None, 

732 extra_icon_styles: List[str] = None, 

733 extra_a_classes: List[str] = None, 

734 extra_a_styles: List[str] = None, 

735 escape_alt: bool = True, 

736 escape_text: bool = True, 

737 hyperlink_together: bool = False, 

738 ) -> str: 

739 """ 

740 Multiple-icon version of :meth:``icon_text``. 

741 """ 

742 return icons_text( 

743 icons=icons, 

744 text=text, 

745 url=url, 

746 alt=alt, 

747 extra_icon_classes=extra_icon_classes, 

748 extra_icon_styles=extra_icon_styles, 

749 extra_a_classes=extra_a_classes, 

750 extra_a_styles=extra_a_styles, 

751 escape_alt=escape_alt, 

752 escape_text=escape_text, 

753 hyperlink_together=hyperlink_together, 

754 ) 

755 

756 # ------------------------------------------------------------------------- 

757 # Low-level HTTP information 

758 # ------------------------------------------------------------------------- 

759 

760 @reify 

761 def remote_port(self) -> Optional[int]: 

762 """ 

763 What port number is the client using? 

764 

765 The ``remote_port`` variable is an optional WSGI extra provided by some 

766 frameworks, such as mod_wsgi. 

767 

768 The WSGI spec: 

769 - https://www.python.org/dev/peps/pep-0333/ 

770 

771 The CGI spec: 

772 - https://en.wikipedia.org/wiki/Common_Gateway_Interface 

773 

774 The Pyramid Request object: 

775 - https://docs.pylonsproject.org/projects/pyramid/en/latest/api/request.html#pyramid.request.Request 

776 - ... note: that includes ``remote_addr``, but not ``remote_port``. 

777 """ # noqa 

778 try: 

779 return int(self.environ.get("REMOTE_PORT", "")) 

780 except (TypeError, ValueError): 

781 return None 

782 

783 # ------------------------------------------------------------------------- 

784 # HTTP request convenience functions 

785 # ------------------------------------------------------------------------- 

786 

787 def has_param(self, key: str) -> bool: 

788 """ 

789 Is the parameter in the request? 

790 

791 Args: 

792 key: the parameter's name 

793 """ 

794 return key in self.params 

795 

796 def get_str_param( 

797 self, 

798 key: str, 

799 default: str = None, 

800 lower: bool = False, 

801 upper: bool = False, 

802 validator: STRING_VALIDATOR_TYPE = validate_alphanum_underscore, 

803 ) -> Optional[str]: 

804 """ 

805 Returns an HTTP parameter from the request (GET or POST). If it does 

806 not exist, or is blank, return ``default``. If it fails the validator, 

807 raise :exc:`pyramid.httpexceptions.HTTPBadRequest`. 

808 

809 Args: 

810 key: the parameter's name 

811 default: the value to return if the parameter is not found 

812 lower: convert to lower case? 

813 upper: convert to upper case? 

814 validator: validator function 

815 

816 Returns: 

817 the parameter's (string) contents, or ``default`` 

818 

819 """ 

820 # HTTP parameters are always strings at heart 

821 if key not in self.params: # missing from request? 

822 return default 

823 value = self.params.get(key) 

824 if not value: # blank, e.g. "source=" in URL? 

825 return default 

826 assert isinstance(value, str) # ... or we wouldn't have got here 

827 if lower: 

828 value = value.lower() 

829 elif upper: 

830 value = value.upper() 

831 try: 

832 validator(value, self) 

833 return value 

834 except ValueError as e: 

835 raise HTTPBadRequest(f"Bad {key!r} parameter: {e}") 

836 

837 def get_str_list_param( 

838 self, 

839 key: str, 

840 lower: bool = False, 

841 upper: bool = False, 

842 validator: STRING_VALIDATOR_TYPE = validate_alphanum_underscore, 

843 ) -> List[str]: 

844 """ 

845 Returns a list of HTTP parameter values from the request. Ensures all 

846 have been validated. 

847 

848 Args: 

849 key: the parameter's name 

850 lower: convert to lower case? 

851 upper: convert to upper case? 

852 validator: validator function 

853 

854 Returns: 

855 a list of string values 

856 

857 """ 

858 values = self.params.getall(key) 

859 if lower: 

860 values = [x.lower() for x in values] 

861 elif upper: 

862 values = [x.upper() for x in values] 

863 try: 

864 for v in values: 

865 validator(v, self) 

866 except ValueError as e: 

867 raise HTTPBadRequest( 

868 f"Parameter {key!r} contains a bad value: {e}" 

869 ) 

870 return values 

871 

872 def get_int_param(self, key: str, default: int = None) -> Optional[int]: 

873 """ 

874 Returns an integer parameter from the HTTP request. 

875 

876 Args: 

877 key: the parameter's name 

878 default: the value to return if the parameter is not found or is 

879 not a valid integer 

880 

881 Returns: 

882 an integer, or ``default`` 

883 

884 """ 

885 try: 

886 return int(self.params[key]) 

887 except (KeyError, TypeError, ValueError): 

888 return default 

889 

890 def get_int_list_param(self, key: str) -> List[int]: 

891 """ 

892 Returns a list of integer parameter values from the HTTP request. 

893 

894 Args: 

895 key: the parameter's name 

896 

897 Returns: 

898 a list of integer values 

899 

900 """ 

901 values = self.params.getall(key) 

902 try: 

903 return [int(x) for x in values] 

904 except (KeyError, TypeError, ValueError): 

905 return [] 

906 

907 def get_bool_param(self, key: str, default: bool) -> bool: 

908 """ 

909 Returns a boolean parameter from the HTTP request. 

910 

911 Args: 

912 key: the parameter's name 

913 default: the value to return if the parameter is not found or is 

914 not a valid boolean value 

915 

916 Returns: 

917 an integer, or ``default`` 

918 

919 Valid "true" and "false" values (case-insensitive): see 

920 ``TRUE_STRINGS_LOWER_CASE``, ``FALSE_STRINGS_LOWER_CASE``. 

921 """ 

922 try: 

923 param_str = self.params[key].lower() 

924 if param_str in TRUE_STRINGS_LOWER_CASE: 

925 return True 

926 elif param_str in FALSE_STRINGS_LOWER_CASE: 

927 return False 

928 else: 

929 return default 

930 except (AttributeError, KeyError, TypeError, ValueError): 

931 return default 

932 

933 def get_date_param(self, key: str) -> Optional[Date]: 

934 """ 

935 Returns a date parameter from the HTTP request. If it is missing or 

936 looks bad, return ``None``. 

937 

938 Args: 

939 key: the parameter's name 

940 

941 Returns: 

942 a :class:`pendulum.Date`, or ``None`` 

943 """ 

944 try: 

945 return coerce_to_pendulum_date(self.params[key]) 

946 except (KeyError, ParserError, TypeError, ValueError): 

947 return None 

948 

949 def get_datetime_param(self, key: str) -> Optional[Pendulum]: 

950 """ 

951 Returns a datetime parameter from the HTTP request. If it is missing or 

952 looks bad, return ``None``. 

953 

954 Args: 

955 key: the parameter's name 

956 

957 Returns: 

958 a :class:`pendulum.DateTime`, or ``None`` 

959 """ 

960 try: 

961 return coerce_to_pendulum(self.params[key]) 

962 except (KeyError, ParserError, TypeError, ValueError): 

963 return None 

964 

965 def get_redirect_url_param( 

966 self, key: str, default: str = None 

967 ) -> Optional[str]: 

968 """ 

969 Returns a redirection URL parameter from the HTTP request, validating 

970 it. (The validation process does not allow all types of URLs!) 

971 If it was missing, return ``default``. If it was bad, raise 

972 :exc:`pyramid.httpexceptions.HTTPBadRequest`. 

973 

974 Args: 

975 key: 

976 the parameter's name 

977 default: 

978 the value to return if the parameter is not found, or is 

979 invalid 

980 

981 Returns: 

982 a URL string, or ``default`` 

983 """ 

984 return self.get_str_param( 

985 key, default=default, validator=validate_redirect_url 

986 ) 

987 

988 # ------------------------------------------------------------------------- 

989 # Routing 

990 # ------------------------------------------------------------------------- 

991 

992 def route_url_params( 

993 self, route_name: str, paramdict: Dict[str, Any] 

994 ) -> str: 

995 """ 

996 Provides a simplified interface to :func:`Request.route_url` when you 

997 have parameters to pass. 

998 

999 It does two things: 

1000 

1001 (1) convert all params to their ``str()`` form; 

1002 (2) allow you to pass parameters more easily using a string 

1003 parameter name. 

1004 

1005 The normal Pyramid Request use is: 

1006 

1007 .. code-block:: python 

1008 

1009 Request.route_url(route_name, param1=value1, param2=value2) 

1010 

1011 where "param1" is the literal name of the parameter, but here we can do 

1012 

1013 .. code-block:: python 

1014 

1015 CamcopsRequest.route_url_params(route_name, { 

1016 PARAM1_NAME: value1_not_necessarily_str, 

1017 PARAM2_NAME: value2 

1018 }) 

1019 

1020 """ 

1021 strparamdict = {k: str(v) for k, v in paramdict.items()} 

1022 return self.route_url(route_name, **strparamdict) 

1023 

1024 # ------------------------------------------------------------------------- 

1025 # Strings 

1026 # ------------------------------------------------------------------------- 

1027 

1028 @reify 

1029 def _all_extra_strings(self) -> Dict[str, Dict[str, Dict[str, str]]]: 

1030 """ 

1031 Returns all CamCOPS "extra strings" (from XML files) in the format 

1032 used by :func:`camcops_server.cc_string.all_extra_strings_as_dicts`. 

1033 """ 

1034 return all_extra_strings_as_dicts(self.config_filename) 

1035 

1036 def xstring( 

1037 self, 

1038 taskname: str, 

1039 stringname: str, 

1040 default: str = None, 

1041 provide_default_if_none: bool = True, 

1042 language: str = None, 

1043 ) -> Optional[str]: 

1044 """ 

1045 Looks up a string from one of the optional extra XML string files. 

1046 

1047 Args: 

1048 taskname: task name (top-level key) 

1049 stringname: string name within task (second-level key) 

1050 default: default to return if the string is not found 

1051 provide_default_if_none: if ``True`` and ``default is None``, 

1052 return a helpful missing-string message in the style 

1053 "string x.y not found" 

1054 language: language code to use, e.g. ``en-GB``; if ``None`` is 

1055 passed, the default behaviour is to look up the current 

1056 language for this request (see :meth:`language`). 

1057 

1058 Returns: 

1059 the "extra string" 

1060 

1061 """ 

1062 # For speed, calculate default only if needed: 

1063 allstrings = self._all_extra_strings 

1064 if taskname in allstrings: 

1065 taskstrings = allstrings[taskname] 

1066 if stringname in taskstrings: 

1067 langversions = taskstrings[stringname] 

1068 if language is None: 

1069 language = self.language 

1070 if language: # Specific language requested 

1071 # 1. Requested language, e.g. "en-GB" 

1072 if language in langversions: 

1073 return langversions[language] 

1074 # 2. Same language, different country, e.g. "en-US" 

1075 shortlang = language[:2] # e.g. "en" 

1076 for key in langversions.keys(): 

1077 if key.startswith(shortlang): 

1078 return langversions[shortlang] 

1079 # 3. Default language 

1080 if DEFAULT_LOCALE in langversions: 

1081 return langversions[DEFAULT_LOCALE] 

1082 # 4. Strings with no language specified in the XML 

1083 if MISSING_LOCALE in langversions: 

1084 return langversions[MISSING_LOCALE] 

1085 # Not found 

1086 if default is None and provide_default_if_none: 

1087 default = ( 

1088 f"EXTRA_STRING_NOT_FOUND({taskname}.{stringname}[{language}])" 

1089 ) 

1090 return default 

1091 

1092 def wxstring( 

1093 self, 

1094 taskname: str, 

1095 stringname: str, 

1096 default: str = None, 

1097 provide_default_if_none: bool = True, 

1098 language: str = None, 

1099 ) -> Optional[str]: 

1100 """ 

1101 Returns a web-safe version of an :func:`xstring` (q.v.). 

1102 """ 

1103 value = self.xstring( 

1104 taskname, 

1105 stringname, 

1106 default, 

1107 provide_default_if_none=provide_default_if_none, 

1108 language=language, 

1109 ) 

1110 if value is None and not provide_default_if_none: 

1111 return None 

1112 return ws.webify(value) 

1113 

1114 def wappstring( 

1115 self, 

1116 stringname: str, 

1117 default: str = None, 

1118 provide_default_if_none: bool = True, 

1119 language: str = None, 

1120 ) -> Optional[str]: 

1121 """ 

1122 Returns a web-safe version of an appstring (an app-wide extra string). 

1123 This uses the XML file shared between the client and the server. 

1124 """ 

1125 value = self.xstring( 

1126 APPSTRING_TASKNAME, 

1127 stringname, 

1128 default, 

1129 provide_default_if_none=provide_default_if_none, 

1130 language=language, 

1131 ) 

1132 if value is None and not provide_default_if_none: 

1133 return None 

1134 return ws.webify(value) 

1135 

1136 def get_all_extra_strings(self) -> List[Tuple[str, str, str, str]]: 

1137 """ 

1138 Returns all extra strings, as a list of ``task, name, language, value`` 

1139 tuples. 

1140 

1141 2019-09-16: these are filtered according to the :ref:`RESTRICTED_TASKS 

1142 <RESTRICTED_TASKS>` option. 

1143 """ 

1144 restricted_tasks = self.config.restricted_tasks 

1145 user_group_names = None # type: Optional[Set[str]] 

1146 

1147 def task_permitted(task_xml_name: str) -> bool: 

1148 nonlocal user_group_names 

1149 if task_xml_name not in restricted_tasks: 

1150 return True 

1151 if user_group_names is None: 

1152 user_group_names = set(self.user.group_names) 

1153 permitted_group_names = set(restricted_tasks[task_xml_name]) 

1154 return bool(permitted_group_names.intersection(user_group_names)) 

1155 

1156 allstrings = self._all_extra_strings 

1157 rows = [] 

1158 for task, taskstrings in allstrings.items(): 

1159 if not task_permitted(task): 

1160 log.debug( 

1161 f"Skipping extra string download for task {task}: " 

1162 f"not permitted for user {self.user.username}" 

1163 ) 

1164 continue 

1165 for name, langversions in taskstrings.items(): 

1166 for language, value in langversions.items(): 

1167 rows.append((task, name, language, value)) 

1168 return rows 

1169 

1170 def task_extrastrings_exist(self, taskname: str) -> bool: 

1171 """ 

1172 Has the server been supplied with any extra strings for a specific 

1173 task? 

1174 """ 

1175 allstrings = self._all_extra_strings 

1176 return taskname in allstrings 

1177 

1178 def extrastring_families(self, sort: bool = True) -> List[str]: 

1179 """ 

1180 Which sets of extra strings do we have? A "family" here means, for 

1181 example, "the server itself", "the PHQ9 task", etc. 

1182 """ 

1183 families = list(self._all_extra_strings.keys()) 

1184 if sort: 

1185 families.sort() 

1186 return families 

1187 

1188 @reify 

1189 def language(self) -> str: 

1190 """ 

1191 Returns the language code selected by the current user, or if none is 

1192 selected (or the user isn't logged in) the server's default language. 

1193 

1194 Returns: 

1195 str: a language code of the form ``en-GB`` 

1196 

1197 """ 

1198 if self.user is not None: 

1199 language = self.user.language 

1200 if language in POSSIBLE_LOCALES: 

1201 return language 

1202 

1203 # Fallback to default 

1204 return self.config.language 

1205 

1206 @reify 

1207 def language_iso_639_1(self) -> str: 

1208 """ 

1209 Returns the language code selected by the current user, or if none is 

1210 selected (or the user isn't logged in) the server's default language. 

1211 

1212 This assumes all the possible supported languages start with a 

1213 two-letter primary language tag, which currently they do. 

1214 

1215 Returns: 

1216 str: a two-letter language code of the form ``en`` 

1217 

1218 """ 

1219 return self.language[:2] 

1220 

1221 def gettext(self, message: str) -> str: 

1222 """ 

1223 Returns a version of ``msg`` translated into the current language. 

1224 This is used for server-only strings. 

1225 

1226 The ``gettext()`` function is normally aliased to ``_()`` for 

1227 auto-translation tools to read the souce code. 

1228 """ 

1229 lang = self.language 

1230 # We can't work out if the string is missing; gettext falls back to 

1231 # the source message. 

1232 if lang == DEFAULT_LOCALE: 

1233 translated = message 

1234 else: 

1235 try: 

1236 translator = gettext.translation( 

1237 domain=GETTEXT_DOMAIN, 

1238 localedir=TRANSLATIONS_DIR, 

1239 languages=[lang], 

1240 ) 

1241 translated = translator.gettext(message) 

1242 except OSError: # e.g. translation file not found 

1243 log.warning(f"Failed to find translation files for {lang}") 

1244 translated = message 

1245 if DEBUG_GETTEXT: 

1246 return f"[{message}→{lang}→{translated}]" 

1247 else: 

1248 return translated 

1249 

1250 def wgettext(self, message: str) -> str: 

1251 """ 

1252 A web-safe version of :func:`gettext`. 

1253 """ 

1254 return ws.webify(self.gettext(message)) 

1255 

1256 def sstring(self, which_string: SS) -> str: 

1257 """ 

1258 Returns a translated server string via a lookup mechanism. 

1259 

1260 Args: 

1261 which_string: 

1262 which string? A :class:`camcops_server.cc_modules.cc_text.SS` 

1263 enumeration value 

1264 

1265 Returns: 

1266 str: the string 

1267 

1268 """ 

1269 try: 

1270 result = self._cached_sstring[which_string] 

1271 except KeyError: 

1272 result = server_string(self, which_string) 

1273 self._cached_sstring[which_string] = result 

1274 return result 

1275 

1276 def wsstring(self, which_string: SS) -> str: 

1277 """ 

1278 Returns a web-safe version of a translated server string via a lookup 

1279 mechanism. 

1280 

1281 Args: 

1282 which_string: 

1283 which string? A :class:`camcops_server.cc_modules.cc_text.SS` 

1284 enumeration value 

1285 

1286 Returns: 

1287 str: the string 

1288 

1289 """ 

1290 return ws.webify(self.sstring(which_string)) 

1291 

1292 # ------------------------------------------------------------------------- 

1293 # PNG versus SVG output, so tasks don't have to care (for e.g. PDF/web) 

1294 # ------------------------------------------------------------------------- 

1295 

1296 def prepare_for_pdf_figures(self) -> None: 

1297 """ 

1298 Switch the server (for this request) to producing figures in a format 

1299 most suitable for PDF. 

1300 """ 

1301 if CSS_PAGED_MEDIA: 

1302 # unlikely -- we use wkhtmltopdf instead now 

1303 self.switch_output_to_png() 

1304 # ... even weasyprint's SVG handling is inadequate 

1305 else: 

1306 # This is the main method -- we use wkhtmltopdf these days 

1307 self.switch_output_to_svg(provide_png_fallback=False) 

1308 # ... wkhtmltopdf can cope with SVGs 

1309 

1310 def prepare_for_html_figures(self) -> None: 

1311 """ 

1312 Switch the server (for this request) to producing figures in a format 

1313 most suitable for HTML. 

1314 """ 

1315 self.switch_output_to_svg() 

1316 

1317 def switch_output_to_png(self) -> None: 

1318 """ 

1319 Switch server (for this request) to producing figures in PNG format. 

1320 """ 

1321 self.use_svg = False 

1322 

1323 def switch_output_to_svg(self, provide_png_fallback: bool = True) -> None: 

1324 """ 

1325 Switch server (for this request) to producing figures in SVG format. 

1326 

1327 Args: 

1328 provide_png_fallback: 

1329 Offer a PNG fallback option/ 

1330 """ 

1331 self.use_svg = True 

1332 self.provide_png_fallback_for_svg = provide_png_fallback 

1333 

1334 @staticmethod 

1335 def create_figure(**kwargs) -> Figure: 

1336 """ 

1337 Creates and returns a :class:`matplotlib.figure.Figure` with a canvas. 

1338 The canvas will be available as ``fig.canvas``. 

1339 """ 

1340 fig = Figure(**kwargs) 

1341 # noinspection PyUnusedLocal 

1342 canvas = FigureCanvas(fig) # noqa: F841 

1343 # The canvas will be now available as fig.canvas, since 

1344 # FigureCanvasBase.__init__ calls fig.set_canvas(self); similarly, the 

1345 # figure is available from the canvas as canvas.figure 

1346 

1347 # How do we set the font, so the caller doesn't have to? 

1348 # The "nasty global" way is: 

1349 # matplotlib.rc('font', **fontdict) 

1350 # matplotlib.rc('legend', **fontdict) 

1351 # or similar. Then matplotlib often works its way round to using its 

1352 # global rcParams object, which is Not OK in a multithreaded context. 

1353 # 

1354 # https://github.com/matplotlib/matplotlib/issues/6514 

1355 # https://github.com/matplotlib/matplotlib/issues/6518 

1356 # 

1357 # The other way is to specify a fontdict with each call, e.g. 

1358 # ax.set_xlabel("some label", **fontdict) 

1359 # https://stackoverflow.com/questions/21321670/how-to-change-fonts-in-matplotlib-python # noqa 

1360 # Relevant calls with explicit "fontdict: Dict" parameters: 

1361 # ax.set_xlabel(..., fontdict=XXX, ...) 

1362 # ax.set_ylabel(..., fontdict=XXX, ...) 

1363 # ax.set_xticklabels(..., fontdict=XXX, ...) 

1364 # ax.set_yticklabels(..., fontdict=XXX, ...) 

1365 # ax.text(..., fontdict=XXX, ...) 

1366 # ax.set_label_text(..., fontdict=XXX, ...) 

1367 # ax.set_title(..., fontdict=XXX, ...) 

1368 # 

1369 # And with "fontproperties: FontProperties" 

1370 # sig.suptitle(..., fontproperties=XXX, ...) 

1371 # 

1372 # And with "prop: FontProperties": 

1373 # ax.legend(..., prop=XXX, ...) 

1374 # 

1375 # Then, some things are automatically plotted... 

1376 

1377 return fig 

1378 

1379 @reify 

1380 def fontdict(self) -> Dict[str, Any]: 

1381 """ 

1382 Returns a font dictionary for use with Matplotlib plotting. 

1383 

1384 **matplotlib font handling and fontdict parameter** 

1385 

1386 - https://stackoverflow.com/questions/3899980 

1387 - https://matplotlib.org/users/customizing.html 

1388 - matplotlib/font_manager.py 

1389 

1390 - Note that the default TrueType font is "DejaVu Sans"; see 

1391 :class:`matplotlib.font_manager.FontManager` 

1392 

1393 - Example sequence: 

1394 

1395 - CamCOPS does e.g. ``ax.set_xlabel("Date/time", 

1396 fontdict=self.req.fontdict)`` 

1397 

1398 - matplotlib.axes.Axes.set_xlabel: 

1399 https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.set_xlabel.html 

1400 

1401 - matplotlib.axes.Axes.text documentation, explaining the fontdict 

1402 parameter: 

1403 https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.text.html 

1404 

1405 - What's created is probably a :class:`matplotlib.text.Text` object, 

1406 whose ``update()`` function is called with the dictionary. Via its 

1407 superclass :class:`matplotlib.artist.Artist` and its ``update()`` 

1408 function, this sets attributes on the Text object. Ultimately, 

1409 without having explored this in too much depth, it's probably the 

1410 ``self._fontproperties`` object of Text that holds this info. 

1411 

1412 - That is an instance of 

1413 :class:`matplotlib.font_manager.FontProperties`. 

1414 

1415 **Linux fonts** 

1416 

1417 Anyway, the main things are (1) that the relevant fonts need to be 

1418 installed, and (2) that the default is DejaVu Sans. 

1419 

1420 - Linux fonts are installed in ``/usr/share/fonts``, and TrueType fonts 

1421 within ``/usr/share/fonts/truetype``. 

1422 

1423 - Use ``fc-match`` to see the font mappings being used. 

1424 

1425 - Use ``fc-list`` to list available fonts. 

1426 

1427 - Use ``fc-cache`` to rebuild the font cache. 

1428 

1429 - Files in ``/etc/fonts/conf.avail/`` do some thinking. 

1430 

1431 **Problems with pixellated fonts in PDFs made via wkhtmltopdf** 

1432 

1433 - See also https://github.com/wkhtmltopdf/wkhtmltopdf/issues/2193, 

1434 about pixellated fonts via wkhtmltopdf (which was our problem for a 

1435 subset of the fonts in trackers, on 2020-06-28, using wkhtmltopd 

1436 0.12.5 with patched Qt). 

1437 

1438 - When you get pixellated fonts in a PDF, look also at the embedded 

1439 font list in the PDF (e.g. in Okular: File -> Properties -> Fonts). 

1440 

1441 - Matplotlib helpfully puts the text (rendered as lines in SVG) as 

1442 comments. 

1443 

1444 - As a debugging sequence, we can manually trim the "pdfhtml" output 

1445 down to just the SVG file. Still has problems. Yet there's no text 

1446 in it; the text is made of pure SVG lines. And Chrome renders it 

1447 perfectly. As does Firefox. 

1448 

1449 - The rendering bug goes away entirely if you delete the opacity 

1450 styling throughout the SVG: 

1451 

1452 .. code-block:: none 

1453 

1454 <g style="opacity:0.5;" transform=...> 

1455 ^^^^^^^^^^^^^^^^^^^^ 

1456 this 

1457 

1458 - So, simple fix: 

1459 

1460 - rather than opacity (alpha) 0.5 and on top... 

1461 

1462 - 50% grey colour and on the bottom. 

1463 

1464 """ # noqa 

1465 fontsize = self.config.plot_fontsize 

1466 return dict( 

1467 family="sans-serif", 

1468 # ... serif, sans-serif, cursive, fantasy, monospace 

1469 style="normal", # normal (roman), italic, oblique 

1470 variant="normal", # normal, small-caps 

1471 weight="normal", 

1472 # ... normal [=400], bold [=700], bolder [relative to current], 

1473 # lighter [relative], 100, 200, 300, ..., 900 

1474 size=fontsize, # in pt (default 12) 

1475 ) 

1476 

1477 @reify 

1478 def fontprops(self) -> FontProperties: 

1479 """ 

1480 Return a :class:`matplotlib.font_manager.FontProperties` object for 

1481 use with Matplotlib plotting. 

1482 """ 

1483 return FontProperties(**self.fontdict) 

1484 

1485 def set_figure_font_sizes( 

1486 self, 

1487 ax: "Axes", # "SubplotBase", 

1488 fontdict: Dict[str, Any] = None, 

1489 x_ticklabels: bool = True, 

1490 y_ticklabels: bool = True, 

1491 ) -> None: 

1492 """ 

1493 Sets font sizes for the axes of the specified Matplotlib figure. 

1494 

1495 Args: 

1496 ax: the figure to modify 

1497 fontdict: the font dictionary to use (if omitted, the default 

1498 will be used) 

1499 x_ticklabels: if ``True``, modify the X-axis tick labels 

1500 y_ticklabels: if ``True``, modify the Y-axis tick labels 

1501 """ 

1502 final_fontdict = self.fontdict.copy() 

1503 if fontdict: 

1504 final_fontdict.update(fontdict) 

1505 fp = FontProperties(**final_fontdict) 

1506 

1507 axes = [] # type: List[Axis] 

1508 if x_ticklabels: # and hasattr(ax, "xaxis"): 

1509 axes.append(ax.xaxis) 

1510 if y_ticklabels: # and hasattr(ax, "yaxis"): 

1511 axes.append(ax.yaxis) 

1512 for axis in axes: 

1513 for ticklabel in axis.get_ticklabels( 

1514 which="both" 

1515 ): # type: Text # I think! 

1516 ticklabel.set_fontproperties(fp) 

1517 

1518 def get_html_from_pyplot_figure(self, fig: Figure) -> str: 

1519 """ 

1520 Make HTML (as PNG or SVG) from pyplot 

1521 :class:`matplotlib.figure.Figure`. 

1522 """ 

1523 if USE_SVG_IN_HTML and self.use_svg: 

1524 result = svg_html_from_pyplot_figure(fig) 

1525 if self.provide_png_fallback_for_svg: 

1526 # return both an SVG and a PNG image, for browsers that can't 

1527 # deal with SVG; the Javascript header will sort this out 

1528 # http://www.voormedia.nl/blog/2012/10/displaying-and-detecting-support-for-svg-images # noqa 

1529 result += png_img_html_from_pyplot_figure( 

1530 fig, PlotDefaults.DEFAULT_PLOT_DPI, "pngfallback" 

1531 ) 

1532 return result 

1533 else: 

1534 return png_img_html_from_pyplot_figure( 

1535 fig, PlotDefaults.DEFAULT_PLOT_DPI 

1536 ) 

1537 

1538 # ------------------------------------------------------------------------- 

1539 # Convenience functions for user information 

1540 # ------------------------------------------------------------------------- 

1541 

1542 @property 

1543 def user(self) -> Optional["User"]: 

1544 """ 

1545 Returns the :class:`camcops_server.cc_modules.cc_user.User` for the 

1546 current request. 

1547 """ 

1548 return self._debugging_user or self.camcops_session.user 

1549 

1550 @property 

1551 def user_id(self) -> Optional[int]: 

1552 """ 

1553 Returns the integer user ID for the current request. 

1554 """ 

1555 if self._debugging_user: 

1556 return self._debugging_user.id 

1557 return self.camcops_session.user_id 

1558 

1559 # ------------------------------------------------------------------------- 

1560 # ID number definitions 

1561 # ------------------------------------------------------------------------- 

1562 

1563 @reify 

1564 def idnum_definitions(self) -> List[IdNumDefinition]: 

1565 """ 

1566 Returns all 

1567 :class:`camcops_server.cc_modules.cc_idnumdef.IdNumDefinition` objects. 

1568 """ 

1569 return get_idnum_definitions(self.dbsession) # no longer cached 

1570 

1571 @reify 

1572 def valid_which_idnums(self) -> List[int]: 

1573 """ 

1574 Returns the ``which_idnum`` values for all ID number definitions. 

1575 """ 

1576 return [iddef.which_idnum for iddef in self.idnum_definitions] 

1577 # ... pre-sorted 

1578 

1579 def get_idnum_definition( 

1580 self, which_idnum: int 

1581 ) -> Optional[IdNumDefinition]: 

1582 """ 

1583 Retrieves an 

1584 :class:`camcops_server.cc_modules.cc_idnumdef.IdNumDefinition` for the 

1585 specified ``which_idnum`` value. 

1586 """ 

1587 return next( 

1588 ( 

1589 iddef 

1590 for iddef in self.idnum_definitions 

1591 if iddef.which_idnum == which_idnum 

1592 ), 

1593 None, 

1594 ) 

1595 

1596 def get_id_desc( 

1597 self, which_idnum: int, default: str = None 

1598 ) -> Optional[str]: 

1599 """ 

1600 Get the server's ID description for the specified ``which_idnum`` 

1601 value. 

1602 """ 

1603 return next( 

1604 ( 

1605 iddef.description 

1606 for iddef in self.idnum_definitions 

1607 if iddef.which_idnum == which_idnum 

1608 ), 

1609 default, 

1610 ) 

1611 

1612 def get_id_shortdesc( 

1613 self, which_idnum: int, default: str = None 

1614 ) -> Optional[str]: 

1615 """ 

1616 Get the server's short ID description for the specified ``which_idnum`` 

1617 value. 

1618 """ 

1619 return next( 

1620 ( 

1621 iddef.short_description 

1622 for iddef in self.idnum_definitions 

1623 if iddef.which_idnum == which_idnum 

1624 ), 

1625 default, 

1626 ) 

1627 

1628 def is_idnum_valid( 

1629 self, which_idnum: int, idnum_value: Optional[int] 

1630 ) -> bool: 

1631 """ 

1632 Does the ID number pass any extended validation checks? 

1633 

1634 Args: 

1635 which_idnum: which ID number type is this? 

1636 idnum_value: ID number value 

1637 

1638 Returns: 

1639 bool: valid? 

1640 """ 

1641 idnumdef = self.get_idnum_definition(which_idnum) 

1642 if not idnumdef: 

1643 return False 

1644 valid, _ = validate_id_number( 

1645 self, idnum_value, idnumdef.validation_method 

1646 ) 

1647 return valid 

1648 

1649 def why_idnum_invalid( 

1650 self, which_idnum: int, idnum_value: Optional[int] 

1651 ) -> str: 

1652 """ 

1653 Why does the ID number fail any extended validation checks? 

1654 

1655 Args: 

1656 which_idnum: which ID number type is this? 

1657 idnum_value: ID number value 

1658 

1659 Returns: 

1660 str: why invalid? (Human-readable string.) 

1661 """ 

1662 idnumdef = self.get_idnum_definition(which_idnum) 

1663 if not idnumdef: 

1664 _ = self.gettext 

1665 return _("Can't fetch ID number definition") 

1666 _, why = validate_id_number( 

1667 self, idnum_value, idnumdef.validation_method 

1668 ) 

1669 return why 

1670 

1671 # ------------------------------------------------------------------------- 

1672 # Server settings 

1673 # ------------------------------------------------------------------------- 

1674 

1675 @reify 

1676 def server_settings(self) -> ServerSettings: 

1677 """ 

1678 Return the 

1679 :class:`camcops_server.cc_modules.cc_serversettings.ServerSettings` for 

1680 the server. 

1681 """ 

1682 return get_server_settings(self) 

1683 

1684 @reify 

1685 def database_title(self) -> str: 

1686 """ 

1687 Return the database friendly title for the server. 

1688 """ 

1689 ss = self.server_settings 

1690 return ss.database_title or "" 

1691 

1692 def set_database_title(self, title: str) -> None: 

1693 """ 

1694 Sets the database friendly title for the server. 

1695 """ 

1696 ss = self.server_settings 

1697 ss.database_title = title 

1698 

1699 # ------------------------------------------------------------------------- 

1700 # SNOMED-CT 

1701 # ------------------------------------------------------------------------- 

1702 

1703 @reify 

1704 def snomed_supported(self) -> bool: 

1705 """ 

1706 Is SNOMED-CT supported for CamCOPS tasks? 

1707 """ 

1708 return bool(self.config.get_task_snomed_concepts()) 

1709 

1710 def snomed(self, lookup: str) -> "SnomedConcept": 

1711 """ 

1712 Fetches a SNOMED-CT concept for a CamCOPS task. 

1713 

1714 Args: 

1715 lookup: a CamCOPS SNOMED lookup string 

1716 

1717 Returns: 

1718 a :class:`camcops_server.cc_modules.cc_snomed.SnomedConcept` 

1719 

1720 Raises: 

1721 :exc:`KeyError`, if the lookup cannot be found (e.g. UK data not 

1722 installed) 

1723 """ 

1724 concepts = self.config.get_task_snomed_concepts() 

1725 assert concepts, "No SNOMED-CT data available for CamCOPS tasks" 

1726 return concepts[lookup] 

1727 

1728 @reify 

1729 def icd9cm_snomed_supported(self) -> bool: 

1730 """ 

1731 Is SNOMED-CT supported for ICD-9-CM codes? 

1732 """ 

1733 return bool(self.config.get_icd9cm_snomed_concepts()) 

1734 

1735 def icd9cm_snomed(self, code: str) -> List["SnomedConcept"]: 

1736 """ 

1737 Fetches a SNOMED-CT concept for an ICD-9-CM code 

1738 

1739 Args: 

1740 code: an ICD-9-CM code 

1741 

1742 Returns: 

1743 a :class:`camcops_server.cc_modules.cc_snomed.SnomedConcept` 

1744 

1745 Raises: 

1746 :exc:`KeyError`, if the lookup cannot be found (e.g. data not 

1747 installed) 

1748 """ 

1749 concepts = self.config.get_icd9cm_snomed_concepts() 

1750 assert concepts, "No SNOMED-CT data available for ICD-9-CM" 

1751 return concepts[code] 

1752 

1753 @reify 

1754 def icd10_snomed_supported(self) -> bool: 

1755 """ 

1756 Is SNOMED-CT supported for ICD-10 codes? 

1757 """ 

1758 return bool(self.config.get_icd9cm_snomed_concepts()) 

1759 

1760 def icd10_snomed(self, code: str) -> List["SnomedConcept"]: 

1761 """ 

1762 Fetches a SNOMED-CT concept for an ICD-10 code 

1763 

1764 Args: 

1765 code: an ICD-10 code 

1766 

1767 Returns: 

1768 a :class:`camcops_server.cc_modules.cc_snomed.SnomedConcept` 

1769 

1770 Raises: 

1771 :exc:`KeyError`, if the lookup cannot be found (e.g. data not 

1772 installed) 

1773 """ 

1774 concepts = self.config.get_icd10_snomed_concepts() 

1775 assert concepts, "No SNOMED-CT data available for ICD-10" 

1776 return concepts[code] 

1777 

1778 # ------------------------------------------------------------------------- 

1779 # Export recipients 

1780 # ------------------------------------------------------------------------- 

1781 

1782 def get_export_recipients( 

1783 self, 

1784 recipient_names: List[str] = None, 

1785 all_recipients: bool = False, 

1786 all_push_recipients: bool = False, 

1787 save: bool = True, 

1788 database_versions: bool = True, 

1789 ) -> List[Union["ExportRecipient", "ExportRecipientInfo"]]: 

1790 """ 

1791 Returns a list of export recipients, with some filtering if desired. 

1792 Validates them against the database. 

1793 

1794 - If ``all_recipients``, return all. 

1795 - Otherwise, if ``all_push_recipients``, return all "push" recipients. 

1796 - Otherwise, return all named in ``recipient_names``. 

1797 

1798 - If any are invalid, raise an error. 

1799 - If any are duplicate, raise an error. 

1800 

1801 - Holds a global export file lock for some database access relating to 

1802 export recipient records. 

1803 

1804 Args: 

1805 all_recipients: use all recipients? 

1806 all_push_recipients: use all "push" recipients? 

1807 recipient_names: recipient names 

1808 save: save any freshly created recipient records to the DB? 

1809 database_versions: return ExportRecipient objects that are attached 

1810 to a database session (rather than ExportRecipientInfo objects 

1811 that aren't)? 

1812 

1813 Returns: 

1814 list: of :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient` 

1815 

1816 Raises: 

1817 - :exc:`ValueError` if a name is invalid 

1818 - :exc:`ValueError` if a name is duplicated 

1819 - :exc:`camcops_server.cc_modules.cc_exportrecipient.InvalidExportRecipient` 

1820 if an export recipient configuration is invalid 

1821 """ # noqa 

1822 # Delayed imports 

1823 from camcops_server.cc_modules.cc_exportrecipient import ( 

1824 ExportRecipient, 

1825 ) # delayed import 

1826 

1827 # Check parameters 

1828 recipient_names = recipient_names or [] # type: List[str] 

1829 if save and not database_versions: 

1830 raise AssertionError("Can't save unless taking database versions") 

1831 

1832 # Start with ExportRecipientInfo objects: 

1833 recipientinfolist = self.config.get_all_export_recipient_info() 

1834 

1835 # Restrict 

1836 if not all_recipients: 

1837 if all_push_recipients: 

1838 recipientinfolist = [r for r in recipientinfolist if r.push] 

1839 else: 

1840 # Specified by name 

1841 duplicates = [ 

1842 name 

1843 for name, count in collections.Counter( 

1844 recipient_names 

1845 ).items() 

1846 if count > 1 

1847 ] 

1848 if duplicates: 

1849 raise ValueError( 

1850 f"Duplicate export recipients " 

1851 f"specified: {duplicates!r}" 

1852 ) 

1853 valid_names = set(r.recipient_name for r in recipientinfolist) 

1854 bad_names = [ 

1855 name for name in recipient_names if name not in valid_names 

1856 ] 

1857 if bad_names: 

1858 raise ValueError( 

1859 f"Bad export recipients specified: {bad_names!r}. " 

1860 f"Valid recipients are: {valid_names!r}" 

1861 ) 

1862 recipientinfolist = [ 

1863 r 

1864 for r in recipientinfolist 

1865 if r.recipient_name in recipient_names 

1866 ] 

1867 

1868 # Complete validation 

1869 for r in recipientinfolist: 

1870 r.validate(self) 

1871 

1872 # Does the caller want them as ExportRecipientInfo objects 

1873 if not database_versions: 

1874 return recipientinfolist 

1875 

1876 # Convert to SQLAlchemy ORM ExportRecipient objects: 

1877 recipients = [ 

1878 ExportRecipient(x) for x in recipientinfolist 

1879 ] # type: List[ExportRecipient] 

1880 

1881 final_recipients = [] # type: List[ExportRecipient] 

1882 dbsession = self.dbsession 

1883 

1884 def process_final_recipients(_save: bool) -> None: 

1885 for r in recipients: 

1886 other = ExportRecipient.get_existing_matching_recipient( 

1887 dbsession, r 

1888 ) 

1889 if other: 

1890 # This other one matches, and is already in the database. 

1891 # Use it. But first... 

1892 for ( 

1893 attrname 

1894 ) in ( 

1895 ExportRecipient.RECOPY_EACH_TIME_FROM_CONFIG_ATTRNAMES 

1896 ): 

1897 setattr(other, attrname, getattr(r, attrname)) 

1898 # OK. 

1899 final_recipients.append(other) 

1900 else: 

1901 # Our new object doesn't match. Use (+/- save) it. 

1902 if save: 

1903 log.debug( 

1904 "Creating new ExportRecipient record in database" 

1905 ) 

1906 dbsession.add(r) 

1907 r.current = True 

1908 final_recipients.append(r) 

1909 

1910 if save: 

1911 lockfilename = ( 

1912 self.config.get_master_export_recipient_lockfilename() 

1913 ) 

1914 with lockfile.FileLock( 

1915 lockfilename, timeout=None 

1916 ): # waits forever if necessary 

1917 process_final_recipients(_save=True) 

1918 else: 

1919 process_final_recipients(_save=False) 

1920 

1921 # OK 

1922 return final_recipients 

1923 

1924 def get_export_recipient( 

1925 self, recipient_name: str, save: bool = True 

1926 ) -> "ExportRecipient": 

1927 """ 

1928 Returns a single validated export recipient, given its name. 

1929 

1930 Args: 

1931 recipient_name: recipient name 

1932 save: save any freshly created recipient records to the DB? 

1933 

1934 Returns: 

1935 list: of :class:`camcops_server.cc_modules.cc_exportrecipient.ExportRecipient` 

1936 

1937 Raises: 

1938 - :exc:`ValueError` if a name is invalid 

1939 - :exc:`camcops_server.cc_modules.cc_exportrecipient.InvalidExportRecipient` 

1940 if an export recipient configuration is invalid 

1941 """ # noqa 

1942 recipients = self.get_export_recipients([recipient_name], save=save) 

1943 assert len(recipients) == 1 

1944 return recipients[0] 

1945 

1946 @reify 

1947 def all_push_recipients(self) -> List["ExportRecipient"]: 

1948 """ 

1949 Cached for speed (will potentially be called for multiple tables in 

1950 a bulk upload). 

1951 """ 

1952 return self.get_export_recipients( 

1953 all_push_recipients=True, 

1954 save=False, 

1955 database_versions=True, # we need group ID info somehow 

1956 ) 

1957 

1958 def add_export_push_request( 

1959 self, recipient_name: str, basetable: str, task_pk: int 

1960 ) -> None: 

1961 """ 

1962 Adds a request to push a task to an export recipient. 

1963 

1964 The reason we use this slightly convoluted approach is because 

1965 otherwise, it's very easy to generate a backend request for a new task 

1966 before it's actually been committed (so the backend finds no task). 

1967 

1968 Args: 

1969 recipient_name: name of the recipient 

1970 basetable: name of the task's base table 

1971 task_pk: server PK of the task 

1972 """ 

1973 self._pending_export_push_requests.append( 

1974 (recipient_name, basetable, task_pk) 

1975 ) 

1976 

1977 def _process_pending_export_push_requests(self) -> None: 

1978 """ 

1979 Sends pending export push requests to the backend. 

1980 

1981 Called after the COMMIT. 

1982 """ 

1983 from camcops_server.cc_modules.celery import ( 

1984 export_task_backend, 

1985 ) # delayed import 

1986 

1987 for ( 

1988 recipient_name, 

1989 basetable, 

1990 task_pk, 

1991 ) in self._pending_export_push_requests: 

1992 log.info( 

1993 "Submitting background job to export task {}.{} to {}", 

1994 basetable, 

1995 task_pk, 

1996 recipient_name, 

1997 ) 

1998 export_task_backend.delay( 

1999 recipient_name=recipient_name, 

2000 basetable=basetable, 

2001 task_pk=task_pk, 

2002 ) 

2003 

2004 # ------------------------------------------------------------------------- 

2005 # User downloads 

2006 # ------------------------------------------------------------------------- 

2007 

2008 @property 

2009 def user_download_dir(self) -> str: 

2010 """ 

2011 The directory in which this user's downloads should be/are stored, or a 

2012 blank string if user downloads are not available. Also ensures it 

2013 exists. 

2014 """ 

2015 if self.config.user_download_max_space_mb <= 0: 

2016 return "" 

2017 basedir = self.config.user_download_dir 

2018 if not basedir: 

2019 return "" 

2020 user_id = self.user_id 

2021 if user_id is None: 

2022 return "" 

2023 userdir = os.path.join(basedir, str(user_id)) 

2024 mkdir_p(userdir) 

2025 return userdir 

2026 

2027 @property 

2028 def user_download_bytes_permitted(self) -> int: 

2029 """ 

2030 Amount of space the user is permitted. 

2031 """ 

2032 if not self.user_download_dir: 

2033 return 0 

2034 return self.config.user_download_max_space_mb * 1024 * 1024 

2035 

2036 @reify 

2037 def user_download_bytes_used(self) -> int: 

2038 """ 

2039 Returns the disk space used by this user. 

2040 """ 

2041 download_dir = self.user_download_dir 

2042 if not download_dir: 

2043 return 0 

2044 return get_directory_contents_size(download_dir) 

2045 

2046 @property 

2047 def user_download_bytes_available(self) -> int: 

2048 """ 

2049 Returns the available space for this user in their download area. 

2050 """ 

2051 permitted = self.user_download_bytes_permitted 

2052 used = self.user_download_bytes_used 

2053 available = permitted - used 

2054 return available 

2055 

2056 @property 

2057 def user_download_lifetime_duration(self) -> Duration: 

2058 """ 

2059 Returns the lifetime of user download objects. 

2060 """ 

2061 return Duration(minutes=self.config.user_download_file_lifetime_min) 

2062 

2063 

2064# noinspection PyUnusedLocal 

2065def complete_request_add_cookies( 

2066 req: CamcopsRequest, response: Response 

2067) -> None: 

2068 """ 

2069 Finializes the response by adding session cookies. 

2070 

2071 See :meth:`CamcopsRequest.complete_request_add_cookies`. 

2072 """ 

2073 req.complete_request_add_cookies() 

2074 

2075 

2076# ============================================================================= 

2077# Configurator 

2078# ============================================================================= 

2079 

2080 

2081@contextmanager 

2082def camcops_pyramid_configurator_context( 

2083 debug_toolbar: bool = False, static_cache_duration_s: int = 0 

2084) -> Configurator: 

2085 """ 

2086 Context manager to create a Pyramid configuration context, for making 

2087 (for example) a WSGI server or a debugging request. That means setting up 

2088 things like: 

2089 

2090 - the authentication and authorization policies 

2091 - our request and session factories 

2092 - our Mako renderer 

2093 - our routes and views 

2094 

2095 Args: 

2096 debug_toolbar: 

2097 Add the Pyramid debug toolbar? 

2098 static_cache_duration_s: 

2099 Lifetime (in seconds) for the HTTP cache-control setting for 

2100 static content. 

2101 

2102 Returns: 

2103 a :class:`Configurator` object 

2104 

2105 Note this includes settings that transcend the config file. 

2106 

2107 Most things should be in the config file. This enables us to run multiple 

2108 configs (e.g. multiple CamCOPS databases) through the same process. 

2109 However, some things we need to know right now, to make the WSGI app. 

2110 Here, OS environment variables and command-line switches are appropriate. 

2111 """ 

2112 

2113 # ------------------------------------------------------------------------- 

2114 # 1. Base app 

2115 # ------------------------------------------------------------------------- 

2116 settings = { # Settings that can't be set directly? 

2117 "debug_authorization": DEBUG_AUTHORIZATION, 

2118 # ... see https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/security.html#debugging-view-authorization-failures # noqa 

2119 } 

2120 with Configurator(settings=settings) as config: 

2121 # --------------------------------------------------------------------- 

2122 # Authentication; authorizaion (permissions) 

2123 # --------------------------------------------------------------------- 

2124 authentication_policy = CamcopsAuthenticationPolicy() 

2125 config.set_authentication_policy(authentication_policy) 

2126 # Let's not use ACLAuthorizationPolicy, which checks an access control 

2127 # list for a resource hierarchy of objects, but instead: 

2128 authorization_policy = CamcopsAuthorizationPolicy() 

2129 config.set_authorization_policy(authorization_policy) 

2130 config.set_default_permission(Permission.HAPPY) 

2131 # ... applies to all SUBSEQUENT view configuration registrations 

2132 

2133 # --------------------------------------------------------------------- 

2134 # Factories 

2135 # --------------------------------------------------------------------- 

2136 config.set_request_factory(CamcopsRequest) 

2137 # ... for request attributes: config, database, etc. 

2138 config.set_session_factory(get_session_factory()) 

2139 # ... for request.session 

2140 config.set_response_factory(camcops_response_factory) 

2141 

2142 # --------------------------------------------------------------------- 

2143 # Renderers 

2144 # --------------------------------------------------------------------- 

2145 camcops_add_mako_renderer(config, extension=".mako") 

2146 

2147 # deform_bootstrap.includeme(config) 

2148 

2149 # --------------------------------------------------------------------- 

2150 # Routes and accompanying views 

2151 # --------------------------------------------------------------------- 

2152 

2153 # Add static views 

2154 # https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/assets.html#serving-static-assets # noqa 

2155 # Hmm. We cannot fail to set up a static file route, because otherwise 

2156 # we can't provide URLs to them. 

2157 static_filepath = STATIC_CAMCOPS_PACKAGE_PATH 

2158 static_name = RouteCollection.STATIC.route 

2159 log.debug( 

2160 "... including static files from {!r} at Pyramid static " 

2161 "name {!r}", 

2162 static_filepath, 

2163 static_name, 

2164 ) 

2165 # ... does the name needs to start with "/" or the pattern "static/" 

2166 # will override the later "deform_static"? Not sure. 

2167 

2168 # We were doing this: 

2169 # config.add_static_view(name=static_name, path=static_filepath) 

2170 # But now we need to (a) add the 

2171 # "cache_max_age=static_cache_duration_s" argument, and (b) set the 

2172 # HTTP header 'Cache-Control: no-cache="Set-Cookie, Set-Cookie2"', 

2173 # for the ZAP penetration tester: 

2174 # ... https://cheatsheetseries.owasp.org/cheatsheets/Session_Management_Cheat_Sheet.html#web-content-caching # noqa 

2175 # We can do the former, but not the latter, via add_static_view(), 

2176 # because it sends its keyword arguments to add_route(), not the view 

2177 # creation. So, alternatives ways... 

2178 # - from https://github.com/Pylons/pyramid/issues/1486 

2179 # - and https://stackoverflow.com/questions/24854300/ 

2180 # - to https://github.com/Pylons/pyramid/pull/2021 

2181 # - to https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/hooks.html#view-derivers # noqa 

2182 

2183 config.add_static_view( 

2184 name=static_name, 

2185 path=static_filepath, 

2186 cache_max_age=static_cache_duration_s, 

2187 ) 

2188 

2189 # Add all the routes: 

2190 for pr in RouteCollection.all_routes(): 

2191 if DEBUG_ADD_ROUTES: 

2192 suffix = ( 

2193 f", pregenerator={pr.pregenerator}" 

2194 if pr.pregenerator 

2195 else "" 

2196 ) 

2197 log.info("Adding route: {} -> {}{}", pr.route, pr.path, suffix) 

2198 config.add_route(pr.route, pr.path, pregenerator=pr.pregenerator) 

2199 # See also: 

2200 # https://stackoverflow.com/questions/19184612/how-to-ensure-urls-generated-by-pyramids-route-url-and-route-path-are-valid # noqa 

2201 

2202 # Routes added EARLIER have priority. So add this AFTER our custom 

2203 # bugfix: 

2204 config.add_static_view( 

2205 name="/deform_static", 

2206 path="deform:static/", 

2207 cache_max_age=static_cache_duration_s, 

2208 ) 

2209 

2210 # Most views are using @view_config() which calls add_view(). 

2211 # Scan for @view_config decorators, to map views to routes: 

2212 # https://docs.pylonsproject.org/projects/venusian/en/latest/api.html 

2213 config.scan( 

2214 "camcops_server.cc_modules", ignore=[re.compile("_tests$").search] 

2215 ) 

2216 

2217 # --------------------------------------------------------------------- 

2218 # Add tweens (inner to outer) 

2219 # --------------------------------------------------------------------- 

2220 # We will use implicit positioning: 

2221 # - https://www.slideshare.net/aconrad/alex-conrad-pyramid-tweens-ploneconf-2011 # noqa 

2222 

2223 # config.add_tween('camcops_server.camcops.http_session_tween_factory') 

2224 

2225 # --------------------------------------------------------------------- 

2226 # Debug toolbar 

2227 # --------------------------------------------------------------------- 

2228 if debug_toolbar: 

2229 log.debug("Enabling Pyramid debug toolbar") 

2230 config.include("pyramid_debugtoolbar") # BEWARE! SIDE EFFECTS 

2231 # ... Will trigger an import that hooks events into all 

2232 # SQLAlchemy queries. There's a bug somewhere relating to that; 

2233 # see notes below relating to the "mergedb" function. 

2234 config.add_route( 

2235 RouteCollection.DEBUG_TOOLBAR.route, 

2236 RouteCollection.DEBUG_TOOLBAR.path, 

2237 ) 

2238 

2239 yield config 

2240 

2241 

2242# ============================================================================= 

2243# Debugging requests 

2244# ============================================================================= 

2245 

2246 

2247def make_post_body_from_dict( 

2248 d: Dict[str, str], encoding: str = "utf8" 

2249) -> bytes: 

2250 """ 

2251 Makes an HTTP POST body from a dictionary. 

2252 

2253 For debugging HTTP requests. 

2254 

2255 It mimics how the tablet operates. 

2256 """ 

2257 # https://docs.pylonsproject.org/projects/pyramid-cookbook/en/latest/testing/testing_post_curl.html # noqa 

2258 txt = urllib.parse.urlencode(query=d) 

2259 # ... this encoding mimics how the tablet operates 

2260 body = txt.encode(encoding) 

2261 return body 

2262 

2263 

2264class CamcopsDummyRequest(CamcopsRequest, DummyRequest): 

2265 """ 

2266 Request class that allows manual manipulation of GET/POST parameters 

2267 for debugging. 

2268 

2269 It reads its config (on first demand) from the config file specified in 

2270 ``os.environ[ENVVAR_CONFIG_FILE]``. 

2271 

2272 Notes: 

2273 

2274 - The important base class is :class:`webob.request.BaseRequest`. 

2275 - ``self.params`` is a :class:`NestedMultiDict` (see 

2276 ``webob/multidict.py``); these are intrinsically read-only. 

2277 - ``self.params`` is also a read-only property. When read, it combines 

2278 data from ``self.GET`` and ``self.POST``. 

2279 - What we do here is to manipulate the underlying GET/POST data. 

2280 

2281 """ 

2282 

2283 _CACHE_KEY = "webob._parsed_query_vars" 

2284 _QUERY_STRING_KEY = "QUERY_STRING" 

2285 

2286 # def __init__(self, *args, **kwargs) -> None: 

2287 # super().__init__(*args, **kwargs) 

2288 # # Just a technique worth noting: 

2289 # # 

2290 # # self._original_params_property = CamcopsRequest.params # type: property # noqa 

2291 # # self._original_params = self._original_params_property.fget(self) # type: NestedMultiDict # noqa 

2292 # # self._fake_params = self._original_params.copy() # type: MultiDict 

2293 # # if params: 

2294 # # self._fake_params.update(params) 

2295 # 

2296 # @property 

2297 # def params(self): 

2298 # log.debug(repr(self._fake_params)) 

2299 # return self._fake_params 

2300 # # Returning the member object allows clients to call 

2301 # # dummyreq.params.update(...) 

2302 # 

2303 # @params.setter 

2304 # def params(self, value): 

2305 # self._fake_params = value 

2306 

2307 def set_method_get(self) -> None: 

2308 """ 

2309 Sets the fictional request method to GET. 

2310 """ 

2311 self.method = HttpMethod.GET 

2312 

2313 def set_method_post(self) -> None: 

2314 """ 

2315 Sets the fictional request method to POST. 

2316 """ 

2317 self.method = HttpMethod.POST 

2318 

2319 def clear_get_params(self) -> None: 

2320 """ 

2321 Clear all GET parameters. 

2322 """ 

2323 env = self.environ 

2324 if self._CACHE_KEY in env: 

2325 del env[self._CACHE_KEY] 

2326 env[self._QUERY_STRING_KEY] = "" 

2327 

2328 def add_get_params( 

2329 self, d: Dict[str, str], set_method_get: bool = True 

2330 ) -> None: 

2331 """ 

2332 Add GET parameters. 

2333 

2334 Args: 

2335 d: dictionary of ``{parameter: value}`` pairs. 

2336 set_method_get: also set the request's method to GET? 

2337 """ 

2338 if not d: 

2339 return 

2340 # webob.request.BaseRequest.GET reads from self.environ['QUERY_STRING'] 

2341 paramdict = self.GET.copy() # type: MultiDict 

2342 paramdict.update(d) 

2343 env = self.environ 

2344 # Delete the cached version. 

2345 if self._CACHE_KEY in env: 

2346 del env[self._CACHE_KEY] 

2347 # Write the new version 

2348 env[self._QUERY_STRING_KEY] = urllib.parse.urlencode(query=paramdict) 

2349 if set_method_get: 

2350 self.set_method_get() 

2351 

2352 def set_get_params( 

2353 self, d: Dict[str, str], set_method_get: bool = True 

2354 ) -> None: 

2355 """ 

2356 Clear any GET parameters, and then set them to new values. 

2357 See :func:`add_get_params`. 

2358 """ 

2359 self.clear_get_params() 

2360 self.add_get_params(d, set_method_get=set_method_get) 

2361 

2362 def set_post_body(self, body: bytes, set_method_post: bool = True) -> None: 

2363 """ 

2364 Sets the fake POST body. 

2365 

2366 Args: 

2367 body: the body to set 

2368 set_method_post: also set the request's method to POST? 

2369 """ 

2370 log.debug("Applying fake POST body: {!r}", body) 

2371 self.body = body 

2372 self.content_length = len(body) 

2373 if set_method_post: 

2374 self.set_method_post() 

2375 

2376 def fake_request_post_from_dict( 

2377 self, 

2378 d: Dict[str, str], 

2379 encoding: str = "utf8", 

2380 set_method_post: bool = True, 

2381 ) -> None: 

2382 """ 

2383 Sets the request's POST body according to a dictionary. 

2384 

2385 Args: 

2386 d: dictionary of ``{parameter: value}`` pairs. 

2387 encoding: character encoding to use 

2388 set_method_post: also set the request's method to POST? 

2389 """ 

2390 # webob.request.BaseRequest.POST reads from 'body' (indirectly). 

2391 body = make_post_body_from_dict(d, encoding=encoding) 

2392 self.set_post_body(body, set_method_post=set_method_post) 

2393 

2394 

2395_ = """ 

2396# A demonstration of the manipulation of superclass properties: 

2397 

2398class Test(object): 

2399 def __init__(self): 

2400 self.a = 3 

2401 

2402 @property 

2403 def b(self): 

2404 return 4 

2405 

2406 

2407class Derived(Test): 

2408 def __init__(self): 

2409 super().__init__() 

2410 self._superclass_b = super().b 

2411 self._b = 4 

2412 

2413 @property 

2414 def b(self): 

2415 print("Superclass b: {}".format(self._superclass_b.fget(self))) 

2416 print("Self _b: {}".format(self._b)) 

2417 return self._b 

2418 @b.setter 

2419 def b(self, value): 

2420 self._b = value 

2421 

2422 

2423x = Test() 

2424x.a # 3 

2425x.a = 5 

2426x.a # 5 

2427x.b # 4 

2428x.b = 6 # can't set attribute 

2429 

2430y = Derived() 

2431y.a # 3 

2432y.a = 5 

2433y.a # 5 

2434y.b # 4 

2435y.b = 6 

2436y.b # 6 

2437 

2438""" 

2439 

2440 

2441def get_core_debugging_request() -> CamcopsDummyRequest: 

2442 """ 

2443 Returns a basic :class:`CamcopsDummyRequest`. 

2444 

2445 It reads its config (on first demand) from the config file specified in 

2446 ``os.environ[ENVVAR_CONFIG_FILE]``. 

2447 """ 

2448 with camcops_pyramid_configurator_context(debug_toolbar=False) as pyr_cfg: 

2449 req = CamcopsDummyRequest( 

2450 environ={ 

2451 # In URL sequence: 

2452 WsgiEnvVar.WSGI_URL_SCHEME: "http", 

2453 WsgiEnvVar.SERVER_NAME: "127.0.0.1", 

2454 WsgiEnvVar.SERVER_PORT: "8000", 

2455 WsgiEnvVar.SCRIPT_NAME: "", 

2456 WsgiEnvVar.PATH_INFO: "/", 

2457 } # environ parameter: goes to pyramid.testing.DummyRequest.__init__ # noqa 

2458 ) 

2459 # ... must pass an actual dict to the "environ" parameter; os.environ 

2460 # itself isn't OK ("TypeError: WSGI environ must be a dict; you passed 

2461 # environ({'key1': 'value1', ...}) 

2462 

2463 # Being a CamcopsRequest, this object will read a config file from 

2464 # os.environ[ENVVAR_CONFIG_FILE] -- not the environ dictionary above -- 

2465 # when needed. That means we can now rewrite some of these URL 

2466 # components to give a valid external URL, if the config has the right 

2467 # information. 

2468 cfg = req.config 

2469 req.environ[WsgiEnvVar.WSGI_URL_SCHEME] = cfg.external_url_scheme 

2470 req.environ[WsgiEnvVar.SERVER_NAME] = cfg.external_server_name 

2471 req.environ[WsgiEnvVar.SERVER_PORT] = cfg.external_server_port 

2472 req.environ[WsgiEnvVar.SCRIPT_NAME] = cfg.external_script_name 

2473 # PATH_INFO remains "/" 

2474 

2475 req.registry = pyr_cfg.registry 

2476 pyr_cfg.begin(request=req) 

2477 return req 

2478 

2479 

2480def get_command_line_request(user_id: int = None) -> CamcopsRequest: 

2481 """ 

2482 Creates a dummy CamcopsRequest for use on the command line. 

2483 By default, it does so for the system user. Optionally, you can specify a 

2484 user by their ID number. 

2485 

2486 - Presupposes that ``os.environ[ENVVAR_CONFIG_FILE]`` has been set, as it 

2487 is in :func:`camcops_server.camcops.main`. 

2488 

2489 **WARNING:** this does not provide a COMMIT/ROLLBACK context. If you use 

2490 this directly, you must manage that yourself. Consider using 

2491 :func:`command_line_request_context` instead. 

2492 """ 

2493 log.debug(f"Creating command-line pseudo-request (user_id={user_id})") 

2494 req = get_core_debugging_request() 

2495 

2496 # If we proceed with an out-of-date database, we will have problems, and 

2497 # those problems may not be immediately apparent, which is bad. So: 

2498 req.config.assert_database_ok() 

2499 

2500 # Ensure we have a user 

2501 if user_id is None: 

2502 req._debugging_user = User.get_system_user(req.dbsession) 

2503 else: 

2504 req._debugging_user = User.get_user_by_id(req.dbsession, user_id) 

2505 

2506 log.debug( 

2507 "Command-line request: external URL is {}", req.route_url(Routes.HOME) 

2508 ) 

2509 return req 

2510 

2511 

2512@contextmanager 

2513def command_line_request_context( 

2514 user_id: int = None, 

2515) -> Generator[CamcopsRequest, None, None]: 

2516 """ 

2517 Request objects are ubiquitous, and allow code to refer to the HTTP 

2518 request, config, HTTP session, database session, and so on. Here we make 

2519 a special sort of request for use from the command line, and provide it 

2520 as a context manager that will COMMIT the database afterwards (because the 

2521 normal method, via the Pyramid router, is unavailable). 

2522 """ 

2523 req = get_command_line_request(user_id=user_id) 

2524 yield req 

2525 # noinspection PyProtectedMember 

2526 req._finish_dbsession() 

2527 

2528 

2529def get_unittest_request( 

2530 dbsession: SqlASession, params: Dict[str, Any] = None 

2531) -> CamcopsDummyRequest: 

2532 """ 

2533 Creates a :class:`CamcopsDummyRequest` for use by unit tests. 

2534 

2535 - Points to an existing database (e.g. SQLite in-memory database). 

2536 - Presupposes that ``os.environ[ENVVAR_CONFIG_FILE]`` has been set, as it 

2537 is in :func:`camcops_server.camcops.main`. 

2538 """ 

2539 log.debug("Creating unit testing pseudo-request") 

2540 req = get_core_debugging_request() 

2541 req.set_get_params(params) 

2542 

2543 req._debugging_db_session = dbsession 

2544 user = User() 

2545 user.superuser = True 

2546 req._debugging_user = user 

2547 

2548 return req