Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2 

3""" 

4camcops_server/cc_modules/cc_client_api_core.py 

5 

6=============================================================================== 

7 

8 Copyright (C) 2012-2020 Rudolf Cardinal (rudolf@pobox.com). 

9 

10 This file is part of CamCOPS. 

11 

12 CamCOPS is free software: you can redistribute it and/or modify 

13 it under the terms of the GNU General Public License as published by 

14 the Free Software Foundation, either version 3 of the License, or 

15 (at your option) any later version. 

16 

17 CamCOPS is distributed in the hope that it will be useful, 

18 but WITHOUT ANY WARRANTY; without even the implied warranty of 

19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

20 GNU General Public License for more details. 

21 

22 You should have received a copy of the GNU General Public License 

23 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

24 

25=============================================================================== 

26 

27**Core constants and functions used by the client (tablet device) API.** 

28 

29""" 

30 

31from typing import (Any, Dict, Iterable, List, NoReturn, 

32 Optional, Set, TYPE_CHECKING) 

33 

34from cardinal_pythonlib.datetimefunc import format_datetime 

35from cardinal_pythonlib.reprfunc import simple_repr 

36from pendulum import DateTime as Pendulum 

37from sqlalchemy.sql.expression import literal, select 

38from sqlalchemy.sql.schema import Table 

39 

40from camcops_server.cc_modules.cc_constants import ( 

41 CLIENT_DATE_FIELD, 

42 DateFormat, 

43 ERA_NOW, 

44 MOVE_OFF_TABLET_FIELD, 

45) 

46from camcops_server.cc_modules.cc_db import ( 

47 FN_ADDITION_PENDING, 

48 FN_CURRENT, 

49 FN_DEVICE_ID, 

50 FN_ERA, 

51 FN_FORCIBLY_PRESERVED, 

52 FN_PK, 

53 FN_PREDECESSOR_PK, 

54 FN_PRESERVING_USER_ID, 

55 FN_REMOVAL_PENDING, 

56 FN_REMOVING_USER_ID, 

57 FN_SUCCESSOR_PK, 

58 FN_WHEN_REMOVED_BATCH_UTC, 

59 FN_WHEN_REMOVED_EXACT, 

60) 

61 

62if TYPE_CHECKING: 

63 from camcops_server.cc_modules.cc_exportrecipient import ExportRecipient 

64 from camcops_server.cc_modules.cc_request import CamcopsRequest 

65 

66 

67# ============================================================================= 

68# Constants 

69# ============================================================================= 

70 

71class TabletParam(object): 

72 """ 

73 Keys used by server or client (in the comments: S server, C client, B 

74 bidirectional). 

75 """ 

76 ADDRESS = "address" # C->S, in JSON, v2.3.0 

77 ANONYMOUS = "anonymous" # S->C; new in v2.4.0 

78 CAMCOPS_VERSION = "camcops_version" # C->S 

79 COMPLETE = "complete" # S->C; new in v2.4.0 

80 DATABASE_TITLE = "databaseTitle" # S->C 

81 DATEVALUES = "datevalues" # C->S 

82 DBDATA = "dbdata" # C->S, v2.3.0 

83 DEVICE = "device" # C->S 

84 DEVICE_FRIENDLY_NAME = "devicefriendlyname" # C->S 

85 DOB = "dob" # C->S, in JSON, v2.3.0 

86 DUE_BY = "due_by" # C->S; new in v2.4.0 

87 DUE_FROM = "due_from" # C->S; new in v2.4.0 

88 EMAIL = "email" # C->S; new in v2.4.0 

89 ERROR = "error" # S->C 

90 FIELDS = "fields" # B 

91 FINALIZING = "finalizing" # C->S, in JSON and upload_entire_database, v2.3.0; synonym for preserving # noqa 

92 FORENAME = "forename" # C->S, in JSON, v2.3.0 

93 GP = "gp" # C->S, in JSON, v2.3.0 

94 ID_DESCRIPTION_PREFIX = "idDescription" # S->C 

95 ID_POLICY_FINALIZE = "idPolicyFinalize" # S->C 

96 ID_POLICY_UPLOAD = "idPolicyUpload" # S->C 

97 ID_SHORT_DESCRIPTION_PREFIX = "idShortDescription" # S->C 

98 ID_VALIDATION_METHOD_PREFIX = "idValidationMethod" # S->C; new in v2.2.8 

99 IDNUM_PREFIX = "idnum" # C->S, in JSON, v2.3.0 

100 IP_USE_INFO = "ip_use_info" # S->C; new in v2.4.0 

101 IP_USE_COMMERCIAL = "ip_use_commercial" # S->C; new in v2.4.0 

102 IP_USE_CLINICAL = "ip_use_clinical" # S->C; new in v2.4.0 

103 IP_USE_EDUCATIONAL = "ip_use_educational" # S->C; new in v2.4.0 

104 IP_USE_RESEARCH = "ip_use_research" # S->C; new in v2.4.0 

105 MOVE_OFF_TABLET_VALUES = "move_off_tablet_values" # C->S, v2.3.0 

106 NFIELDS = "nfields" # B 

107 NRECORDS = "nrecords" # B 

108 OPERATION = "operation" # C->S 

109 OTHER = "other" # C->S, in JSON, v2.3.0 

110 PASSWORD = "password" # C->S 

111 PATIENT_INFO = "patient_info" # C->S; new in v2.3.0, S->C new in v2.4.0 

112 PATIENT_PROQUINT = "patient_proquint" # C->S; new in v2.4.0 

113 PKNAME = "pkname" # C->S 

114 PKNAMEINFO = "pknameinfo" # C->S, new in v2.3.0 

115 PKVALUES = "pkvalues" # C->S 

116 RECORD_PREFIX = "record" # B 

117 RESULT = "result" # S->C 

118 SERVER_CAMCOPS_VERSION = "serverCamcopsVersion" # S->C 

119 SESSION_ID = "session_id" # B 

120 SESSION_TOKEN = "session_token" # B 

121 SETTINGS = "settings" # S->C; new in v2.4.0 

122 SEX = "sex" # C->S, in JSON, v2.3.0 

123 SUCCESS = "success" # S->C 

124 SURNAME = "surname" # C->S, in JSON, v2.3.0 

125 TABLE = "table" # C->S 

126 TABLES = "tables" # C->S 

127 TASK_SCHEDULES = "task_schedules" # S->C; new in v2.4.0 

128 TASK_SCHEDULE_ITEMS = "task_schedule_items" # S->C; new in v2.4.0 

129 TASK_SCHEDULE_NAME = "task_schedule_name" # S->C; new in v2.4.0 

130 USER = "user" # C->S 

131 VALUES = "values" # C->S 

132 WHEN_COMPLETED = "when_completed" # S->C; new in v2.4.0 

133 

134 # Retired (part of defunct mobileweb interface): 

135 # WHEREFIELDS = "wherefields" 

136 # WHERENOTFIELDS = "wherenotfields" 

137 # WHERENOTVALUES = "wherenotvalues" 

138 # WHEREVALUES = "wherevalues" 

139 

140 

141class ExtraStringFieldNames(object): 

142 """ 

143 To match ``extrastring.cpp`` on the tablet. 

144 """ 

145 TASK = "task" 

146 NAME = "name" 

147 LANGUAGE = "language" 

148 VALUE = "value" 

149 

150 

151class AllowedTablesFieldNames(object): 

152 """ 

153 To match ``allowedservertable.cpp`` on the tablet 

154 """ 

155 TABLENAME = "tablename" 

156 MIN_CLIENT_VERSION = "min_client_version" 

157 

158 

159# ============================================================================= 

160# Exceptions used by client API 

161# ============================================================================= 

162# Note the following about exception strings: 

163# 

164# class Blah(Exception): 

165# pass 

166# 

167# x = Blah("hello") 

168# str(x) # 'hello' 

169 

170class UserErrorException(Exception): 

171 """ 

172 Exception class for when the input from the tablet is dodgy. 

173 """ 

174 pass 

175 

176 

177class ServerErrorException(Exception): 

178 """ 

179 Exception class for when something's broken on the server side. 

180 """ 

181 pass 

182 

183 

184class IgnoringAntiqueTableException(Exception): 

185 """ 

186 Special exception to return success when we're ignoring an old tablet's 

187 request to upload the "storedvars" table. 

188 """ 

189 pass 

190 

191 

192# ============================================================================= 

193# Return message functions 

194# ============================================================================= 

195 

196def exception_description(e: Exception) -> str: 

197 """ 

198 Returns a formatted description of a Python exception. 

199 """ 

200 return f"{type(e).__name__}: {str(e)}" 

201 

202 

203# NO LONGER USED: 

204# def succeed_generic(operation: str) -> str: 

205# """ 

206# Generic success message to tablet. 

207# """ 

208# return "CamCOPS: {}".format(operation) 

209 

210 

211def fail_user_error(msg: str) -> NoReturn: 

212 """ 

213 Function to abort the script when the input is dodgy. 

214 

215 Raises :exc:`UserErrorException`. 

216 """ 

217 # While Titanium-Android can extract error messages from e.g. 

218 # finish("400 Bad Request: @_"), Titanium-iOS can't, and we need the error 

219 # messages. Therefore, we will return an HTTP success code but "Success: 0" 

220 # in the reply details. 

221 raise UserErrorException(msg) 

222 

223 

224def require_keys(dictionary: Dict[Any, Any], keys: List[Any]) -> None: 

225 """ 

226 Ensure that all listed keys are present in the specified dictionary, or 

227 raise a :exc:`UserErrorException`. 

228 """ 

229 for k in keys: 

230 if k not in dictionary: 

231 fail_user_error(f"Field {repr(k)} missing in client input") 

232 

233 

234def fail_user_error_from_exception(e: Exception) -> NoReturn: 

235 """ 

236 Raise :exc:`UserErrorException` with a description that comes from 

237 the specified exception. 

238 """ 

239 fail_user_error(exception_description(e)) 

240 

241 

242def fail_server_error(msg: str) -> NoReturn: 

243 """ 

244 Function to abort the script when something's broken server-side. 

245 

246 Raises :exc:`ServerErrorException`. 

247 """ 

248 raise ServerErrorException(msg) 

249 

250 

251def fail_server_error_from_exception(e: Exception) -> NoReturn: 

252 """ 

253 Raise :exc:`ServerErrorException` with a description that comes from 

254 the specified exception. 

255 """ 

256 fail_server_error(exception_description(e)) 

257 

258 

259def fail_unsupported_operation(operation: str) -> NoReturn: 

260 """ 

261 Abort the script (with a :exc:`UserErrorException`) when the 

262 operation is invalid. 

263 """ 

264 fail_user_error(f"operation={operation}: not supported") 

265 

266 

267# ============================================================================= 

268# Information classes used during upload 

269# ============================================================================= 

270 

271class BatchDetails(object): 

272 """ 

273 Represents a current upload batch. 

274 """ 

275 def __init__(self, 

276 batchtime: Optional[Pendulum] = None, 

277 preserving: bool = False, 

278 onestep: bool = False) -> None: 

279 """ 

280 Args: 

281 batchtime: 

282 the batchtime; UTC time this upload batch started; will be 

283 applied to all changes 

284 preserving: 

285 are we preserving (finalizing) the records -- that is, moving 

286 them from the current era (``NOW``) to the ``batchtime`` era, 

287 so they can be deleted from the tablet without apparent loss on 

288 the server? 

289 onestep: 

290 is this a one-step whole-database upload? 

291 """ 

292 self.batchtime = batchtime 

293 self.preserving = preserving 

294 self.onestep = onestep 

295 

296 def __repr__(self) -> str: 

297 return simple_repr(self, ["batchtime", "preserving", "onestep"]) 

298 

299 @property 

300 def new_era(self) -> str: 

301 """ 

302 Returns the string used for the new era for this batch, in case we 

303 are preserving records. 

304 """ 

305 return format_datetime(self.batchtime, DateFormat.ERA) 

306 

307 

308class WhichKeyToSendInfo(object): 

309 """ 

310 Represents information the client has sent, asking us which records it 

311 needs to upload recordwise. 

312 """ 

313 def __init__(self, 

314 client_pk: int, 

315 client_when: Pendulum, 

316 client_move_off_tablet: bool) -> None: 

317 self.client_pk = client_pk 

318 self.client_when = client_when 

319 self.client_move_off_tablet = client_move_off_tablet 

320 

321 

322class ServerRecord(object): 

323 """ 

324 Class to represent whether a server record exists, and/or the results of 

325 retrieving server records. 

326 """ 

327 def __init__(self, 

328 client_pk: int = None, 

329 exists_on_server: bool = False, 

330 server_pk: int = None, 

331 server_when: Pendulum = None, 

332 move_off_tablet: bool = False, 

333 current: bool = False, 

334 addition_pending: bool = False, 

335 removal_pending: bool = False, 

336 predecessor_pk: int = None, 

337 successor_pk: int = None) -> None: 

338 """ 

339 Args: 

340 client_pk: client's PK 

341 exists_on_server: does the record exist on the server? 

342 server_pk: if it exists, what's the server PK? 

343 server_when: if it exists, what's the server's "when" 

344 (``when_last_modified``) field? 

345 move_off_tablet: is the ``__move_off_tablet`` flag set? 

346 current: is the record current (``_current`` flag set)? 

347 addition_pending: is the ``_addition_pending`` flag set? 

348 removal_pending: is the ``_removal_pending`` flag set? 

349 predecessor_pk: predecessor server PK, or ``None`` 

350 successor_pk: successor server PK, or ``None`` 

351 """ 

352 self.client_pk = client_pk 

353 self.exists = exists_on_server 

354 self.server_pk = server_pk 

355 self.server_when = server_when 

356 self.move_off_tablet = move_off_tablet 

357 self.current = current 

358 self.addition_pending = addition_pending 

359 self.removal_pending = removal_pending 

360 self.predecessor_pk = predecessor_pk 

361 self.successor_pk = successor_pk 

362 

363 def __repr__(self) -> str: 

364 return simple_repr(self, [ 

365 "client_pk", "exists", "server_pk", "server_when", 

366 "move_off_tablet", "current", 

367 "addition_pending", "removal_pending", 

368 "predecessor_pk", "successor_pk", 

369 ]) 

370 

371 

372class UploadRecordResult(object): 

373 """ 

374 Represents the result of uploading a record. 

375 """ 

376 def __init__(self, 

377 oldserverpk: Optional[int] = None, 

378 newserverpk: Optional[int] = None, 

379 dirty: bool = False, 

380 specifically_marked_for_preservation: bool = False): 

381 """ 

382 Args: 

383 oldserverpk: 

384 the server's PK of the old version of the record; ``None`` if 

385 the record is new 

386 newserverpk: 

387 the server's PK of the new version of the record; ``None`` if 

388 the record was unmodified 

389 dirty: 

390 was the database table modified? (May be true even if 

391 ``newserverpk`` is ``None``, if ``_move_off_tablet`` was set. 

392 specifically_marked_for_preservation: 

393 should the record(s) be preserved? 

394 """ 

395 self.oldserverpk = oldserverpk 

396 self.newserverpk = newserverpk 

397 self.dirty = dirty 

398 self.specifically_marked_for_preservation = specifically_marked_for_preservation # noqa 

399 self._specifically_marked_preservation_pks = [] # type: List[int] 

400 

401 def __repr__(self) -> str: 

402 return simple_repr(self, [ 

403 "oldserverpk", "newserverpk", "dirty", 

404 "to_be_preserved", "specifically_marked_preservation_pks"]) 

405 

406 def note_specifically_marked_preservation_pks(self, 

407 pks: List[int]) -> None: 

408 """ 

409 Notes that some PKs are marked specifically for preservation. 

410 """ 

411 self._specifically_marked_preservation_pks.extend(pks) 

412 

413 @property 

414 def latest_pk(self) -> Optional[int]: 

415 """ 

416 Returns the latest of the two PKs. 

417 """ 

418 if self.newserverpk is not None: 

419 return self.newserverpk 

420 return self.oldserverpk 

421 

422 @property 

423 def specifically_marked_preservation_pks(self) -> List[int]: 

424 """ 

425 Returns a list of server PKs of records specifically marked to be 

426 preserved. This may include older versions (the predecessor chain) of 

427 records being uploaded. 

428 """ 

429 return self._specifically_marked_preservation_pks 

430 

431 @property 

432 def addition_pks(self) -> List[int]: 

433 """ 

434 Returns a list of PKs representing new records being added. 

435 """ 

436 return [self.newserverpk] if self.newserverpk is not None else [] 

437 

438 @property 

439 def removal_modified_pks(self) -> List[int]: 

440 """ 

441 Returns a list of PKs representing records removed because they have 

442 been "modified out". 

443 """ 

444 if self.oldserverpk is not None and self.newserverpk is not None: 

445 return [self.oldserverpk] 

446 return [] 

447 

448 @property 

449 def all_pks(self) -> List[int]: 

450 """ 

451 Returns all PKs (old, new, or both). 

452 """ 

453 return list(x for x in [self.oldserverpk, self.newserverpk] 

454 if x is not None) 

455 

456 @property 

457 def current_pks(self) -> List[int]: 

458 """ 

459 Returns PKs that represent current records on the server. 

460 """ 

461 if self.newserverpk is not None: 

462 return [self.newserverpk] # record was replaced; new one's current 

463 if self.oldserverpk is not None: 

464 return [self.oldserverpk] # not replaced; old one's current 

465 return [] 

466 

467 

468class UploadTableChanges(object): 

469 """ 

470 Represents information to process and audit an upload to a table. 

471 """ 

472 

473 def __init__(self, table: Table) -> None: 

474 self.table = table 

475 self._addition_pks = set() # type: Set[int] 

476 self._removal_modified_pks = set() # type: Set[int] 

477 self._removal_deleted_pks = set() # type: Set[int] 

478 self._preservation_pks = set() # type: Set[int] 

479 self._current_pks = set() # type: Set[int] 

480 

481 # ------------------------------------------------------------------------- 

482 # Basic info 

483 # ------------------------------------------------------------------------- 

484 

485 @property 

486 def tablename(self) -> str: 

487 """ 

488 The table's name. 

489 """ 

490 return self.table.name 

491 

492 # ------------------------------------------------------------------------- 

493 # Tell us about PKs 

494 # ------------------------------------------------------------------------- 

495 

496 def note_addition_pk(self, pk: int) -> None: 

497 """ 

498 Records an "addition" PK. 

499 """ 

500 self._addition_pks.add(pk) 

501 

502 def note_addition_pks(self, pks: Iterable[int]) -> None: 

503 """ 

504 Records multiple "addition" PKs. 

505 """ 

506 self._addition_pks.update(pks) 

507 

508 def note_removal_modified_pk(self, pk: int) -> None: 

509 """ 

510 Records a "removal because modified" PK (replaced by successor). 

511 """ 

512 self._removal_modified_pks.add(pk) 

513 

514 def note_removal_modified_pks(self, pks: Iterable[int]) -> None: 

515 """ 

516 Records multiple "removal because modified" PKs. 

517 """ 

518 self._removal_modified_pks.update(pks) 

519 

520 def note_removal_deleted_pk(self, pk: int) -> None: 

521 """ 

522 Records a "deleted" PK (removed with no successor). 

523 """ 

524 self._removal_deleted_pks.add(pk) 

525 

526 def note_removal_deleted_pks(self, pks: Iterable[int]) -> None: 

527 """ 

528 Records multiple "deleted" PKs (see :func:`note_removal_deleted_pk`). 

529 """ 

530 self._removal_deleted_pks.update(pks) 

531 

532 def note_preservation_pk(self, pk: int) -> None: 

533 """ 

534 Records a "preservation" PK (a record that's being finalized). 

535 """ 

536 self._preservation_pks.add(pk) 

537 

538 def note_preservation_pks(self, pks: Iterable[int]) -> None: 

539 """ 

540 Records multiple "preservation" PKs (records that are being finalized). 

541 """ 

542 self._preservation_pks.update(pks) 

543 

544 def note_current_pk(self, pk: int) -> None: 

545 """ 

546 Records that a record is current on the server. For indexing. 

547 """ 

548 self._current_pks.add(pk) 

549 

550 def note_current_pks(self, pks: Iterable[int]) -> None: 

551 """ 

552 Records multiple "current" PKs. 

553 """ 

554 self._current_pks.update(pks) 

555 

556 def note_urr(self, urr: UploadRecordResult, 

557 preserving_new_records: bool) -> None: 

558 """ 

559 Records information from a :class:`UploadRecordResult`, which is itself 

560 the result of calling 

561 :func:`camcops_server.cc_modules.client_api.upload_record_core`. 

562 

563 Called by 

564 :func:`camcops_server.cc_modules.client_api.process_table_for_onestep_upload`. 

565 

566 Args: 

567 urr: a :class:`UploadRecordResult` 

568 preserving_new_records: are new records being preserved? 

569 """ # noqa 

570 self.note_addition_pks(urr.addition_pks) 

571 self.note_removal_modified_pks(urr.removal_modified_pks) 

572 if preserving_new_records: 

573 self.note_preservation_pks(urr.addition_pks) 

574 self.note_preservation_pks(urr.specifically_marked_preservation_pks) 

575 self.note_current_pks(urr.current_pks) 

576 

577 def note_serverrec(self, sr: ServerRecord, 

578 preserving: bool) -> None: 

579 """ 

580 Records information from a :class:`ServerRecord`. Called by 

581 :func:`camcops_server.cc_modules.client_api.commit_table`. 

582 

583 Args: 

584 sr: a :class:`ServerRecord` 

585 preserving: are we preserving uploaded records? 

586 """ 

587 pk = sr.server_pk 

588 if sr.addition_pending: 

589 self.note_addition_pk(pk) 

590 self.note_current_pk(pk) 

591 elif sr.removal_pending: 

592 if sr.successor_pk is None: 

593 self.note_removal_deleted_pk(pk) 

594 else: 

595 self.note_removal_modified_pk(pk) 

596 elif sr.current: 

597 self.note_current_pk(pk) 

598 if preserving or sr.move_off_tablet: 

599 self.note_preservation_pk(pk) 

600 

601 # ------------------------------------------------------------------------- 

602 # Counts 

603 # ------------------------------------------------------------------------- 

604 

605 @property 

606 def n_added(self) -> int: 

607 """ 

608 Number of server records added. 

609 """ 

610 return len(self._addition_pks) 

611 

612 @property 

613 def n_removed_modified(self) -> int: 

614 """ 

615 Number of server records "modified out" -- replaced by a modified 

616 version and marked as removed. 

617 """ 

618 return len(self._removal_modified_pks) 

619 

620 @property 

621 def n_removed_deleted(self) -> int: 

622 """ 

623 Number of server records "deleted" -- marked as removed with no 

624 successor. 

625 """ 

626 return len(self._removal_deleted_pks) 

627 

628 @property 

629 def n_removed(self) -> int: 

630 """ 

631 Number of server records "removed" -- marked as removed (either with or 

632 without a successor). 

633 """ 

634 return self.n_removed_modified + self.n_removed_deleted 

635 

636 @property 

637 def n_preserved(self) -> int: 

638 """ 

639 Number of server records "preserved" (finalized) -- moved from the 

640 ``NOW`` era to the batch era (and no longer modifiable by the client 

641 device). 

642 """ 

643 return len(self._preservation_pks) 

644 

645 # ------------------------------------------------------------------------- 

646 # PKs for various purposes 

647 # ------------------------------------------------------------------------- 

648 

649 @property 

650 def addition_pks(self) -> List[int]: 

651 """ 

652 Server PKs of records being added. 

653 """ 

654 return sorted(self._addition_pks) 

655 

656 @property 

657 def removal_modified_pks(self) -> List[int]: 

658 """ 

659 Server PKs of records being modified out. 

660 """ 

661 return sorted(self._removal_modified_pks) 

662 

663 @property 

664 def removal_deleted_pks(self) -> List[int]: 

665 """ 

666 Server PKs of records being deleted. 

667 """ 

668 return sorted(self._removal_deleted_pks) 

669 

670 @property 

671 def removal_pks(self) -> List[int]: 

672 """ 

673 Server PKs of records being removed (modified out, or deleted). 

674 """ 

675 return sorted(self._removal_modified_pks | self._removal_deleted_pks) 

676 

677 @property 

678 def preservation_pks(self) -> List[int]: 

679 """ 

680 Server PKs of records being preserved. 

681 """ 

682 return sorted(self._preservation_pks) 

683 

684 @property 

685 def current_pks(self) -> List[int]: 

686 return sorted(self._current_pks) 

687 

688 @property 

689 def idnum_delete_index_pks(self) -> List[int]: 

690 """ 

691 Server PKs of records to delete old index entries for, if this is the 

692 ID number table. (Includes records that need re-indexing.) 

693 

694 We don't care about preservation PKs here, as the ID number index 

695 doesn't incorporate that. 

696 """ 

697 return sorted(self._removal_modified_pks | self._removal_deleted_pks) 

698 

699 @property 

700 def idnum_add_index_pks(self) -> List[int]: 

701 """ 

702 Server PKs of records to add index entries for, if this is the ID 

703 number table. 

704 """ 

705 return sorted(self._addition_pks) 

706 

707 @property 

708 def task_delete_index_pks(self) -> List[int]: 

709 """ 

710 Server PKs of records to delete old index entries for, if this is a 

711 task table. (Includes records that need re-indexing.) 

712 """ 

713 return sorted( 

714 (self._removal_modified_pks | # needs reindexing 

715 self._removal_deleted_pks | # gone 

716 self._preservation_pks) - # needs reindexing 

717 self._addition_pks # won't be indexed, so no need to delete index 

718 ) 

719 

720 @property 

721 def task_reindex_pks(self) -> List[int]: 

722 """ 

723 Server PKs of records to rebuild index entries for, if this is a task 

724 table. (Includes records that need re-indexing.) 

725 

726 We include records being preserved, because their era has changed, 

727 and the index includes era. Unless they are being removed! 

728 """ 

729 return sorted( 

730 ( 

731 (self._addition_pks | # new; index 

732 self._preservation_pks) - # reindex (but only if current) 

733 (self._removal_modified_pks | # modified out; don't index 

734 self._removal_deleted_pks) # deleted; don't index 

735 ) & self._current_pks # only reindex current PKs 

736 ) 

737 # A quick reminder, since I got this wrong: 

738 # | union (A or B) 

739 # & intersection (A and B) 

740 # ^ xor (A or B but not both) 

741 # - difference (A - B) 

742 

743 def get_task_push_export_pks(self, 

744 recipient: "ExportRecipient", 

745 uploading_group_id: int) -> List[int]: 

746 """ 

747 Returns PKs for tasks matching the requirements of a particular 

748 export recipient. 

749 

750 (In practice, only "push" recipients will come our way, so we can 

751 ignore this.) 

752 """ 

753 if not recipient.is_upload_suitable_for_push( 

754 tablename=self.tablename, 

755 uploading_group_id=uploading_group_id): 

756 # Not suitable 

757 return [] 

758 

759 if recipient.finalized_only: 

760 return sorted( 

761 self._preservation_pks # finalized 

762 & self._current_pks # only send current tasks 

763 ) 

764 else: 

765 return sorted( 

766 ( 

767 self._addition_pks | # new (may be unfinalized) 

768 self._preservation_pks # finalized 

769 ) & self._current_pks # only send current tasks 

770 ) 

771 

772 # ------------------------------------------------------------------------- 

773 # Audit info 

774 # ------------------------------------------------------------------------- 

775 

776 @property 

777 def any_changes(self) -> bool: 

778 """ 

779 Has anything changed that we're aware of? 

780 """ 

781 return (self.n_added > 0 or self.n_removed_modified > 0 or 

782 self.n_removed_deleted > 0 or self.n_preserved > 0) 

783 

784 def __str__(self) -> str: 

785 return ( 

786 f"{self.tablename}: " 

787 f"({self.n_added} added, " 

788 f"PKs {self.addition_pks}; " 

789 f"{self.n_removed_modified} modified out, " 

790 f"PKs {self.removal_modified_pks}; " 

791 f"{self.n_removed_deleted} deleted, " 

792 f"PKs {self.removal_deleted_pks}; " 

793 f"{self.n_preserved} preserved, " 

794 f"PKs {self.preservation_pks}; " 

795 f"current PKs {self.current_pks})" 

796 ) 

797 

798 def description(self, always_show_current_pks: bool = True) -> str: 

799 """ 

800 Short description, only including bits that have changed. 

801 """ 

802 parts = [] # type: List[str] 

803 if self._addition_pks: 

804 parts.append(f"{self.n_added} added, PKs {self.addition_pks}") 

805 if self._removal_modified_pks: 

806 parts.append( 

807 f"{self.n_removed_modified} modified out, " 

808 f"PKs {self.removal_modified_pks}") 

809 if self._removal_deleted_pks: 

810 parts.append( 

811 f"{self.n_removed_deleted} deleted, " 

812 f"PKs {self.removal_deleted_pks}") 

813 if self._preservation_pks: 

814 parts.append( 

815 f"{self.n_preserved} preserved, " 

816 f"PKs {self.preservation_pks}") 

817 if not parts: 

818 parts.append("no changes") 

819 if always_show_current_pks or self.any_changes: 

820 parts.append(f"current PKs {self.current_pks}") 

821 return f"{self.tablename} ({'; '.join(parts)})" 

822 

823 

824# ============================================================================= 

825# Value dictionaries for updating records, to reduce repetition 

826# ============================================================================= 

827 

828def values_delete_later() -> Dict[str, Any]: 

829 """ 

830 Field/value pairs to mark a record as "to be deleted later". 

831 """ 

832 return { 

833 FN_REMOVAL_PENDING: 1, 

834 FN_SUCCESSOR_PK: None 

835 } 

836 

837 

838def values_delete_now(req: "CamcopsRequest", 

839 batchdetails: BatchDetails) -> Dict[str, Any]: 

840 """ 

841 Field/value pairs to mark a record as deleted now. 

842 """ 

843 return { 

844 FN_CURRENT: 0, 

845 FN_REMOVAL_PENDING: 0, 

846 FN_REMOVING_USER_ID: req.user_id, 

847 FN_WHEN_REMOVED_EXACT: req.now, 

848 FN_WHEN_REMOVED_BATCH_UTC: batchdetails.batchtime 

849 } 

850 

851 

852def values_preserve_now(req: "CamcopsRequest", 

853 batchdetails: BatchDetails, 

854 forcibly_preserved: bool = False) -> Dict[str, Any]: 

855 """ 

856 Field/value pairs to mark a record as preserved now. 

857 """ 

858 return { 

859 FN_ERA: batchdetails.new_era, 

860 FN_PRESERVING_USER_ID: req.user_id, 

861 MOVE_OFF_TABLET_FIELD: 0, 

862 FN_FORCIBLY_PRESERVED: forcibly_preserved, 

863 } 

864 

865 

866# ============================================================================= 

867# CamCOPS table reading functions 

868# ============================================================================= 

869 

870def get_server_live_records(req: "CamcopsRequest", 

871 device_id: int, 

872 table: Table, 

873 clientpk_name: str = None, 

874 current_only: bool = True) -> List[ServerRecord]: 

875 """ 

876 Gets details of all records on the server, for the specified table, 

877 that are live on this client device. 

878 

879 Args: 

880 req: the :class:`camcops_server.cc_modules.cc_request.CamcopsRequest` 

881 device_id: ID of the 

882 :class:`camcops_server.cc_modules.cc_device.Device` 

883 table: an SQLAlchemy :class:`Table` 

884 clientpk_name: the column name of the client's PK; if none is supplied, 

885 the client_pk fields of the results will be ``None`` 

886 current_only: restrict to "current" (``_current``) records only? 

887 

888 Returns: 

889 :class:`ServerRecord` objects for active records (``_current`` and in 

890 the 'NOW' era) for the specified device/table. 

891 """ 

892 recs = [] # type: List[ServerRecord] 

893 client_pk_clause = table.c[clientpk_name] if clientpk_name else literal(None) # noqa 

894 query = ( 

895 select([ 

896 client_pk_clause, # 0: client PK (or None) 

897 table.c[FN_PK], # 1: server PK 

898 table.c[CLIENT_DATE_FIELD], # 2: when last modified (on the server) 

899 table.c[MOVE_OFF_TABLET_FIELD], # 3: move_off_tablet 

900 table.c[FN_CURRENT], # 4: current 

901 table.c[FN_ADDITION_PENDING], # 5 

902 table.c[FN_REMOVAL_PENDING], # 6 

903 table.c[FN_PREDECESSOR_PK], # 7 

904 table.c[FN_SUCCESSOR_PK], # 8 

905 ]) 

906 .where(table.c[FN_DEVICE_ID] == device_id) 

907 .where(table.c[FN_ERA] == ERA_NOW) 

908 ) 

909 if current_only: 

910 query = query.where(table.c[FN_CURRENT]) 

911 rows = req.dbsession.execute(query) 

912 for row in rows: 

913 recs.append(ServerRecord( 

914 client_pk=row[0], 

915 exists_on_server=True, 

916 server_pk=row[1], 

917 server_when=row[2], 

918 move_off_tablet=row[3], 

919 current=row[4], 

920 addition_pending=row[5], 

921 removal_pending=row[6], 

922 predecessor_pk=row[7], 

923 successor_pk=row[8], 

924 )) 

925 return recs