Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2# cardinal_pythonlib/colander_utils.py 

3 

4""" 

5=============================================================================== 

6 

7 Original code copyright (C) 2009-2021 Rudolf Cardinal (rudolf@pobox.com). 

8 

9 This file is part of cardinal_pythonlib. 

10 

11 Licensed under the Apache License, Version 2.0 (the "License"); 

12 you may not use this file except in compliance with the License. 

13 You may obtain a copy of the License at 

14 

15 https://www.apache.org/licenses/LICENSE-2.0 

16 

17 Unless required by applicable law or agreed to in writing, software 

18 distributed under the License is distributed on an "AS IS" BASIS, 

19 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

20 See the License for the specific language governing permissions and 

21 limitations under the License. 

22 

23=============================================================================== 

24 

25**Functions for working with colander.** 

26 

27Colander: https://docs.pylonsproject.org/projects/colander/en/latest/ 

28 

29""" 

30 

31import random 

32from typing import (Any, Callable, Dict, Iterable, List, Optional, 

33 Tuple, TYPE_CHECKING, Union) 

34 

35from cardinal_pythonlib.datetimefunc import ( 

36 coerce_to_pendulum, 

37 PotentialDatetimeType, 

38) 

39from cardinal_pythonlib.logs import get_brace_style_log_with_null_handler 

40# noinspection PyUnresolvedReferences 

41import colander 

42# noinspection PyUnresolvedReferences 

43from colander import ( 

44 Boolean, 

45 Date, 

46 DateTime, # NB name clash with pendulum 

47 Email, 

48 Integer, 

49 Invalid, 

50 Length, 

51 MappingSchema, 

52 SchemaNode, 

53 SchemaType, 

54 String, 

55) 

56from deform.widget import ( 

57 CheckboxWidget, 

58 DateTimeInputWidget, 

59 HiddenWidget, 

60) 

61from pendulum import DateTime as Pendulum # NB name clash with colander 

62from pendulum.parsing.exceptions import ParserError 

63 

64if TYPE_CHECKING: 

65 # noinspection PyProtectedMember,PyUnresolvedReferences 

66 from colander import _SchemaNode 

67 

68log = get_brace_style_log_with_null_handler(__name__) 

69 

70ColanderNullType = type(colander.null) 

71ValidatorType = Callable[[SchemaNode, Any], None] # called as v(node, value) 

72 

73# ============================================================================= 

74# Debugging options 

75# ============================================================================= 

76 

77DEBUG_DANGER_VALIDATION = False 

78 

79if DEBUG_DANGER_VALIDATION: 

80 log.warning("Debugging options enabled!") 

81 

82# ============================================================================= 

83# Constants 

84# ============================================================================= 

85 

86EMAIL_ADDRESS_MAX_LEN = 255 # https://en.wikipedia.org/wiki/Email_address 

87SERIALIZED_NONE = "" # has to be a string; avoid "None" like the plague! 

88 

89 

90# ============================================================================= 

91# New generic SchemaType classes 

92# ============================================================================= 

93 

94class PendulumType(SchemaType): 

95 """ 

96 Colander :class:`SchemaType` for :class:`Pendulum` date/time objects. 

97 """ 

98 def __init__(self, use_local_tz: bool = True): 

99 self.use_local_tz = use_local_tz 

100 super().__init__() # not necessary; SchemaType has no __init__ 

101 

102 def serialize(self, 

103 node: SchemaNode, 

104 appstruct: Union[PotentialDatetimeType, 

105 ColanderNullType]) \ 

106 -> Union[str, ColanderNullType]: 

107 """ 

108 Serializes Python object to string representation. 

109 """ 

110 if not appstruct: 

111 return colander.null 

112 try: 

113 appstruct = coerce_to_pendulum(appstruct, 

114 assume_local=self.use_local_tz) 

115 except (ValueError, ParserError) as e: 

116 raise Invalid( 

117 node, 

118 f"{appstruct!r} is not a pendulum.DateTime object; " 

119 f"error was {e!r}") 

120 return appstruct.isoformat() 

121 

122 def deserialize(self, 

123 node: SchemaNode, 

124 cstruct: Union[str, ColanderNullType]) \ 

125 -> Optional[Pendulum]: 

126 """ 

127 Deserializes string representation to Python object. 

128 """ 

129 if not cstruct: 

130 return colander.null 

131 try: 

132 result = coerce_to_pendulum(cstruct, 

133 assume_local=self.use_local_tz) 

134 except (ValueError, ParserError) as e: 

135 raise Invalid(node, 

136 f"Invalid date/time: value={cstruct!r}, error={e!r}") 

137 return result 

138 

139 

140class AllowNoneType(SchemaType): 

141 """ 

142 Serializes ``None`` to ``''``, and deserializes ``''`` to ``None``; 

143 otherwise defers to the parent type. 

144 

145 A type which accepts serializing ``None`` to ``''`` and deserializing 

146 ``''`` to ``None``. When the value is not equal to ``None``/``''``, it will 

147 use (de)serialization of the given type. This can be used to make nodes 

148 optional. 

149 

150 Example: 

151 

152 .. code-block:: python 

153 

154 date = colander.SchemaNode( 

155 colander.NoneType(colander.DateTime()), 

156 default=None, 

157 missing=None, 

158 ) 

159 

160 NOTE ALSO that Colander nodes explicitly never validate a missing value; 

161 see ``colander/__init__.py``, in :func:`_SchemaNode.deserialize`. We want 

162 them to do so, essentially so we can pass in ``None`` to a form but have 

163 the form refuse to validate if it's still ``None`` at submission. 

164 

165 """ 

166 def __init__(self, type_: SchemaType) -> None: 

167 self.type_ = type_ 

168 

169 def serialize(self, node: SchemaNode, 

170 value: Any) -> Union[str, ColanderNullType]: 

171 """ 

172 Serializes Python object to string representation. 

173 """ 

174 if value is None: 

175 retval = '' 

176 else: 

177 # noinspection PyUnresolvedReferences 

178 retval = self.type_.serialize(node, value) 

179 # log.debug("AllowNoneType.serialize: {!r} -> {!r}", value, retval) 

180 return retval 

181 

182 def deserialize(self, node: SchemaNode, 

183 value: Union[str, ColanderNullType]) -> Any: 

184 """ 

185 Deserializes string representation to Python object. 

186 """ 

187 if value is None or value == '': 

188 retval = None 

189 else: 

190 # noinspection PyUnresolvedReferences 

191 retval = self.type_.deserialize(node, value) 

192 # log.debug("AllowNoneType.deserialize: {!r} -> {!r}", value, retval) 

193 return retval 

194 

195 

196# ============================================================================= 

197# Node helper functions 

198# ============================================================================= 

199 

200def get_values_and_permissible(values: Iterable[Tuple[Any, str]], 

201 add_none: bool = False, 

202 none_description: str = "[None]") \ 

203 -> Tuple[List[Tuple[Any, str]], List[Any]]: 

204 """ 

205 Used when building Colander nodes. 

206 

207 Args: 

208 values: an iterable of tuples like ``(value, description)`` used in 

209 HTML forms 

210 

211 add_none: add a tuple ``(None, none_description)`` at the start of 

212 ``values`` in the result? 

213 

214 none_description: the description used for ``None`` if ``add_none`` 

215 is set 

216 

217 Returns: 

218 a tuple ``(values, permissible_values)``, where 

219 

220 - ``values`` is what was passed in (perhaps with the addition of the 

221 "None" tuple at the start) 

222 - ``permissible_values`` is a list of all the ``value`` elements of 

223 the original ``values`` 

224 

225 """ 

226 permissible_values = list(x[0] for x in values) 

227 # ... does not include the None value; those do not go to the validator 

228 if add_none: 

229 none_tuple = (SERIALIZED_NONE, none_description) 

230 values = [none_tuple] + list(values) 

231 return values, permissible_values 

232 

233 

234def get_child_node(parent: "_SchemaNode", child_name: str) -> "_SchemaNode": 

235 """ 

236 Returns a child node from an instantiated :class:`colander.SchemaNode` 

237 object. Such nodes are not accessible via ``self.mychild`` but must be 

238 accessed via ``self.children``, which is a list of child nodes. 

239 

240 Args: 

241 parent: the parent node object 

242 child_name: the name of the child node 

243 

244 Returns: 

245 the child node 

246 

247 Raises: 

248 :exc:`StopIteration` if there isn't one 

249 """ 

250 return next(c for c in parent.children if c.name == child_name) 

251 

252 

253# ============================================================================= 

254# Validators 

255# ============================================================================= 

256 

257class EmailValidatorWithLengthConstraint(Email): 

258 """ 

259 The Colander ``Email`` validator doesn't check length. This does. 

260 """ 

261 def __init__(self, *args, min_length: int = 0, **kwargs) -> None: 

262 self._length = Length(min_length, EMAIL_ADDRESS_MAX_LEN) 

263 super().__init__(*args, **kwargs) 

264 

265 def __call__(self, node: SchemaNode, value: Any) -> None: 

266 self._length(node, value) 

267 super().__call__(node, value) # call Email regex validator 

268 

269 

270# ============================================================================= 

271# Other new generic SchemaNode classes 

272# ============================================================================= 

273# Note that we must pass both *args and **kwargs upwards, because SchemaNode 

274# does some odd stuff with clone(). 

275 

276# ----------------------------------------------------------------------------- 

277# Simple types 

278# ----------------------------------------------------------------------------- 

279 

280class OptionalIntNode(SchemaNode): 

281 """ 

282 Colander node accepting integers but also blank values (i.e. it's 

283 optional). 

284 """ 

285 # YOU CANNOT USE ARGUMENTS THAT INFLUENCE THE STRUCTURE, because these Node 

286 # objects get default-copied by Deform. 

287 @staticmethod 

288 def schema_type() -> SchemaType: 

289 return AllowNoneType(Integer()) 

290 

291 default = None 

292 missing = None 

293 

294 

295class OptionalStringNode(SchemaNode): 

296 """ 

297 Colander node accepting strings but allowing them to be blank (optional). 

298 

299 Coerces None to ``""`` when serializing; otherwise it is coerced to 

300 ``"None"``, i.e. a string literal containing the word "None", which is much 

301 more wrong. 

302 """ 

303 @staticmethod 

304 def schema_type() -> SchemaType: 

305 return AllowNoneType(String(allow_empty=True)) 

306 

307 default = "" 

308 missing = "" 

309 

310 

311class MandatoryStringNode(SchemaNode): 

312 """ 

313 Colander string node, where the string is obligatory. 

314 

315 CAVEAT: WHEN YOU PASS DATA INTO THE FORM, YOU MUST USE 

316 

317 .. code-block:: python 

318 

319 appstruct = { 

320 somekey: somevalue or "", 

321 # ^^^^^ 

322 # without this, None is converted to "None" 

323 } 

324 """ 

325 @staticmethod 

326 def schema_type() -> SchemaType: 

327 return String(allow_empty=False) 

328 

329 

330class HiddenIntegerNode(OptionalIntNode): 

331 """ 

332 Colander node containing an integer, that is hidden to the user. 

333 """ 

334 widget = HiddenWidget() 

335 

336 

337class HiddenStringNode(OptionalStringNode): 

338 """ 

339 Colander node containing an optional string, that is hidden to the user. 

340 """ 

341 widget = HiddenWidget() 

342 

343 

344class BooleanNode(SchemaNode): 

345 """ 

346 Colander node representing a boolean value with a checkbox widget. 

347 """ 

348 schema_type = Boolean 

349 widget = CheckboxWidget() 

350 

351 def __init__(self, *args, title: str = "?", label: str = "", 

352 default: bool = False, **kwargs) -> None: 

353 self.title = title # above the checkbox 

354 self.label = label or title # to the right of the checkbox 

355 self.default = default 

356 self.missing = default 

357 super().__init__(*args, **kwargs) 

358 

359 

360# ----------------------------------------------------------------------------- 

361# Email addresses 

362# ----------------------------------------------------------------------------- 

363 

364class OptionalEmailNode(OptionalStringNode): 

365 """ 

366 Colander string node, where the string can be blank but if not then it 

367 must look like a valid e-mail address. 

368 """ 

369 validator = EmailValidatorWithLengthConstraint() 

370 

371 

372class MandatoryEmailNode(MandatoryStringNode): 

373 """ 

374 Colander string node, requiring something that looks like a valid e-mail 

375 address. 

376 """ 

377 validator = EmailValidatorWithLengthConstraint() 

378 

379 

380# ----------------------------------------------------------------------------- 

381# Date/time types 

382# ----------------------------------------------------------------------------- 

383 

384class DateTimeSelectorNode(SchemaNode): 

385 """ 

386 Colander node containing a date/time. 

387 """ 

388 schema_type = DateTime 

389 missing = None 

390 

391 

392class DateSelectorNode(SchemaNode): 

393 """ 

394 Colander node containing a date. 

395 """ 

396 schema_type = Date 

397 missing = None 

398 

399 

400DEFAULT_WIDGET_DATE_OPTIONS_FOR_PENDULUM = dict( 

401 # http://amsul.ca/pickadate.js/date/#formatting-rules 

402 format='yyyy-mm-dd', 

403 selectMonths=True, 

404 selectYears=True, 

405) 

406DEFAULT_WIDGET_TIME_OPTIONS_FOR_PENDULUM = dict( 

407 # See http://amsul.ca/pickadate.js/time/#formatting-rules 

408 # format='h:i A', # the default, e.g. "11:30 PM" 

409 format='HH:i', # e.g. "23:30" 

410 interval=30, 

411) 

412 

413 

414class OptionalPendulumNodeLocalTZ(SchemaNode): 

415 """ 

416 Colander node containing an optional :class:`Pendulum` date/time, in which 

417 the date/time is assumed to be in the local timezone. 

418 """ 

419 @staticmethod 

420 def schema_type() -> SchemaType: 

421 return AllowNoneType(PendulumType(use_local_tz=True)) 

422 

423 default = None 

424 missing = None 

425 widget = DateTimeInputWidget( 

426 date_options=DEFAULT_WIDGET_DATE_OPTIONS_FOR_PENDULUM, 

427 time_options=DEFAULT_WIDGET_TIME_OPTIONS_FOR_PENDULUM, 

428 ) 

429 

430 

431OptionalPendulumNode = OptionalPendulumNodeLocalTZ # synonym for back-compatibility # noqa 

432 

433 

434class OptionalPendulumNodeUTC(SchemaNode): 

435 """ 

436 Colander node containing an optional :class:`Pendulum` date/time, in which 

437 the date/time is assumed to be UTC. 

438 """ 

439 @staticmethod 

440 def schema_type() -> SchemaType: 

441 return AllowNoneType(PendulumType(use_local_tz=False)) 

442 

443 default = None 

444 missing = None 

445 widget = DateTimeInputWidget( 

446 date_options=DEFAULT_WIDGET_DATE_OPTIONS_FOR_PENDULUM, 

447 time_options=DEFAULT_WIDGET_TIME_OPTIONS_FOR_PENDULUM, 

448 ) 

449 

450 

451# ----------------------------------------------------------------------------- 

452# Safety-checking nodes 

453# ----------------------------------------------------------------------------- 

454 

455class ValidateDangerousOperationNode(MappingSchema): 

456 """ 

457 Colander node that can be added to forms allowing dangerous operations 

458 (e.g. deletion of data). The node shows the user a code and requires the 

459 user to type that code in, before it will permit the form to proceed. 

460 

461 For this to work, the containing form *must* inherit from 

462 :class:`DynamicDescriptionsForm` with ``dynamic_descriptions=True``. 

463 

464 Usage is simple, like this: 

465 

466 .. code-block:: python 

467 

468 class AddSpecialNoteSchema(CSRFSchema): 

469 table_name = HiddenStringNode() 

470 server_pk = HiddenIntegerNode() 

471 note = MandatoryStringNode(widget=TextAreaWidget(rows=20, cols=80)) 

472 danger = ValidateDangerousOperationNode() 

473 

474 """ 

475 target = HiddenStringNode() 

476 user_entry = MandatoryStringNode(title="Validate this dangerous operation") 

477 

478 def __init__(self, *args, length: int = 4, allowed_chars: str = None, 

479 **kwargs) -> None: 

480 """ 

481 Args: 

482 length: code length required from the user 

483 allowed_chars: string containing the permitted characters 

484 (by default, digits) 

485 """ 

486 self.allowed_chars = allowed_chars or "0123456789" 

487 self.length = length 

488 super().__init__(*args, **kwargs) 

489 

490 # noinspection PyUnusedLocal 

491 def after_bind(self, node: SchemaNode, kw: Dict[str, Any]) -> None: 

492 # Accessing the nodes is fiddly! 

493 target_node = get_child_node(self, "target") 

494 # Also, this whole thing is a bit hard to get your head around. 

495 # - This function will be called every time the form is accessed. 

496 # - The first time (fresh form load), there will be no value in 

497 # "target", so we set "target.default", and "target" will pick up 

498 # that default value. 

499 # - On subsequent times (e.g. form submission), there will be a value 

500 # in "target", so the default is irrelevant. 

501 # - This matters because we want "user_entry_node.description" to 

502 # be correct. 

503 # - Actually, easier is just to make "target" a static display? 

504 # No; if you use widget=TextInputWidget(readonly=True), there is no 

505 # form value rendered. 

506 # - But it's hard to get the new value out of "target" at this point. 

507 # - Should we do that in validate()? 

508 # - No: on second rendering, after_bind() is called, and then 

509 # validator() is called, but the visible form reflects changes made 

510 # by after_bind() but NOT validator(); presumably Deform pulls the 

511 # contents in between those two. Hmm. 

512 # - Particularly "hmm" as we don't have access to form data at the 

513 # point of after_bind(). 

514 # - The problem is probably that deform.field.Field.__init__ copies its 

515 # schema.description. Yes, that's the problem. 

516 # - So: a third option: a display value (which we won't get back) as 

517 # well as a hidden value that we will? No, no success. 

518 # - Or a fourth: something whose "description" is a property, not a 

519 # str? No -- when you copy a property, you copy the value not the 

520 # function. 

521 # - Fifthly: a new DangerValidationForm that rewrites its field 

522 # descriptions after validation. That works! 

523 target_value = ''.join(random.choice(self.allowed_chars) 

524 for i in range(self.length)) 

525 target_node.default = target_value 

526 # Set the description: 

527 if DEBUG_DANGER_VALIDATION: 

528 log.debug("after_bind: setting description to {!r}", target_value) 

529 self.set_description(target_value) 

530 # ... may be overridden immediately by validator() if this is NOT the 

531 # first rendering 

532 

533 def validator(self, node: SchemaNode, value: Any) -> None: 

534 user_entry_value = value['user_entry'] 

535 target_value = value['target'] 

536 # Set the description: 

537 if DEBUG_DANGER_VALIDATION: 

538 log.debug("validator: setting description to {!r}", target_value) 

539 self.set_description(target_value) 

540 # arse! 

541 value['display_target'] = target_value 

542 # Check the value 

543 if user_entry_value != target_value: 

544 raise Invalid( 

545 node, 

546 f"Not correctly validated " 

547 f"(user_entry_value={user_entry_value!r}, " 

548 f"target_value={target_value!r}") 

549 

550 def set_description(self, target_value: str) -> None: 

551 user_entry_node = get_child_node(self, "user_entry") 

552 prefix = "Please enter the following: " 

553 user_entry_node.description = prefix + target_value 

554 if DEBUG_DANGER_VALIDATION: 

555 log.debug("user_entry_node.description: {!r}", 

556 user_entry_node.description)