Coverage for src/configuraptor/core.py: 100%

156 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-19 17:38 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import math 

7import types 

8import typing 

9import warnings 

10from collections import ChainMap 

11from pathlib import Path 

12 

13from typeguard import TypeCheckError 

14from typeguard import check_type as _check_type 

15 

16from . import loaders 

17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey 

18from .helpers import camel_to_snake 

19from .postpone import Postponed 

20 

21# T is a reusable typevar 

22T = typing.TypeVar("T") 

23# t_typelike is anything that can be type hinted 

24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

25# t_data is anything that can be fed to _load_data 

26T_data = str | Path | dict[str, typing.Any] 

27# c = a config class instance, can be any (user-defined) class 

28C = typing.TypeVar("C") 

29# type c is a config class 

30Type_C = typing.Type[C] 

31 

32 

33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

34 """ 

35 If a key contains a dot, traverse the raw dict until the right key was found. 

36 

37 Example: 

38 key = some.nested.key 

39 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

40 -> {"with": "data"} 

41 """ 

42 parts = key.split(".") 

43 while parts: 

44 raw = raw[parts.pop(0)] 

45 

46 return raw 

47 

48 

49def _guess_key(clsname: str) -> str: 

50 """ 

51 If no key is manually defined for `load_into`, \ 

52 the class' name is converted to snake_case to use as the default key. 

53 """ 

54 return camel_to_snake(clsname) 

55 

56 

57def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

58 """ 

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

60 

61 E.g. class Tool will be mapped to key tool. 

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

63 """ 

64 if isinstance(data, str): 

65 data = Path(data) 

66 if isinstance(data, Path): 

67 with data.open("rb") as f: 

68 loader = loaders.get(data.suffix) 

69 data = loader(f) 

70 

71 if not data: 

72 return {} 

73 

74 if key is None: 

75 # try to guess key by grabbing the first one or using the class name 

76 if len(data) == 1: 

77 key = list(data.keys())[0] 

78 elif classname is not None: 

79 key = _guess_key(classname) 

80 

81 if key: 

82 return _data_for_nested_key(key, data) 

83 else: 

84 # no key found, just return all data 

85 return data 

86 

87 

88def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

89 """ 

90 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

91 

92 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

93 """ 

94 try: 

95 _check_type(value, expected_type) 

96 return True 

97 except TypeCheckError: 

98 return False 

99 

100 

101def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]: 

102 """ 

103 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

104 

105 If an annotated key in missing from data, it will be filled with None for convenience. 

106 

107 TODO: python 3.11 exception groups to throw multiple errors at once! 

108 """ 

109 # custom object to use instead of None, since typing.Optional can be None! 

110 # cast to T to make mypy happy 

111 notfound = typing.cast(T, object()) 

112 postponed = Postponed() 

113 

114 final: dict[str, T | None] = {} 

115 for key, _type in annotations.items(): 

116 compare = data.get(key, notfound) 

117 if compare is notfound: # pragma: nocover 

118 warnings.warn( 

119 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

120 ) 

121 # skip! 

122 continue 

123 

124 if compare is postponed: 

125 # don't do anything with this item! 

126 continue 

127 

128 if not check_type(compare, _type): 

129 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

130 

131 final[key] = compare 

132 

133 return final 

134 

135 

136def convert_config(items: dict[str, T]) -> dict[str, T]: 

137 """ 

138 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

139 

140 1. removes any items where the value is None, since in that case the default should be used; 

141 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

142 """ 

143 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

144 

145 

146Type = typing.Type[typing.Any] 

147T_Type = typing.TypeVar("T_Type", bound=Type) 

148 

149 

150def is_builtin_type(_type: Type) -> bool: 

151 """ 

152 Returns whether _type is one of the builtin types. 

153 """ 

154 return _type.__module__ in ("__builtin__", "builtins") 

155 

156 

157# def is_builtin_class_instance(obj: typing.Any) -> bool: 

158# return is_builtin_type(obj.__class__) 

159 

160 

161def is_from_types_or_typing(_type: Type) -> bool: 

162 """ 

163 Returns whether _type is one of the stlib typing/types types. 

164 

165 e.g. types.UnionType or typing.Union 

166 """ 

167 return _type.__module__ in ("types", "typing") 

168 

169 

170def is_from_other_toml_supported_module(_type: Type) -> bool: 

171 """ 

172 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

173 so this returns whether _type is a type from these stdlib modules. 

174 """ 

175 return _type.__module__ in ("datetime", "math") 

176 

177 

178def is_parameterized(_type: Type) -> bool: 

179 """ 

180 Returns whether _type is a parameterized type. 

181 

182 Examples: 

183 list[str] -> True 

184 str -> False 

185 """ 

186 return typing.get_origin(_type) is not None 

187 

188 

189def is_custom_class(_type: Type) -> bool: 

190 """ 

191 Tries to guess if _type is a builtin or a custom (user-defined) class. 

192 

193 Other logic in this module depends on knowing that. 

194 """ 

195 return ( 

196 type(_type) is type 

197 and not is_builtin_type(_type) 

198 and not is_from_other_toml_supported_module(_type) 

199 and not is_from_types_or_typing(_type) 

200 ) 

201 

202 

203def instance_of_custom_class(var: typing.Any) -> bool: 

204 """ 

205 Calls `is_custom_class` on an instance of a (possibly custom) class. 

206 """ 

207 return is_custom_class(var.__class__) 

208 

209 

210def is_optional(_type: Type | typing.Any) -> bool: 

211 """ 

212 Tries to guess if _type could be optional. 

213 

214 Examples: 

215 None -> True 

216 NoneType -> True 

217 typing.Union[str, None] -> True 

218 str | None -> True 

219 list[str | None] -> False 

220 list[str] -> False 

221 """ 

222 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan): 

223 # e.g. list[str] 

224 # will crash issubclass to test it first here 

225 return False 

226 

227 return ( 

228 _type is None 

229 or types.NoneType in typing.get_args(_type) # union with Nonetype 

230 or issubclass(types.NoneType, _type) 

231 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

232 ) 

233 

234 

235def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

236 """ 

237 Get Field info for a dataclass cls. 

238 """ 

239 fields = getattr(cls, "__dataclass_fields__", {}) 

240 return fields.get(key) 

241 

242 

243def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]: 

244 """ 

245 For all annotations (recursively gathered from parents with `all_annotations`), \ 

246 try to resolve the tree of annotations. 

247 

248 Uses `load_into_recurse`, not itself directly. 

249 

250 Example: 

251 class First: 

252 key: str 

253 

254 class Second: 

255 other: First 

256 

257 # step 1 

258 cls = Second 

259 data = {"second": {"other": {"key": "anything"}}} 

260 annotations: {"other": First} 

261 

262 # step 1.5 

263 data = {"other": {"key": "anything"} 

264 annotations: {"other": First} 

265 

266 # step 2 

267 cls = First 

268 data = {"key": "anything"} 

269 annotations: {"key": str} 

270 

271 

272 TODO: python 3.11 exception groups to throw multiple errors at once! 

273 """ 

274 updated = {} 

275 

276 for _key, _type in annotations.items(): 

277 if _key in data: 

278 value: typing.Any = data[_key] # value can change so define it as any instead of T 

279 if is_parameterized(_type): 

280 origin = typing.get_origin(_type) 

281 arguments = typing.get_args(_type) 

282 if origin is list and arguments and is_custom_class(arguments[0]): 

283 subtype = arguments[0] 

284 value = [_load_into_recurse(subtype, subvalue) for subvalue in value] 

285 

286 elif origin is dict and arguments and is_custom_class(arguments[1]): 

287 # e.g. dict[str, Point] 

288 subkeytype, subvaluetype = arguments 

289 # subkey(type) is not a custom class, so don't try to convert it: 

290 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()} 

291 # elif origin is dict: 

292 # keep data the same 

293 elif origin is typing.Union and arguments: 

294 for arg in arguments: 

295 if is_custom_class(arg): 

296 value = _load_into_recurse(arg, value) 

297 else: 

298 # print(_type, arg, value) 

299 ... 

300 

301 # todo: other parameterized/unions/typing.Optional 

302 

303 elif is_custom_class(_type): 

304 # type must be C (custom class) at this point 

305 value = _load_into_recurse( 

306 # make mypy and pycharm happy by telling it _type is of type C... 

307 # actually just passing _type as first arg! 

308 typing.cast(Type_C[typing.Any], _type), 

309 value, 

310 ) 

311 

312 elif _key in cls.__dict__: 

313 # property has default, use that instead. 

314 value = cls.__dict__[_key] 

315 elif is_optional(_type): 

316 # type is optional and not found in __dict__ -> default is None 

317 value = None 

318 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

319 # could have a default factory 

320 # todo: do something with field.default? 

321 value = field.default_factory() 

322 else: 

323 raise ConfigErrorMissingKey(_key, cls, _type) 

324 

325 updated[_key] = value 

326 

327 return updated 

328 

329 

330def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

331 """ 

332 Returns a dictionary-like ChainMap that includes annotations for all \ 

333 attributes defined in cls or inherited from superclasses. 

334 """ 

335 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

336 

337 

338def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]: 

339 """ 

340 Wrapper around `_all_annotations` that filters away any keys in _except. 

341 

342 It also flattens the ChainMap to a regular dict. 

343 """ 

344 if _except is None: 

345 _except = set() 

346 

347 _all = _all_annotations(cls) 

348 return {k: v for k, v in _all.items() if k not in _except} 

349 

350 

351def check_and_convert_data( 

352 cls: typing.Type[C], 

353 data: dict[str, typing.Any], 

354 _except: typing.Iterable[str], 

355 strict: bool = True, 

356) -> dict[str, typing.Any]: 

357 """ 

358 Based on class annotations, this prepares the data for `load_into_recurse`. 

359 

360 1. convert config-keys to python compatible config_keys 

361 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

362 3. ensures the annotated types match the actual types after loading the config file. 

363 """ 

364 annotations = all_annotations(cls, _except=_except) 

365 

366 to_load = convert_config(data) 

367 to_load = load_recursive(cls, to_load, annotations) 

368 if strict: 

369 to_load = ensure_types(to_load, annotations) 

370 

371 return to_load 

372 

373 

374def _load_into_recurse( 

375 cls: typing.Type[C], 

376 data: dict[str, typing.Any], 

377 init: dict[str, typing.Any] = None, 

378 strict: bool = True, 

379) -> C: 

380 """ 

381 Loads an instance of `cls` filled with `data`. 

382 

383 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

384 `init` can be used to optionally pass extra __init__ arguments. \ 

385 NOTE: This will overwrite a config key with the same name! 

386 """ 

387 if init is None: 

388 init = {} 

389 

390 # fixme: cls.__init__ can set other keys than the name is in kwargs!! 

391 

392 if dc.is_dataclass(cls): 

393 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict) 

394 to_load |= init # add extra init variables (should not happen for a dataclass but whatev) 

395 

396 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

397 inst = typing.cast(C, cls(**to_load)) 

398 else: 

399 inst = cls(**init) 

400 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict) 

401 inst.__dict__.update(**to_load) 

402 

403 return inst 

404 

405 

406def _load_into_instance( 

407 inst: C, 

408 cls: typing.Type[C], 

409 data: dict[str, typing.Any], 

410 init: dict[str, typing.Any] = None, 

411 strict: bool = True, 

412) -> C: 

413 """ 

414 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

415 and thus does not support init. 

416 

417 """ 

418 if init is not None: 

419 raise ValueError("Can not init an existing instance!") 

420 

421 existing_data = inst.__dict__ 

422 

423 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict) 

424 

425 inst.__dict__.update(**to_load) 

426 

427 return inst 

428 

429 

430def load_into_class( 

431 cls: typing.Type[C], 

432 data: T_data, 

433 /, 

434 key: str = None, 

435 init: dict[str, typing.Any] = None, 

436 strict: bool = True, 

437) -> C: 

438 """ 

439 Shortcut for _load_data + load_into_recurse. 

440 """ 

441 to_load = _load_data(data, key, cls.__name__) 

442 return _load_into_recurse(cls, to_load, init=init, strict=strict) 

443 

444 

445def load_into_instance( 

446 inst: C, 

447 data: T_data, 

448 /, 

449 key: str = None, 

450 init: dict[str, typing.Any] = None, 

451 strict: bool = True, 

452) -> C: 

453 """ 

454 Shortcut for _load_data + load_into_existing. 

455 """ 

456 cls = inst.__class__ 

457 to_load = _load_data(data, key, cls.__name__) 

458 return _load_into_instance(inst, cls, to_load, init=init, strict=strict) 

459 

460 

461def load_into( 

462 cls: typing.Type[C], 

463 data: T_data, 

464 /, 

465 key: str = None, 

466 init: dict[str, typing.Any] = None, 

467 strict: bool = True, 

468) -> C: 

469 """ 

470 Load your config into a class (instance). 

471 

472 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

473 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

474 

475 Args: 

476 cls: either a class or an existing instance of that class. 

477 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

478 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

479 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

480 strict: enable type checks or allow anything? 

481 

482 """ 

483 if not isinstance(cls, type): 

484 # would not be supported according to mypy, but you can still load_into(instance) 

485 return load_into_instance(cls, data, key=key, init=init, strict=strict) 

486 

487 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

488 # _cls = typing.cast(typing.Type[C], cls) 

489 return load_into_class(cls, data, key=key, init=init, strict=strict)