Coverage for src/configuraptor/core.py: 100%
212 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-03 15:36 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-03 15:36 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import math
7import types
8import typing
9import warnings
10from collections import ChainMap
11from pathlib import Path
13from typeguard import TypeCheckError
14from typeguard import check_type as _check_type
16from . import loaders
17from .errors import (
18 ConfigErrorCouldNotConvert,
19 ConfigErrorInvalidType,
20 ConfigErrorMissingKey,
21)
22from .helpers import camel_to_snake
23from .postpone import Postponed
25# T is a reusable typevar
26T = typing.TypeVar("T")
27# t_typelike is anything that can be type hinted
28T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union
29# t_data is anything that can be fed to _load_data
30T_data = str | Path | dict[str, typing.Any]
31# c = a config class instance, can be any (user-defined) class
32C = typing.TypeVar("C")
33# type c is a config class
34Type_C = typing.Type[C]
37def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
38 """
39 If a key contains a dot, traverse the raw dict until the right key was found.
41 Example:
42 key = some.nested.key
43 raw = {"some": {"nested": {"key": {"with": "data"}}}}
44 -> {"with": "data"}
45 """
46 parts = key.split(".")
47 while parts:
48 raw = raw[parts.pop(0)]
50 return raw
53def _guess_key(clsname: str) -> str:
54 """
55 If no key is manually defined for `load_into`, \
56 the class' name is converted to snake_case to use as the default key.
57 """
58 return camel_to_snake(clsname)
61def __load_data(
62 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False
63) -> dict[str, typing.Any]:
64 """
65 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
67 E.g. class Tool will be mapped to key tool.
68 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
69 """
70 if isinstance(data, str):
71 data = Path(data)
72 if isinstance(data, Path):
73 with data.open("rb") as f:
74 loader = loaders.get(data.suffix or data.name)
75 data = loader(f, data.resolve())
77 if not data:
78 return {}
80 if key is None:
81 # try to guess key by grabbing the first one or using the class name
82 if len(data) == 1:
83 key = list(data.keys())[0]
84 elif classname is not None:
85 key = _guess_key(classname)
87 if key:
88 data = _data_for_nested_key(key, data)
90 if not data:
91 raise ValueError("No data found!")
93 if not isinstance(data, dict):
94 raise ValueError("Data is not a dict!")
96 if lower_keys:
97 data = {k.lower(): v for k, v in data.items()}
99 return data
102def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]:
103 """
104 Wrapper around __load_data that retries with key="" if anything goes wrong.
105 """
106 try:
107 return __load_data(data, key, classname, lower_keys=lower_keys)
108 except Exception as e:
109 if key != "":
110 return __load_data(data, "", classname, lower_keys=lower_keys)
111 else: # pragma: no cover
112 warnings.warn(f"Data could not be loaded: {e}", source=e)
113 # key already was "", just return data!
114 # (will probably not happen but fallback)
115 return {}
118def check_type(value: typing.Any, expected_type: T_typelike) -> bool:
119 """
120 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).
122 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError
123 """
124 try:
125 _check_type(value, expected_type)
126 return True
127 except TypeCheckError:
128 return False
131F = typing.TypeVar("F")
134def str_to_bool(value: str) -> bool:
135 """
136 Used by convert_between, usually for .env loads.
138 Example:
139 SOME_VALUE=TRUE -> True
140 SOME_VALUE=1 -> True
141 SOME_VALUE=Yes -> True
143 SOME_VALUE -> None
144 SOME_VALUE=NOpe -> False
146 SOME_VALUE=Unrelated -> Error
147 """
148 if not value:
149 return False
151 first_letter = value[0].lower()
152 # yes, true, 1
153 if first_letter in {"y", "t", "1"}:
154 return True
155 elif first_letter in {"n", "f", "0"}:
156 return False
157 else:
158 raise ValueError("Not booly.")
161def str_to_none(value: str) -> typing.Optional[str]:
162 """
163 Convert a string value of null/none to None, or keep the original string otherwise.
164 """
165 if value.lower() in {"", "null", "none"}:
166 return None
167 else:
168 return value
171def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:
172 """
173 Convert a value between types.
174 """
175 if from_type is str:
176 if to_type is bool:
177 return str_to_bool(from_value) # type: ignore
178 elif to_type is None or to_type is types.NoneType: # noqa: E721
179 return str_to_none(from_value) # type: ignore
180 # default: just convert type:
181 return to_type(from_value) # type: ignore
184def ensure_types(
185 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False
186) -> dict[str, T | None]:
187 """
188 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
190 If an annotated key in missing from data, it will be filled with None for convenience.
192 TODO: python 3.11 exception groups to throw multiple errors at once!
193 """
194 # custom object to use instead of None, since typing.Optional can be None!
195 # cast to T to make mypy happy
196 notfound = typing.cast(T, object())
197 postponed = Postponed()
199 final: dict[str, T | None] = {}
200 for key, _type in annotations.items():
201 compare = data.get(key, notfound)
202 if compare is notfound: # pragma: nocover
203 warnings.warn(
204 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
205 )
206 # skip!
207 continue
209 if compare is postponed:
210 # don't do anything with this item!
211 continue
213 if not check_type(compare, _type):
214 if convert_types:
215 try:
216 compare = convert_between(compare, type(compare), _type)
217 except (TypeError, ValueError) as e:
218 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e
219 else:
220 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
222 final[key] = compare
224 return final
227def convert_config(items: dict[str, T]) -> dict[str, T]:
228 """
229 Converts the config dict (from toml) or 'overwrites' dict in two ways.
231 1. removes any items where the value is None, since in that case the default should be used;
232 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
233 """
234 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
237Type = typing.Type[typing.Any]
238T_Type = typing.TypeVar("T_Type", bound=Type)
241def is_builtin_type(_type: Type) -> bool:
242 """
243 Returns whether _type is one of the builtin types.
244 """
245 return _type.__module__ in ("__builtin__", "builtins")
248# def is_builtin_class_instance(obj: typing.Any) -> bool:
249# return is_builtin_type(obj.__class__)
252def is_from_types_or_typing(_type: Type) -> bool:
253 """
254 Returns whether _type is one of the stlib typing/types types.
256 e.g. types.UnionType or typing.Union
257 """
258 return _type.__module__ in ("types", "typing")
261def is_from_other_toml_supported_module(_type: Type) -> bool:
262 """
263 Besides builtins, toml also supports 'datetime' and 'math' types, \
264 so this returns whether _type is a type from these stdlib modules.
265 """
266 return _type.__module__ in ("datetime", "math")
269def is_parameterized(_type: Type) -> bool:
270 """
271 Returns whether _type is a parameterized type.
273 Examples:
274 list[str] -> True
275 str -> False
276 """
277 return typing.get_origin(_type) is not None
280def is_custom_class(_type: Type) -> bool:
281 """
282 Tries to guess if _type is a builtin or a custom (user-defined) class.
284 Other logic in this module depends on knowing that.
285 """
286 return (
287 type(_type) is type
288 and not is_builtin_type(_type)
289 and not is_from_other_toml_supported_module(_type)
290 and not is_from_types_or_typing(_type)
291 )
294def instance_of_custom_class(var: typing.Any) -> bool:
295 """
296 Calls `is_custom_class` on an instance of a (possibly custom) class.
297 """
298 return is_custom_class(var.__class__)
301def is_optional(_type: Type | typing.Any) -> bool:
302 """
303 Tries to guess if _type could be optional.
305 Examples:
306 None -> True
307 NoneType -> True
308 typing.Union[str, None] -> True
309 str | None -> True
310 list[str | None] -> False
311 list[str] -> False
312 """
313 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):
314 # e.g. list[str]
315 # will crash issubclass to test it first here
316 return False
318 return (
319 _type is None
320 or types.NoneType in typing.get_args(_type) # union with Nonetype
321 or issubclass(types.NoneType, _type)
322 or issubclass(types.NoneType, type(_type)) # no type # Nonetype
323 )
326def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:
327 """
328 Get Field info for a dataclass cls.
329 """
330 fields = getattr(cls, "__dataclass_fields__", {})
331 return fields.get(key)
334def load_recursive(
335 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False
336) -> dict[str, T]:
337 """
338 For all annotations (recursively gathered from parents with `all_annotations`), \
339 try to resolve the tree of annotations.
341 Uses `load_into_recurse`, not itself directly.
343 Example:
344 class First:
345 key: str
347 class Second:
348 other: First
350 # step 1
351 cls = Second
352 data = {"second": {"other": {"key": "anything"}}}
353 annotations: {"other": First}
355 # step 1.5
356 data = {"other": {"key": "anything"}
357 annotations: {"other": First}
359 # step 2
360 cls = First
361 data = {"key": "anything"}
362 annotations: {"key": str}
365 TODO: python 3.11 exception groups to throw multiple errors at once!
366 """
367 updated = {}
369 for _key, _type in annotations.items():
370 if _key in data:
371 value: typing.Any = data[_key] # value can change so define it as any instead of T
372 if is_parameterized(_type):
373 origin = typing.get_origin(_type)
374 arguments = typing.get_args(_type)
375 if origin is list and arguments and is_custom_class(arguments[0]):
376 subtype = arguments[0]
377 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]
379 elif origin is dict and arguments and is_custom_class(arguments[1]):
380 # e.g. dict[str, Point]
381 subkeytype, subvaluetype = arguments
382 # subkey(type) is not a custom class, so don't try to convert it:
383 value = {
384 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)
385 for subkey, subvalue in value.items()
386 }
387 # elif origin is dict:
388 # keep data the same
389 elif origin is typing.Union and arguments:
390 for arg in arguments:
391 if is_custom_class(arg):
392 value = _load_into_recurse(arg, value, convert_types=convert_types)
393 else:
394 # print(_type, arg, value)
395 ...
397 # todo: other parameterized/unions/typing.Optional
399 elif is_custom_class(_type):
400 # type must be C (custom class) at this point
401 value = _load_into_recurse(
402 # make mypy and pycharm happy by telling it _type is of type C...
403 # actually just passing _type as first arg!
404 typing.cast(Type_C[typing.Any], _type),
405 value,
406 convert_types=convert_types,
407 )
409 elif _key in cls.__dict__:
410 # property has default, use that instead.
411 value = cls.__dict__[_key]
412 elif is_optional(_type):
413 # type is optional and not found in __dict__ -> default is None
414 value = None
415 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
416 # could have a default factory
417 # todo: do something with field.default?
418 value = field.default_factory()
419 else:
420 raise ConfigErrorMissingKey(_key, cls, _type)
422 updated[_key] = value
424 return updated
427def _all_annotations(cls: Type) -> ChainMap[str, Type]:
428 """
429 Returns a dictionary-like ChainMap that includes annotations for all \
430 attributes defined in cls or inherited from superclasses.
431 """
432 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
435def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, type[object]]:
436 """
437 Wrapper around `_all_annotations` that filters away any keys in _except.
439 It also flattens the ChainMap to a regular dict.
440 """
441 if _except is None:
442 _except = set()
444 _all = _all_annotations(cls)
445 return {k: v for k, v in _all.items() if k not in _except}
448def check_and_convert_data(
449 cls: typing.Type[C],
450 data: dict[str, typing.Any],
451 _except: typing.Iterable[str],
452 strict: bool = True,
453 convert_types: bool = False,
454) -> dict[str, typing.Any]:
455 """
456 Based on class annotations, this prepares the data for `load_into_recurse`.
458 1. convert config-keys to python compatible config_keys
459 2. loads custom class type annotations with the same logic (see also `load_recursive`)
460 3. ensures the annotated types match the actual types after loading the config file.
461 """
462 annotations = all_annotations(cls, _except=_except)
464 to_load = convert_config(data)
465 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)
466 if strict:
467 to_load = ensure_types(to_load, annotations, convert_types=convert_types)
469 return to_load
472T_init_list = list[typing.Any]
473T_init_dict = dict[str, typing.Any]
474T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None
477@typing.no_type_check # (mypy doesn't understand 'match' fully yet)
478def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:
479 """
480 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.
481 """
482 if not init:
483 return [], {}
485 args: T_init_list = []
486 kwargs: T_init_dict = {}
487 match init:
488 case (args, kwargs):
489 return args, kwargs
490 case [*args]:
491 return args, {}
492 case {**kwargs}:
493 return [], kwargs
494 case _:
495 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")
498def _load_into_recurse(
499 cls: typing.Type[C],
500 data: dict[str, typing.Any],
501 init: T_init = None,
502 strict: bool = True,
503 convert_types: bool = False,
504) -> C:
505 """
506 Loads an instance of `cls` filled with `data`.
508 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
509 `init` can be used to optionally pass extra __init__ arguments. \
510 NOTE: This will overwrite a config key with the same name!
511 """
512 init_args, init_kwargs = _split_init(init)
514 if dc.is_dataclass(cls):
515 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)
516 if init:
517 raise ValueError("Init is not allowed for dataclasses!")
519 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
520 inst = typing.cast(C, cls(**to_load))
521 else:
522 inst = cls(*init_args, **init_kwargs)
523 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)
524 inst.__dict__.update(**to_load)
526 return inst
529def _load_into_instance(
530 inst: C,
531 cls: typing.Type[C],
532 data: dict[str, typing.Any],
533 init: T_init = None,
534 strict: bool = True,
535 convert_types: bool = False,
536) -> C:
537 """
538 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
539 and thus does not support init.
541 """
542 if init is not None:
543 raise ValueError("Can not init an existing instance!")
545 existing_data = inst.__dict__
547 to_load = check_and_convert_data(
548 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types
549 )
551 inst.__dict__.update(**to_load)
553 return inst
556def load_into_class(
557 cls: typing.Type[C],
558 data: T_data,
559 /,
560 key: str = None,
561 init: T_init = None,
562 strict: bool = True,
563 lower_keys: bool = False,
564 convert_types: bool = False,
565) -> C:
566 """
567 Shortcut for _load_data + load_into_recurse.
568 """
569 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)
570 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)
573def load_into_instance(
574 inst: C,
575 data: T_data,
576 /,
577 key: str = None,
578 init: T_init = None,
579 strict: bool = True,
580 lower_keys: bool = False,
581 convert_types: bool = False,
582) -> C:
583 """
584 Shortcut for _load_data + load_into_existing.
585 """
586 cls = inst.__class__
587 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)
588 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)
591def load_into(
592 cls: typing.Type[C],
593 data: T_data,
594 /,
595 key: str = None,
596 init: T_init = None,
597 strict: bool = True,
598 lower_keys: bool = False,
599 convert_types: bool = False,
600) -> C:
601 """
602 Load your config into a class (instance).
604 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
605 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
607 Args:
608 cls: either a class or an existing instance of that class.
609 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
610 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
611 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
612 strict: enable type checks or allow anything?
613 lower_keys: should the config keys be lowercased? (for .env)
614 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)
616 """
617 if not isinstance(cls, type):
618 # would not be supported according to mypy, but you can still load_into(instance)
619 return load_into_instance(
620 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
621 )
623 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
624 # _cls = typing.cast(typing.Type[C], cls)
625 return load_into_class(
626 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
627 )