Coverage for src/navdict/navdict.py: 60%
279 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-06-06 13:57 +0200
« prev ^ index » next coverage.py v7.8.2, created at 2025-06-06 13:57 +0200
1"""
2NavDict: A navigable dictionary with dot notation access and automatic file loading.
4NavDict extends Python's built-in dictionary to support convenient dot notation
5access (data.user.name) alongside traditional key access (data["user"]["name"]).
6It automatically loads data files and can instantiate classes dynamically based
7on configuration.
9Features:
10 - Dot notation access for nested data structures
11 - Automatic file loading (CSV, YAML, JSON, etc.)
12 - Dynamic class instantiation from configuration
13 - Full backward compatibility with standard dictionaries
15Example:
16 >>> from navdict import navdict
17 >>> data = navdict({"user": {"name": "Alice", "config_file": "yaml//settings.yaml"}})
18 >>> data.user.name # "Alice"
19 >>> data.user.config_file # Automatically loads and parses settings.yaml
20 >>> data["user"]["name"] # Still works with traditional access
22Author: Rik Huygen
23License: MIT
24"""
26from __future__ import annotations
28__all__ = [
29 "navdict", # noqa: ignore typo
30 "NavDict",
31]
33import csv
34import datetime
35import enum
36import importlib
37import logging
38import textwrap
39import warnings
40from enum import Enum
41from pathlib import Path
42from typing import Any
43from typing import Type
44from typing import Union
46from _ruamel_yaml import ScannerError
47from rich.text import Text
48from rich.tree import Tree
49from ruamel.yaml import YAML
51logger = logging.getLogger("navdict")
54def _load_class(class_name: str):
55 """
56 Find and returns a class based on the fully qualified name.
58 A class name can be preceded with the string `class//` or `factory//`. This is used in YAML
59 files where the class is then instantiated on load.
61 Args:
62 class_name (str): a fully qualified name for the class
63 """
64 if class_name.startswith("class//"): 64 ↛ 66line 64 didn't jump to line 66 because the condition on line 64 was always true
65 class_name = class_name[7:]
66 elif class_name.startswith("factory//"):
67 class_name = class_name[9:]
69 module_name, class_name = class_name.rsplit(".", 1)
70 module = importlib.import_module(module_name)
71 return getattr(module, class_name)
74def _load_csv(resource_name: str, **kwargs):
75 """Find and return the content of a CSV file."""
77 if resource_name.startswith("csv//"): 77 ↛ 80line 77 didn't jump to line 80 because the condition on line 77 was always true
78 resource_name = resource_name[5:]
80 parts = resource_name.rsplit("/", 1)
81 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]]
83 try:
84 n_header_rows = int(kwargs["header_rows"])
85 except KeyError:
86 n_header_rows = 0
88 try:
89 csv_location = Path(in_dir or '.').expanduser() / fn
90 with open(csv_location, 'r', encoding='utf-8') as file:
91 csv_reader = csv.reader(file)
92 data = list(csv_reader)
93 except FileNotFoundError:
94 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True)
95 raise
97 return data[:n_header_rows], data[n_header_rows:]
100def _load_int_enum(enum_name: str, enum_content) -> Type[Enum]:
101 """Dynamically build (and return) and IntEnum.
103 In the YAML file this will look like below.
104 The IntEnum directive (where <name> is the class name):
106 enum: int_enum//<name>
108 The IntEnum content:
110 content:
111 E:
112 alias: ['E_SIDE', 'RIGHT_SIDE']
113 value: 1
114 F:
115 alias: ['F_SIDE', 'LEFT_SIDE']
116 value: 0
118 Args:
119 - enum_name: Enumeration name (potentially prepended with "int_enum//").
120 - enum_content: Content of the enumeration, as read from the navdict field.
121 """
122 if enum_name.startswith("int_enum//"): 122 ↛ 125line 122 didn't jump to line 125 because the condition on line 122 was always true
123 enum_name = enum_name[10:]
125 definition = {}
126 for side_name, side_definition in enum_content.items():
127 if "alias" in side_definition: 127 ↛ 130line 127 didn't jump to line 130 because the condition on line 127 was always true
128 aliases = side_definition["alias"]
129 else:
130 aliases = []
131 value = side_definition["value"]
133 definition[side_name] = value
135 for alias in aliases:
136 definition[alias] = value
138 return enum.IntEnum(enum_name, definition)
141def _load_yaml(resource_name: str) -> NavigableDict:
142 """Find and return the content of a YAML file."""
144 if resource_name.startswith("yaml//"):
145 resource_name = resource_name[6:]
147 parts = resource_name.rsplit("/", 1)
149 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]]
151 try:
152 yaml_location = Path(in_dir or '.').expanduser()
154 yaml = YAML(typ='safe')
155 with open(yaml_location / fn, 'r') as file:
156 data = yaml.load(file)
158 except FileNotFoundError:
159 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True)
160 raise
161 except IsADirectoryError:
162 logger.error(f"Couldn't load resource '{resource_name}', file seems to be a directory", exc_info=True)
163 raise
165 return navdict(data)
168def _get_attribute(self, name, default):
169 try:
170 attr = object.__getattribute__(self, name)
171 except AttributeError:
172 attr = default
173 return attr
176class NavigableDict(dict):
177 """
178 A NavigableDict is a dictionary where all keys in the original dictionary are also accessible
179 as attributes to the class instance. So, if the original dictionary (setup) has a key
180 "site_id" which is accessible as `setup['site_id']`, it will also be accessible as
181 `setup.site_id`.
183 Examples:
184 >>> setup = NavigableDict({'site_id': 'KU Leuven', 'version': "0.1.0"})
185 >>> assert setup['site_id'] == setup.site_id
186 >>> assert setup['version'] == setup.version
188 Note:
189 We always want **all** keys to be accessible as attributes, or none. That means all
190 keys of the original dictionary shall be of type `str`.
192 """
194 def __init__(self, head: dict = None, label: str = None):
195 """
196 Args:
197 head (dict): the original dictionary
198 label (str): a label or name that is used when printing the navdict
199 """
201 head = head or {}
202 super().__init__(head)
203 self.__dict__["_memoized"] = {}
204 self.__dict__["_label"] = label
206 # By agreement, we only want the keys to be set as attributes if all keys are strings.
207 # That way we enforce that always all keys are navigable, or none.
209 if any(True for k in head.keys() if not isinstance(k, str)):
210 # invalid_keys = list(k for k in head.keys() if not isinstance(k, str))
211 # logger.warning(f"Dictionary will not be dot-navigable, not all keys are strings [{invalid_keys=}].")
212 return
214 for key, value in head.items():
215 if isinstance(value, dict):
216 setattr(self, key, NavigableDict(head.__getitem__(key)))
217 else:
218 setattr(self, key, head.__getitem__(key))
220 @property
221 def label(self) -> str | None:
222 return self._label
224 def add(self, key: str, value: Any):
225 """Set a value for the given key.
227 If the value is a dictionary, it will be converted into a NavigableDict and the keys
228 will become available as attributes provided that all the keys are strings.
230 Args:
231 key (str): the name of the key / attribute to access the value
232 value (Any): the value to assign to the key
233 """
234 if isinstance(value, dict) and not isinstance(value, NavigableDict):
235 value = NavigableDict(value)
236 setattr(self, key, value)
238 def clear(self) -> None:
239 for key in list(self.keys()):
240 self.__delitem__(key)
242 def __repr__(self):
243 return f"{self.__class__.__name__}({super()!r})"
245 def __delitem__(self, key):
246 dict.__delitem__(self, key)
247 object.__delattr__(self, key)
249 def __setattr__(self, key, value):
250 # logger.info(f"called __setattr__({self!r}, {key}, {value})")
251 if isinstance(value, dict) and not isinstance(value, NavigableDict): 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true
252 value = NavigableDict(value)
253 self.__dict__[key] = value
254 super().__setitem__(key, value)
255 try:
256 del self.__dict__["_memoized"][key]
257 except KeyError:
258 pass
260 def __getattribute__(self, key):
261 # logger.info(f"called __getattribute__({key})")
262 value = object.__getattribute__(self, key)
263 if isinstance(value, str) and value.startswith("class//"):
264 try:
265 dev_args = object.__getattribute__(self, f"{key}_args")
266 except AttributeError:
267 dev_args = ()
268 return _load_class(value)(*dev_args)
269 elif isinstance(value, str) and value.startswith("factory//"): 269 ↛ 270line 269 didn't jump to line 270 because the condition on line 269 was never true
270 factory_args = _get_attribute(self, f"{key}_args", {})
271 return _load_class(value)().create(**factory_args)
272 elif isinstance(value, str) and value.startswith("int_enum//"):
273 content = object.__getattribute__(self, "content")
274 return _load_int_enum(value, content)
275 elif isinstance(value, str) and value.startswith("csv//"):
276 if key in self.__dict__["_memoized"]: 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true
277 return self.__dict__["_memoized"][key]
278 try:
279 kwargs = object.__getattribute__(self, "kwargs")
280 except AttributeError:
281 kwargs = {}
283 content = _load_csv(value, **kwargs)
284 self.__dict__["_memoized"][key] = content
285 return content
286 elif isinstance(value, str) and value.startswith("yaml//"):
287 if key in self.__dict__["_memoized"]: 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true
288 return self.__dict__["_memoized"][key]
289 content = _load_yaml(value)
290 self.__dict__["_memoized"][key] = content
291 return content
292 else:
293 return value
295 def __delattr__(self, item):
296 # logger.info(f"called __delattr__({self!r}, {item})")
297 object.__delattr__(self, item)
298 dict.__delitem__(self, item)
300 def __setitem__(self, key, value):
301 # logger.info(f"called __setitem__({self!r}, {key}, {value})")
302 if isinstance(value, dict) and not isinstance(value, NavigableDict):
303 value = NavigableDict(value)
304 super().__setitem__(key, value)
305 self.__dict__[key] = value
306 try:
307 del self.__dict__["_memoized"][key]
308 except KeyError:
309 pass
311 def __getitem__(self, key):
312 # logger.info(f"called __getitem__({self!r}, {key})")
313 value = super().__getitem__(key)
314 if isinstance(value, str) and value.startswith("class//"): 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true
315 try:
316 dev_args = object.__getattribute__(self, "device_args")
317 except AttributeError:
318 dev_args = ()
319 return _load_class(value)(*dev_args)
320 if isinstance(value, str) and value.startswith("csv//"): 320 ↛ 321line 320 didn't jump to line 321 because the condition on line 320 was never true
321 try:
322 kwargs = object.__getattribute__(self, "kwargs")
323 except AttributeError:
324 kwargs = {}
325 return _load_csv(value, **kwargs)
326 if isinstance(value, str) and value.startswith("int_enum//"): 326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true
327 content = object.__getattribute__(self, "content")
328 return _load_int_enum(value, content)
329 else:
330 return value
332 def set_private_attribute(self, key: str, value: Any) -> None:
333 """Sets a private attribute for this object.
335 The name in key will be accessible as an attribute for this object, but the key will not
336 be added to the dictionary and not be returned by methods like keys().
338 The idea behind this private attribute is to have the possibility to add status information
339 or identifiers to this classes object that can be used by save() or load() methods.
341 Args:
342 key (str): the name of the private attribute (must start with an underscore character).
343 value: the value for this private attribute
345 Examples:
346 >>> setup = NavigableDict({'a': 1, 'b': 2, 'c': 3})
347 >>> setup.set_private_attribute("_loaded_from_dict", True)
348 >>> assert "c" in setup
349 >>> assert "_loaded_from_dict" not in setup
350 >>> assert setup.get_private_attribute("_loaded_from_dict") == True
352 """
353 if key in self: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 raise ValueError(f"Invalid argument key='{key}', this key already exists in the dictionary.")
355 if not key.startswith("_"): 355 ↛ 356line 355 didn't jump to line 356 because the condition on line 355 was never true
356 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.")
357 self.__dict__[key] = value
359 def get_private_attribute(self, key: str) -> Any:
360 """Returns the value of the given private attribute.
362 Args:
363 key (str): the name of the private attribute (must start with an underscore character).
365 Returns:
366 the value of the private attribute given in `key`.
368 Note:
369 Because of the implementation, this private attribute can also be accessed as a 'normal'
370 attribute of the object. This use is however discouraged as it will make your code less
371 understandable. Use the methods to access these 'private' attributes.
372 """
373 if not key.startswith("_"):
374 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.")
375 return self.__dict__[key]
377 def has_private_attribute(self, key):
378 """
379 Check if the given key is defined as a private attribute.
381 Args:
382 key (str): the name of a private attribute (must start with an underscore)
383 Returns:
384 True if the given key is a known private attribute.
385 Raises:
386 ValueError: when the key doesn't start with an underscore.
387 """
388 if not key.startswith("_"):
389 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.")
391 try:
392 _ = self.__dict__[key]
393 return True
394 except KeyError:
395 return False
397 def get_raw_value(self, key):
398 """
399 Returns the raw value of the given key.
401 Some keys have special values that are interpreted by the NavigableDict class. An example is
402 a value that starts with 'class//'. When you access these values, they are first converted
403 from their raw value into their expected value, e.g. the instantiated object in the above
404 example. This method allows you to access the raw value before conversion.
405 """
406 try:
407 return object.__getattribute__(self, key)
408 except AttributeError:
409 raise KeyError(f"The key '{key}' is not defined.")
411 def __str__(self):
412 return self._pretty_str()
414 def _pretty_str(self, indent: int = 0):
415 msg = ""
417 for k, v in self.items():
418 if isinstance(v, NavigableDict):
419 msg += f"{' ' * indent}{k}:\n"
420 msg += v._pretty_str(indent + 1)
421 else:
422 msg += f"{' ' * indent}{k}: {v}\n"
424 return msg
426 def __rich__(self) -> Tree:
427 tree = Tree(self.__dict__["_label"] or "NavigableDict", guide_style="dim")
428 _walk_dict_tree(self, tree, text_style="dark grey")
429 return tree
431 def _save(self, fd, indent: int = 0):
432 """
433 Recursive method to write the dictionary to the file descriptor.
435 Indentation is done in steps of four spaces, i.e. `' '*indent`.
437 Args:
438 fd: a file descriptor as returned by the open() function
439 indent (int): indentation level of each line [default = 0]
441 """
443 # Note that the .items() method returns the actual values of the keys and doesn't use the
444 # __getattribute__ or __getitem__ methods. So the raw value is returned and not the
445 # _processed_ value.
447 for k, v in self.items():
448 # history shall be saved last, skip it for now
450 if k == "history": 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true
451 continue
453 # make sure to escape a colon in the key name
455 if isinstance(k, str) and ":" in k: 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true
456 k = '"' + k + '"'
458 if isinstance(v, NavigableDict):
459 fd.write(f"{' ' * indent}{k}:\n")
460 v._save(fd, indent + 1)
461 fd.flush()
462 continue
464 if isinstance(v, float): 464 ↛ 465line 464 didn't jump to line 465 because the condition on line 464 was never true
465 v = f"{v:.6E}"
466 fd.write(f"{' ' * indent}{k}: {v}\n")
467 fd.flush()
469 # now save the history as the last item
471 if "history" in self: 471 ↛ 472line 471 didn't jump to line 472 because the condition on line 471 was never true
472 fd.write(f"{' ' * indent}history:\n")
473 self.history._save(fd, indent + 1) # noqa
475 def get_memoized_keys(self):
476 return list(self.__dict__["_memoized"].keys())
478 @staticmethod
479 def from_dict(my_dict: dict, label: str = None) -> NavigableDict:
480 """Create a NavigableDict from a given dictionary.
482 Remember that all keys in the given dictionary shall be of type 'str' in order to be
483 accessible as attributes.
485 Args:
486 my_dict: a Python dictionary
487 label: a label that will be attached to this navdict
489 Examples:
490 >>> setup = navdict.from_dict({"ID": "my-setup-001", "version": "0.1.0"}, label="Setup")
491 >>> assert setup["ID"] == setup.ID == "my-setup-001"
493 """
494 return NavigableDict(my_dict, label=label)
496 @staticmethod
497 def from_yaml_string(yaml_content: str = None, label: str = None) -> NavigableDict:
498 """Creates a NavigableDict from the given YAML string.
500 This method is mainly used for easy creation of a navdict from strings during unit tests.
502 Args:
503 yaml_content: a string containing YAML
504 label: a label that will be attached to this navdict
506 Returns:
507 a navdict that was loaded from the content of the given string.
508 """
510 if not yaml_content:
511 raise ValueError("Invalid argument to function: No input string or None given.")
513 yaml = YAML(typ='safe')
514 try:
515 data = yaml.load(yaml_content)
516 except ScannerError as exc:
517 raise ValueError(f"Invalid YAML string: {exc}")
519 return NavigableDict(data, label=label)
521 @staticmethod
522 def from_yaml_file(filename: Union[str, Path] = None) -> NavigableDict:
523 """Creates a navigable dictionary from the given YAML file.
525 Args:
526 filename (str): the path of the YAML file to be loaded
528 Returns:
529 a navdict that was loaded from the given location.
531 Raises:
532 ValueError: when no filename is given.
533 """
535 if not filename: 535 ↛ 536line 535 didn't jump to line 536 because the condition on line 535 was never true
536 raise ValueError("Invalid argument to function: No filename or None given.")
538 data = _load_yaml(str(filename))
540 if data == {}: 540 ↛ 541line 540 didn't jump to line 541 because the condition on line 540 was never true
541 warnings.warn(f"Empty YAML file: {filename!s}")
543 data.set_private_attribute("_filename", Path(filename))
545 return data
547 def to_yaml_file(self, filename: str | Path = None) -> None:
548 """Saves a NavigableDict to a YAML file.
550 When no filename is provided, this method will look for a 'private' attribute
551 `_filename` and use that to save the data.
553 Args:
554 filename (str|Path): the path of the YAML file where to save the data
556 Note:
557 This method will **overwrite** the original or given YAML file and therefore you might
558 lose proper formatting and/or comments.
560 """
561 if not filename: 561 ↛ 562line 561 didn't jump to line 562 because the condition on line 561 was never true
562 try:
563 filename = self.get_private_attribute("_filename")
564 except KeyError:
565 raise ValueError("No filename given or known, can not save navdict.")
567 with Path(filename).open("w") as fd:
568 fd.write(
569 textwrap.dedent(
570 f"""
571 # This YAML file is generated by:
572 #
573 # Setup.to_yaml_file(setup, filename="{filename}')
574 #
575 # Created on {datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}
577 """
578 )
579 )
581 self._save(fd, indent=0)
583 self.set_private_attribute("_filename", Path(filename))
585 def get_filename(self) -> str | None:
586 """Returns the filename for this navdict or None when no filename could be determined."""
587 if self.has_private_attribute("_filename"):
588 return self.get_private_attribute("_filename")
589 else:
590 return None
593navdict = NavDict = NavigableDict # noqa: ignore typo
594"""Shortcuts for NavigableDict and more Pythonic."""
597def _walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"):
598 for k, v in dictionary.items():
599 if isinstance(v, dict):
600 branch = tree.add(f"[purple]{k}", style="", guide_style="dim")
601 _walk_dict_tree(v, branch, text_style=text_style)
602 else:
603 text = Text.assemble((str(k), "medium_purple1"), ": ", (str(v), text_style))
604 tree.add(text)