Coverage for src/navdict/navdict.py: 64%
277 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-06-06 22:49 +0200
« prev ^ index » next coverage.py v7.8.2, created at 2025-06-06 22:49 +0200
1"""
2NavDict: A navigable dictionary with dot notation access and automatic file loading.
4NavDict extends Python's built-in dictionary to support convenient dot notation
5access (data.user.name) alongside traditional key access (data["user"]["name"]).
6It automatically loads data files and can instantiate classes dynamically based
7on configuration.
9Features:
10 - Dot notation access for nested data structures
11 - Automatic file loading (CSV, YAML, JSON, etc.)
12 - Dynamic class instantiation from configuration
13 - Full backward compatibility with standard dictionaries
15Example:
16 >>> from navdict import navdict
17 >>> data = navdict({"user": {"name": "Alice", "config_file": "yaml//settings.yaml"}})
18 >>> data.user.name # "Alice"
19 >>> data.user.config_file # Automatically loads and parses settings.yaml
20 >>> data["user"]["name"] # Still works with traditional access
22Author: Rik Huygen
23License: MIT
24"""
26from __future__ import annotations
28__all__ = [
29 "navdict", # noqa: ignore typo
30 "NavDict",
31]
33import csv
34import datetime
35import enum
36import importlib
37import logging
38import textwrap
39import warnings
40from enum import Enum
41from pathlib import Path
42from typing import Any
43from typing import Type
44from typing import Union
46from _ruamel_yaml import ScannerError
47from rich.text import Text
48from rich.tree import Tree
49from ruamel.yaml import YAML
51logger = logging.getLogger("navdict")
54def _search_directive_plugin():
55 ...
58def _load_class(class_name: str):
59 """
60 Find and returns a class based on the fully qualified name.
62 A class name can be preceded with the string `class//` or `factory//`. This is used in YAML
63 files where the class is then instantiated on load.
65 Args:
66 class_name (str): a fully qualified name for the class
67 """
68 if class_name.startswith("class//"): 68 ↛ 70line 68 didn't jump to line 70 because the condition on line 68 was always true
69 class_name = class_name[7:]
70 elif class_name.startswith("factory//"):
71 class_name = class_name[9:]
73 module_name, class_name = class_name.rsplit(".", 1)
74 module = importlib.import_module(module_name)
75 return getattr(module, class_name)
78def _load_csv(resource_name: str, *args, **kwargs):
79 """Find and return the content of a CSV file."""
81 if resource_name.startswith("csv//"): 81 ↛ 84line 81 didn't jump to line 84 because the condition on line 81 was always true
82 resource_name = resource_name[5:]
84 parts = resource_name.rsplit("/", 1)
85 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]]
87 try:
88 n_header_rows = int(kwargs["header_rows"])
89 except KeyError:
90 n_header_rows = 0
92 try:
93 csv_location = Path(in_dir or '.').expanduser() / fn
94 with open(csv_location, 'r', encoding='utf-8') as file:
95 csv_reader = csv.reader(file)
96 data = list(csv_reader)
97 except FileNotFoundError:
98 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True)
99 raise
101 return data[:n_header_rows], data[n_header_rows:]
104def _load_int_enum(enum_name: str, enum_content) -> Type[Enum]:
105 """Dynamically build (and return) and IntEnum.
107 In the YAML file this will look like below.
108 The IntEnum directive (where <name> is the class name):
110 enum: int_enum//<name>
112 The IntEnum content:
114 content:
115 E:
116 alias: ['E_SIDE', 'RIGHT_SIDE']
117 value: 1
118 F:
119 alias: ['F_SIDE', 'LEFT_SIDE']
120 value: 0
122 Args:
123 - enum_name: Enumeration name (potentially prepended with "int_enum//").
124 - enum_content: Content of the enumeration, as read from the navdict field.
125 """
126 if enum_name.startswith("int_enum//"): 126 ↛ 129line 126 didn't jump to line 129 because the condition on line 126 was always true
127 enum_name = enum_name[10:]
129 definition = {}
130 for side_name, side_definition in enum_content.items():
131 if "alias" in side_definition: 131 ↛ 134line 131 didn't jump to line 134 because the condition on line 131 was always true
132 aliases = side_definition["alias"]
133 else:
134 aliases = []
135 value = side_definition["value"]
137 definition[side_name] = value
139 for alias in aliases:
140 definition[alias] = value
142 return enum.IntEnum(enum_name, definition)
145def _load_yaml(resource_name: str) -> NavigableDict:
146 """Find and return the content of a YAML file."""
148 if resource_name.startswith("yaml//"):
149 resource_name = resource_name[6:]
151 parts = resource_name.rsplit("/", 1)
153 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]]
155 try:
156 yaml_location = Path(in_dir or '.').expanduser()
158 yaml = YAML(typ='safe')
159 with open(yaml_location / fn, 'r') as file:
160 data = yaml.load(file)
162 except FileNotFoundError:
163 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True)
164 raise
165 except IsADirectoryError:
166 logger.error(f"Couldn't load resource '{resource_name}', file seems to be a directory", exc_info=True)
167 raise
169 return navdict(data)
172def _get_attribute(self, name, default):
173 try:
174 attr = object.__getattribute__(self, name)
175 except AttributeError:
176 attr = default
177 return attr
180class NavigableDict(dict):
181 """
182 A NavigableDict is a dictionary where all keys in the original dictionary are also accessible
183 as attributes to the class instance. So, if the original dictionary (setup) has a key
184 "site_id" which is accessible as `setup['site_id']`, it will also be accessible as
185 `setup.site_id`.
187 Args:
188 head (dict): the original dictionary
189 label (str): a label or name that is used when printing the navdict
191 Examples:
192 >>> setup = NavigableDict({'site_id': 'KU Leuven', 'version': "0.1.0"})
193 >>> assert setup['site_id'] == setup.site_id
194 >>> assert setup['version'] == setup.version
196 Note:
197 We always want **all** keys to be accessible as attributes, or none. That means all
198 keys of the original dictionary shall be of type `str`.
200 """
202 def __init__(self, head: dict = None, label: str = None):
204 head = head or {}
205 super().__init__(head)
206 self.__dict__["_memoized"] = {}
207 self.__dict__["_label"] = label
209 # By agreement, we only want the keys to be set as attributes if all keys are strings.
210 # That way we enforce that always all keys are navigable, or none.
212 if any(True for k in head.keys() if not isinstance(k, str)):
213 # invalid_keys = list(k for k in head.keys() if not isinstance(k, str))
214 # logger.warning(f"Dictionary will not be dot-navigable, not all keys are strings [{invalid_keys=}].")
215 return
217 for key, value in head.items():
218 if isinstance(value, dict):
219 setattr(self, key, NavigableDict(head.__getitem__(key)))
220 else:
221 setattr(self, key, head.__getitem__(key))
223 @property
224 def label(self) -> str | None:
225 return self._label
227 def add(self, key: str, value: Any):
228 """Set a value for the given key.
230 If the value is a dictionary, it will be converted into a NavigableDict and the keys
231 will become available as attributes provided that all the keys are strings.
233 Args:
234 key (str): the name of the key / attribute to access the value
235 value (Any): the value to assign to the key
236 """
237 if isinstance(value, dict) and not isinstance(value, NavigableDict):
238 value = NavigableDict(value)
239 setattr(self, key, value)
241 def clear(self) -> None:
242 for key in list(self.keys()):
243 self.__delitem__(key)
245 def __repr__(self):
246 return f"{self.__class__.__name__}({super()!r})"
248 def __delitem__(self, key):
249 dict.__delitem__(self, key)
250 object.__delattr__(self, key)
252 def __setattr__(self, key, value):
253 # logger.info(f"called __setattr__({self!r}, {key}, {value})")
254 if isinstance(value, dict) and not isinstance(value, NavigableDict): 254 ↛ 255line 254 didn't jump to line 255 because the condition on line 254 was never true
255 value = NavigableDict(value)
256 self.__dict__[key] = value
257 super().__setitem__(key, value)
258 try:
259 del self.__dict__["_memoized"][key]
260 except KeyError:
261 pass
263 # This method is called:
264 # - for *every* single attribute access on an object using dot notation.
265 # - when using the `getattr(obj, 'name') function
266 # - accessing any kind of attributes, e.g. instance or class variables,
267 # methods, properties, dunder methods, ...
268 #
269 # Note: `__getattr__` is only called when an attribute cannot be found
270 # through normal means.
271 def __getattribute__(self, key):
272 # logger.info(f"called __getattribute__({key})")
273 value = object.__getattribute__(self, key)
274 if key.startswith('__'): # small optimization
275 return value
276 m = object.__getattribute__(self, "_handle_directive")
277 return m(key, value)
279 def __delattr__(self, item):
280 # logger.info(f"called __delattr__({self!r}, {item})")
281 object.__delattr__(self, item)
282 dict.__delitem__(self, item)
284 def __setitem__(self, key, value):
285 # logger.info(f"called __setitem__({self!r}, {key}, {value})")
286 if isinstance(value, dict) and not isinstance(value, NavigableDict):
287 value = NavigableDict(value)
288 super().__setitem__(key, value)
289 self.__dict__[key] = value
290 try:
291 del self.__dict__["_memoized"][key]
292 except KeyError:
293 pass
295 # This method is called:
296 # - whenever square brackets `[]` are used on an object, e.g. indexing or slicing.
297 # - during iteration, if an object doesn't have __iter__ defined, Python will try
298 # to iterate using __getitem__ with successive integer indices starting from 0.
299 def __getitem__(self, key):
300 # logger.info(f"called __getitem__({self!r}, {key})")
301 value = super().__getitem__(key)
302 if key.startswith('__'): 302 ↛ 303line 302 didn't jump to line 303 because the condition on line 302 was never true
303 return value
304 return self._handle_directive(key, value)
306 def _handle_directive(self, key, value) -> Any:
307 """
308 This method will handle the available directives. This may be builtin directives
309 like `class/` or `yaml//`, or it may be external directives that were provided
310 as a plugin. TO BE IMPLEMENTED
312 Args:
313 key: the key of the field that might contain a directive
314 value: the value which might be a directive
316 Returns:
317 This function will return the value, either the original value or the result of
318 evaluating and executing a directive.
319 """
320 # logger.info(f"called _handle_directive({key}, {value!r})")
321 if isinstance(value, str) and value.startswith("class//"):
322 args, kwargs = self._get_args_and_kwargs(key)
323 return _load_class(value)(*args, **kwargs)
325 elif isinstance(value, str) and value.startswith("factory//"): 325 ↛ 326line 325 didn't jump to line 326 because the condition on line 325 was never true
326 factory_args = _get_attribute(self, f"{key}_args", {})
327 return _load_class(value)().create(**factory_args)
329 elif isinstance(value, str) and value.startswith("yaml//"):
330 if key in self.__dict__["_memoized"]: 330 ↛ 331line 330 didn't jump to line 331 because the condition on line 330 was never true
331 return self.__dict__["_memoized"][key]
332 content = _load_yaml(value)
333 self.__dict__["_memoized"][key] = content
334 return content
336 elif isinstance(value, str) and value.startswith("csv//"):
337 if key in self.__dict__["_memoized"]: 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true
338 return self.__dict__["_memoized"][key]
339 args, kwargs = self._get_args_and_kwargs(key)
340 content = _load_csv(value, **kwargs)
341 self.__dict__["_memoized"][key] = content
342 return content
344 elif isinstance(value, str) and value.startswith("int_enum//"):
345 content = object.__getattribute__(self, "content")
346 return _load_int_enum(value, content)
348 else:
349 return value
351 def _get_args_and_kwargs(self, key):
352 """
353 Read the args and kwargs that are associated with the key of a directive.
355 An example of such a directive:
357 hexapod:
358 device: class//egse.hexapod.PunaProxy
359 device_args: [PUNA_01]
360 device_kwargs:
361 sim: true
363 There might not be any positional nor keyword arguments provided in which
364 case and empty tuple and/or dictionary is returned.
366 Returns:
367 A tuple containing any positional arguments and a dictionary containing
368 keyword arguments.
369 """
370 try:
371 args = object.__getattribute__(self, f"{key}_args")
372 except AttributeError:
373 args = ()
374 try:
375 kwargs = object.__getattribute__(self, f"{key}_kwargs")
376 except AttributeError:
377 kwargs = {}
379 return args, kwargs
381 def set_private_attribute(self, key: str, value: Any) -> None:
382 """Sets a private attribute for this object.
384 The name in key will be accessible as an attribute for this object, but the key will not
385 be added to the dictionary and not be returned by methods like keys().
387 The idea behind this private attribute is to have the possibility to add status information
388 or identifiers to this classes object that can be used by save() or load() methods.
390 Args:
391 key (str): the name of the private attribute (must start with an underscore character).
392 value: the value for this private attribute
394 Examples:
395 >>> setup = NavigableDict({'a': 1, 'b': 2, 'c': 3})
396 >>> setup.set_private_attribute("_loaded_from_dict", True)
397 >>> assert "c" in setup
398 >>> assert "_loaded_from_dict" not in setup
399 >>> assert setup.get_private_attribute("_loaded_from_dict") == True
401 """
402 if key in self: 402 ↛ 403line 402 didn't jump to line 403 because the condition on line 402 was never true
403 raise ValueError(f"Invalid argument key='{key}', this key already exists in the dictionary.")
404 if not key.startswith("_"): 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true
405 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.")
406 self.__dict__[key] = value
408 def get_private_attribute(self, key: str) -> Any:
409 """Returns the value of the given private attribute.
411 Args:
412 key (str): the name of the private attribute (must start with an underscore character).
414 Returns:
415 the value of the private attribute given in `key`.
417 Note:
418 Because of the implementation, this private attribute can also be accessed as a 'normal'
419 attribute of the object. This use is however discouraged as it will make your code less
420 understandable. Use the methods to access these 'private' attributes.
421 """
422 if not key.startswith("_"):
423 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.")
424 return self.__dict__[key]
426 def has_private_attribute(self, key):
427 """
428 Check if the given key is defined as a private attribute.
430 Args:
431 key (str): the name of a private attribute (must start with an underscore)
432 Returns:
433 True if the given key is a known private attribute.
434 Raises:
435 ValueError: when the key doesn't start with an underscore.
436 """
437 if not key.startswith("_"):
438 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.")
440 try:
441 _ = self.__dict__[key]
442 return True
443 except KeyError:
444 return False
446 def get_raw_value(self, key):
447 """
448 Returns the raw value of the given key.
450 Some keys have special values that are interpreted by the NavigableDict class. An example is
451 a value that starts with 'class//'. When you access these values, they are first converted
452 from their raw value into their expected value, e.g. the instantiated object in the above
453 example. This method allows you to access the raw value before conversion.
454 """
455 try:
456 return object.__getattribute__(self, key)
457 except AttributeError:
458 raise KeyError(f"The key '{key}' is not defined.")
460 def __str__(self):
461 return self._pretty_str()
463 def _pretty_str(self, indent: int = 0):
464 msg = ""
466 for k, v in self.items():
467 if isinstance(v, NavigableDict):
468 msg += f"{' ' * indent}{k}:\n"
469 msg += v._pretty_str(indent + 1)
470 else:
471 msg += f"{' ' * indent}{k}: {v}\n"
473 return msg
475 def __rich__(self) -> Tree:
476 tree = Tree(self.__dict__["_label"] or "NavigableDict", guide_style="dim")
477 _walk_dict_tree(self, tree, text_style="dark grey")
478 return tree
480 def _save(self, fd, indent: int = 0):
481 """
482 Recursive method to write the dictionary to the file descriptor.
484 Indentation is done in steps of four spaces, i.e. `' '*indent`.
486 Args:
487 fd: a file descriptor as returned by the open() function
488 indent (int): indentation level of each line [default = 0]
490 """
492 # Note that the .items() method returns the actual values of the keys and doesn't use the
493 # __getattribute__ or __getitem__ methods. So the raw value is returned and not the
494 # _processed_ value.
496 for k, v in self.items():
497 # history shall be saved last, skip it for now
499 if k == "history": 499 ↛ 500line 499 didn't jump to line 500 because the condition on line 499 was never true
500 continue
502 # make sure to escape a colon in the key name
504 if isinstance(k, str) and ":" in k: 504 ↛ 505line 504 didn't jump to line 505 because the condition on line 504 was never true
505 k = '"' + k + '"'
507 if isinstance(v, NavigableDict):
508 fd.write(f"{' ' * indent}{k}:\n")
509 v._save(fd, indent + 1)
510 fd.flush()
511 continue
513 if isinstance(v, float): 513 ↛ 514line 513 didn't jump to line 514 because the condition on line 513 was never true
514 v = f"{v:.6E}"
515 fd.write(f"{' ' * indent}{k}: {v}\n")
516 fd.flush()
518 # now save the history as the last item
520 if "history" in self: 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true
521 fd.write(f"{' ' * indent}history:\n")
522 self.history._save(fd, indent + 1) # noqa
524 def get_memoized_keys(self):
525 return list(self.__dict__["_memoized"].keys())
527 @staticmethod
528 def from_dict(my_dict: dict, label: str = None) -> NavigableDict:
529 """Create a NavigableDict from a given dictionary.
531 Remember that all keys in the given dictionary shall be of type 'str' in order to be
532 accessible as attributes.
534 Args:
535 my_dict: a Python dictionary
536 label: a label that will be attached to this navdict
538 Examples:
539 >>> setup = navdict.from_dict({"ID": "my-setup-001", "version": "0.1.0"}, label="Setup")
540 >>> assert setup["ID"] == setup.ID == "my-setup-001"
542 """
543 return NavigableDict(my_dict, label=label)
545 @staticmethod
546 def from_yaml_string(yaml_content: str = None, label: str = None) -> NavigableDict:
547 """Creates a NavigableDict from the given YAML string.
549 This method is mainly used for easy creation of a navdict from strings during unit tests.
551 Args:
552 yaml_content: a string containing YAML
553 label: a label that will be attached to this navdict
555 Returns:
556 a navdict that was loaded from the content of the given string.
557 """
559 if not yaml_content:
560 raise ValueError("Invalid argument to function: No input string or None given.")
562 yaml = YAML(typ='safe')
563 try:
564 data = yaml.load(yaml_content)
565 except ScannerError as exc:
566 raise ValueError(f"Invalid YAML string: {exc}")
568 return NavigableDict(data, label=label)
570 @staticmethod
571 def from_yaml_file(filename: Union[str, Path] = None) -> NavigableDict:
572 """Creates a navigable dictionary from the given YAML file.
574 Args:
575 filename (str): the path of the YAML file to be loaded
577 Returns:
578 a navdict that was loaded from the given location.
580 Raises:
581 ValueError: when no filename is given.
582 """
584 if not filename: 584 ↛ 585line 584 didn't jump to line 585 because the condition on line 584 was never true
585 raise ValueError("Invalid argument to function: No filename or None given.")
587 data = _load_yaml(str(filename))
589 if data == {}: 589 ↛ 590line 589 didn't jump to line 590 because the condition on line 589 was never true
590 warnings.warn(f"Empty YAML file: {filename!s}")
592 data.set_private_attribute("_filename", Path(filename))
594 return data
596 def to_yaml_file(self, filename: str | Path = None) -> None:
597 """Saves a NavigableDict to a YAML file.
599 When no filename is provided, this method will look for a 'private' attribute
600 `_filename` and use that to save the data.
602 Args:
603 filename (str|Path): the path of the YAML file where to save the data
605 Note:
606 This method will **overwrite** the original or given YAML file and therefore you might
607 lose proper formatting and/or comments.
609 """
610 if not filename: 610 ↛ 611line 610 didn't jump to line 611 because the condition on line 610 was never true
611 try:
612 filename = self.get_private_attribute("_filename")
613 except KeyError:
614 raise ValueError("No filename given or known, can not save navdict.")
616 with Path(filename).open("w") as fd:
617 fd.write(
618 textwrap.dedent(
619 f"""
620 # This YAML file is generated by:
621 #
622 # Setup.to_yaml_file(setup, filename="{filename}')
623 #
624 # Created on {datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}
626 """
627 )
628 )
630 self._save(fd, indent=0)
632 self.set_private_attribute("_filename", Path(filename))
634 def get_filename(self) -> str | None:
635 """Returns the filename for this navdict or None when no filename could be determined."""
636 if self.has_private_attribute("_filename"):
637 return self.get_private_attribute("_filename")
638 else:
639 return None
642navdict = NavDict = NavigableDict # noqa: ignore typo
643"""Shortcuts for NavigableDict and more Pythonic."""
646def _walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"):
647 for k, v in dictionary.items():
648 if isinstance(v, dict):
649 branch = tree.add(f"[purple]{k}", style="", guide_style="dim")
650 _walk_dict_tree(v, branch, text_style=text_style)
651 else:
652 text = Text.assemble((str(k), "medium_purple1"), ": ", (str(v), text_style))
653 tree.add(text)