Coverage for src/navdict/navdict.py: 64%

277 statements  

« prev     ^ index     » next       coverage.py v7.8.2, created at 2025-06-06 22:49 +0200

1""" 

2NavDict: A navigable dictionary with dot notation access and automatic file loading. 

3 

4NavDict extends Python's built-in dictionary to support convenient dot notation 

5access (data.user.name) alongside traditional key access (data["user"]["name"]). 

6It automatically loads data files and can instantiate classes dynamically based 

7on configuration. 

8 

9Features: 

10 - Dot notation access for nested data structures 

11 - Automatic file loading (CSV, YAML, JSON, etc.) 

12 - Dynamic class instantiation from configuration 

13 - Full backward compatibility with standard dictionaries 

14 

15Example: 

16 >>> from navdict import navdict 

17 >>> data = navdict({"user": {"name": "Alice", "config_file": "yaml//settings.yaml"}}) 

18 >>> data.user.name # "Alice" 

19 >>> data.user.config_file # Automatically loads and parses settings.yaml 

20 >>> data["user"]["name"] # Still works with traditional access 

21 

22Author: Rik Huygen 

23License: MIT 

24""" 

25 

26from __future__ import annotations 

27 

28__all__ = [ 

29 "navdict", # noqa: ignore typo 

30 "NavDict", 

31] 

32 

33import csv 

34import datetime 

35import enum 

36import importlib 

37import logging 

38import textwrap 

39import warnings 

40from enum import Enum 

41from pathlib import Path 

42from typing import Any 

43from typing import Type 

44from typing import Union 

45 

46from _ruamel_yaml import ScannerError 

47from rich.text import Text 

48from rich.tree import Tree 

49from ruamel.yaml import YAML 

50 

51logger = logging.getLogger("navdict") 

52 

53 

54def _search_directive_plugin(): 

55 ... 

56 

57 

58def _load_class(class_name: str): 

59 """ 

60 Find and returns a class based on the fully qualified name. 

61 

62 A class name can be preceded with the string `class//` or `factory//`. This is used in YAML 

63 files where the class is then instantiated on load. 

64 

65 Args: 

66 class_name (str): a fully qualified name for the class 

67 """ 

68 if class_name.startswith("class//"): 68 ↛ 70line 68 didn't jump to line 70 because the condition on line 68 was always true

69 class_name = class_name[7:] 

70 elif class_name.startswith("factory//"): 

71 class_name = class_name[9:] 

72 

73 module_name, class_name = class_name.rsplit(".", 1) 

74 module = importlib.import_module(module_name) 

75 return getattr(module, class_name) 

76 

77 

78def _load_csv(resource_name: str, *args, **kwargs): 

79 """Find and return the content of a CSV file.""" 

80 

81 if resource_name.startswith("csv//"): 81 ↛ 84line 81 didn't jump to line 84 because the condition on line 81 was always true

82 resource_name = resource_name[5:] 

83 

84 parts = resource_name.rsplit("/", 1) 

85 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]] 

86 

87 try: 

88 n_header_rows = int(kwargs["header_rows"]) 

89 except KeyError: 

90 n_header_rows = 0 

91 

92 try: 

93 csv_location = Path(in_dir or '.').expanduser() / fn 

94 with open(csv_location, 'r', encoding='utf-8') as file: 

95 csv_reader = csv.reader(file) 

96 data = list(csv_reader) 

97 except FileNotFoundError: 

98 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True) 

99 raise 

100 

101 return data[:n_header_rows], data[n_header_rows:] 

102 

103 

104def _load_int_enum(enum_name: str, enum_content) -> Type[Enum]: 

105 """Dynamically build (and return) and IntEnum. 

106 

107 In the YAML file this will look like below. 

108 The IntEnum directive (where <name> is the class name): 

109 

110 enum: int_enum//<name> 

111 

112 The IntEnum content: 

113 

114 content: 

115 E: 

116 alias: ['E_SIDE', 'RIGHT_SIDE'] 

117 value: 1 

118 F: 

119 alias: ['F_SIDE', 'LEFT_SIDE'] 

120 value: 0 

121 

122 Args: 

123 - enum_name: Enumeration name (potentially prepended with "int_enum//"). 

124 - enum_content: Content of the enumeration, as read from the navdict field. 

125 """ 

126 if enum_name.startswith("int_enum//"): 126 ↛ 129line 126 didn't jump to line 129 because the condition on line 126 was always true

127 enum_name = enum_name[10:] 

128 

129 definition = {} 

130 for side_name, side_definition in enum_content.items(): 

131 if "alias" in side_definition: 131 ↛ 134line 131 didn't jump to line 134 because the condition on line 131 was always true

132 aliases = side_definition["alias"] 

133 else: 

134 aliases = [] 

135 value = side_definition["value"] 

136 

137 definition[side_name] = value 

138 

139 for alias in aliases: 

140 definition[alias] = value 

141 

142 return enum.IntEnum(enum_name, definition) 

143 

144 

145def _load_yaml(resource_name: str) -> NavigableDict: 

146 """Find and return the content of a YAML file.""" 

147 

148 if resource_name.startswith("yaml//"): 

149 resource_name = resource_name[6:] 

150 

151 parts = resource_name.rsplit("/", 1) 

152 

153 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]] 

154 

155 try: 

156 yaml_location = Path(in_dir or '.').expanduser() 

157 

158 yaml = YAML(typ='safe') 

159 with open(yaml_location / fn, 'r') as file: 

160 data = yaml.load(file) 

161 

162 except FileNotFoundError: 

163 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True) 

164 raise 

165 except IsADirectoryError: 

166 logger.error(f"Couldn't load resource '{resource_name}', file seems to be a directory", exc_info=True) 

167 raise 

168 

169 return navdict(data) 

170 

171 

172def _get_attribute(self, name, default): 

173 try: 

174 attr = object.__getattribute__(self, name) 

175 except AttributeError: 

176 attr = default 

177 return attr 

178 

179 

180class NavigableDict(dict): 

181 """ 

182 A NavigableDict is a dictionary where all keys in the original dictionary are also accessible 

183 as attributes to the class instance. So, if the original dictionary (setup) has a key 

184 "site_id" which is accessible as `setup['site_id']`, it will also be accessible as 

185 `setup.site_id`. 

186 

187 Args: 

188 head (dict): the original dictionary 

189 label (str): a label or name that is used when printing the navdict 

190 

191 Examples: 

192 >>> setup = NavigableDict({'site_id': 'KU Leuven', 'version': "0.1.0"}) 

193 >>> assert setup['site_id'] == setup.site_id 

194 >>> assert setup['version'] == setup.version 

195 

196 Note: 

197 We always want **all** keys to be accessible as attributes, or none. That means all 

198 keys of the original dictionary shall be of type `str`. 

199 

200 """ 

201 

202 def __init__(self, head: dict = None, label: str = None): 

203 

204 head = head or {} 

205 super().__init__(head) 

206 self.__dict__["_memoized"] = {} 

207 self.__dict__["_label"] = label 

208 

209 # By agreement, we only want the keys to be set as attributes if all keys are strings. 

210 # That way we enforce that always all keys are navigable, or none. 

211 

212 if any(True for k in head.keys() if not isinstance(k, str)): 

213 # invalid_keys = list(k for k in head.keys() if not isinstance(k, str)) 

214 # logger.warning(f"Dictionary will not be dot-navigable, not all keys are strings [{invalid_keys=}].") 

215 return 

216 

217 for key, value in head.items(): 

218 if isinstance(value, dict): 

219 setattr(self, key, NavigableDict(head.__getitem__(key))) 

220 else: 

221 setattr(self, key, head.__getitem__(key)) 

222 

223 @property 

224 def label(self) -> str | None: 

225 return self._label 

226 

227 def add(self, key: str, value: Any): 

228 """Set a value for the given key. 

229 

230 If the value is a dictionary, it will be converted into a NavigableDict and the keys 

231 will become available as attributes provided that all the keys are strings. 

232 

233 Args: 

234 key (str): the name of the key / attribute to access the value 

235 value (Any): the value to assign to the key 

236 """ 

237 if isinstance(value, dict) and not isinstance(value, NavigableDict): 

238 value = NavigableDict(value) 

239 setattr(self, key, value) 

240 

241 def clear(self) -> None: 

242 for key in list(self.keys()): 

243 self.__delitem__(key) 

244 

245 def __repr__(self): 

246 return f"{self.__class__.__name__}({super()!r})" 

247 

248 def __delitem__(self, key): 

249 dict.__delitem__(self, key) 

250 object.__delattr__(self, key) 

251 

252 def __setattr__(self, key, value): 

253 # logger.info(f"called __setattr__({self!r}, {key}, {value})") 

254 if isinstance(value, dict) and not isinstance(value, NavigableDict): 254 ↛ 255line 254 didn't jump to line 255 because the condition on line 254 was never true

255 value = NavigableDict(value) 

256 self.__dict__[key] = value 

257 super().__setitem__(key, value) 

258 try: 

259 del self.__dict__["_memoized"][key] 

260 except KeyError: 

261 pass 

262 

263 # This method is called: 

264 # - for *every* single attribute access on an object using dot notation. 

265 # - when using the `getattr(obj, 'name') function 

266 # - accessing any kind of attributes, e.g. instance or class variables, 

267 # methods, properties, dunder methods, ... 

268 # 

269 # Note: `__getattr__` is only called when an attribute cannot be found 

270 # through normal means. 

271 def __getattribute__(self, key): 

272 # logger.info(f"called __getattribute__({key})") 

273 value = object.__getattribute__(self, key) 

274 if key.startswith('__'): # small optimization 

275 return value 

276 m = object.__getattribute__(self, "_handle_directive") 

277 return m(key, value) 

278 

279 def __delattr__(self, item): 

280 # logger.info(f"called __delattr__({self!r}, {item})") 

281 object.__delattr__(self, item) 

282 dict.__delitem__(self, item) 

283 

284 def __setitem__(self, key, value): 

285 # logger.info(f"called __setitem__({self!r}, {key}, {value})") 

286 if isinstance(value, dict) and not isinstance(value, NavigableDict): 

287 value = NavigableDict(value) 

288 super().__setitem__(key, value) 

289 self.__dict__[key] = value 

290 try: 

291 del self.__dict__["_memoized"][key] 

292 except KeyError: 

293 pass 

294 

295 # This method is called: 

296 # - whenever square brackets `[]` are used on an object, e.g. indexing or slicing. 

297 # - during iteration, if an object doesn't have __iter__ defined, Python will try 

298 # to iterate using __getitem__ with successive integer indices starting from 0. 

299 def __getitem__(self, key): 

300 # logger.info(f"called __getitem__({self!r}, {key})") 

301 value = super().__getitem__(key) 

302 if key.startswith('__'): 302 ↛ 303line 302 didn't jump to line 303 because the condition on line 302 was never true

303 return value 

304 return self._handle_directive(key, value) 

305 

306 def _handle_directive(self, key, value) -> Any: 

307 """ 

308 This method will handle the available directives. This may be builtin directives 

309 like `class/` or `yaml//`, or it may be external directives that were provided 

310 as a plugin. TO BE IMPLEMENTED 

311 

312 Args: 

313 key: the key of the field that might contain a directive 

314 value: the value which might be a directive 

315 

316 Returns: 

317 This function will return the value, either the original value or the result of 

318 evaluating and executing a directive. 

319 """ 

320 # logger.info(f"called _handle_directive({key}, {value!r})") 

321 if isinstance(value, str) and value.startswith("class//"): 

322 args, kwargs = self._get_args_and_kwargs(key) 

323 return _load_class(value)(*args, **kwargs) 

324 

325 elif isinstance(value, str) and value.startswith("factory//"): 325 ↛ 326line 325 didn't jump to line 326 because the condition on line 325 was never true

326 factory_args = _get_attribute(self, f"{key}_args", {}) 

327 return _load_class(value)().create(**factory_args) 

328 

329 elif isinstance(value, str) and value.startswith("yaml//"): 

330 if key in self.__dict__["_memoized"]: 330 ↛ 331line 330 didn't jump to line 331 because the condition on line 330 was never true

331 return self.__dict__["_memoized"][key] 

332 content = _load_yaml(value) 

333 self.__dict__["_memoized"][key] = content 

334 return content 

335 

336 elif isinstance(value, str) and value.startswith("csv//"): 

337 if key in self.__dict__["_memoized"]: 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true

338 return self.__dict__["_memoized"][key] 

339 args, kwargs = self._get_args_and_kwargs(key) 

340 content = _load_csv(value, **kwargs) 

341 self.__dict__["_memoized"][key] = content 

342 return content 

343 

344 elif isinstance(value, str) and value.startswith("int_enum//"): 

345 content = object.__getattribute__(self, "content") 

346 return _load_int_enum(value, content) 

347 

348 else: 

349 return value 

350 

351 def _get_args_and_kwargs(self, key): 

352 """ 

353 Read the args and kwargs that are associated with the key of a directive. 

354 

355 An example of such a directive: 

356 

357 hexapod: 

358 device: class//egse.hexapod.PunaProxy 

359 device_args: [PUNA_01] 

360 device_kwargs: 

361 sim: true 

362 

363 There might not be any positional nor keyword arguments provided in which 

364 case and empty tuple and/or dictionary is returned. 

365 

366 Returns: 

367 A tuple containing any positional arguments and a dictionary containing 

368 keyword arguments. 

369 """ 

370 try: 

371 args = object.__getattribute__(self, f"{key}_args") 

372 except AttributeError: 

373 args = () 

374 try: 

375 kwargs = object.__getattribute__(self, f"{key}_kwargs") 

376 except AttributeError: 

377 kwargs = {} 

378 

379 return args, kwargs 

380 

381 def set_private_attribute(self, key: str, value: Any) -> None: 

382 """Sets a private attribute for this object. 

383 

384 The name in key will be accessible as an attribute for this object, but the key will not 

385 be added to the dictionary and not be returned by methods like keys(). 

386 

387 The idea behind this private attribute is to have the possibility to add status information 

388 or identifiers to this classes object that can be used by save() or load() methods. 

389 

390 Args: 

391 key (str): the name of the private attribute (must start with an underscore character). 

392 value: the value for this private attribute 

393 

394 Examples: 

395 >>> setup = NavigableDict({'a': 1, 'b': 2, 'c': 3}) 

396 >>> setup.set_private_attribute("_loaded_from_dict", True) 

397 >>> assert "c" in setup 

398 >>> assert "_loaded_from_dict" not in setup 

399 >>> assert setup.get_private_attribute("_loaded_from_dict") == True 

400 

401 """ 

402 if key in self: 402 ↛ 403line 402 didn't jump to line 403 because the condition on line 402 was never true

403 raise ValueError(f"Invalid argument key='{key}', this key already exists in the dictionary.") 

404 if not key.startswith("_"): 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true

405 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

406 self.__dict__[key] = value 

407 

408 def get_private_attribute(self, key: str) -> Any: 

409 """Returns the value of the given private attribute. 

410 

411 Args: 

412 key (str): the name of the private attribute (must start with an underscore character). 

413 

414 Returns: 

415 the value of the private attribute given in `key`. 

416 

417 Note: 

418 Because of the implementation, this private attribute can also be accessed as a 'normal' 

419 attribute of the object. This use is however discouraged as it will make your code less 

420 understandable. Use the methods to access these 'private' attributes. 

421 """ 

422 if not key.startswith("_"): 

423 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

424 return self.__dict__[key] 

425 

426 def has_private_attribute(self, key): 

427 """ 

428 Check if the given key is defined as a private attribute. 

429 

430 Args: 

431 key (str): the name of a private attribute (must start with an underscore) 

432 Returns: 

433 True if the given key is a known private attribute. 

434 Raises: 

435 ValueError: when the key doesn't start with an underscore. 

436 """ 

437 if not key.startswith("_"): 

438 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

439 

440 try: 

441 _ = self.__dict__[key] 

442 return True 

443 except KeyError: 

444 return False 

445 

446 def get_raw_value(self, key): 

447 """ 

448 Returns the raw value of the given key. 

449 

450 Some keys have special values that are interpreted by the NavigableDict class. An example is 

451 a value that starts with 'class//'. When you access these values, they are first converted 

452 from their raw value into their expected value, e.g. the instantiated object in the above 

453 example. This method allows you to access the raw value before conversion. 

454 """ 

455 try: 

456 return object.__getattribute__(self, key) 

457 except AttributeError: 

458 raise KeyError(f"The key '{key}' is not defined.") 

459 

460 def __str__(self): 

461 return self._pretty_str() 

462 

463 def _pretty_str(self, indent: int = 0): 

464 msg = "" 

465 

466 for k, v in self.items(): 

467 if isinstance(v, NavigableDict): 

468 msg += f"{' ' * indent}{k}:\n" 

469 msg += v._pretty_str(indent + 1) 

470 else: 

471 msg += f"{' ' * indent}{k}: {v}\n" 

472 

473 return msg 

474 

475 def __rich__(self) -> Tree: 

476 tree = Tree(self.__dict__["_label"] or "NavigableDict", guide_style="dim") 

477 _walk_dict_tree(self, tree, text_style="dark grey") 

478 return tree 

479 

480 def _save(self, fd, indent: int = 0): 

481 """ 

482 Recursive method to write the dictionary to the file descriptor. 

483 

484 Indentation is done in steps of four spaces, i.e. `' '*indent`. 

485 

486 Args: 

487 fd: a file descriptor as returned by the open() function 

488 indent (int): indentation level of each line [default = 0] 

489 

490 """ 

491 

492 # Note that the .items() method returns the actual values of the keys and doesn't use the 

493 # __getattribute__ or __getitem__ methods. So the raw value is returned and not the 

494 # _processed_ value. 

495 

496 for k, v in self.items(): 

497 # history shall be saved last, skip it for now 

498 

499 if k == "history": 499 ↛ 500line 499 didn't jump to line 500 because the condition on line 499 was never true

500 continue 

501 

502 # make sure to escape a colon in the key name 

503 

504 if isinstance(k, str) and ":" in k: 504 ↛ 505line 504 didn't jump to line 505 because the condition on line 504 was never true

505 k = '"' + k + '"' 

506 

507 if isinstance(v, NavigableDict): 

508 fd.write(f"{' ' * indent}{k}:\n") 

509 v._save(fd, indent + 1) 

510 fd.flush() 

511 continue 

512 

513 if isinstance(v, float): 513 ↛ 514line 513 didn't jump to line 514 because the condition on line 513 was never true

514 v = f"{v:.6E}" 

515 fd.write(f"{' ' * indent}{k}: {v}\n") 

516 fd.flush() 

517 

518 # now save the history as the last item 

519 

520 if "history" in self: 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true

521 fd.write(f"{' ' * indent}history:\n") 

522 self.history._save(fd, indent + 1) # noqa 

523 

524 def get_memoized_keys(self): 

525 return list(self.__dict__["_memoized"].keys()) 

526 

527 @staticmethod 

528 def from_dict(my_dict: dict, label: str = None) -> NavigableDict: 

529 """Create a NavigableDict from a given dictionary. 

530 

531 Remember that all keys in the given dictionary shall be of type 'str' in order to be 

532 accessible as attributes. 

533 

534 Args: 

535 my_dict: a Python dictionary 

536 label: a label that will be attached to this navdict 

537 

538 Examples: 

539 >>> setup = navdict.from_dict({"ID": "my-setup-001", "version": "0.1.0"}, label="Setup") 

540 >>> assert setup["ID"] == setup.ID == "my-setup-001" 

541 

542 """ 

543 return NavigableDict(my_dict, label=label) 

544 

545 @staticmethod 

546 def from_yaml_string(yaml_content: str = None, label: str = None) -> NavigableDict: 

547 """Creates a NavigableDict from the given YAML string. 

548 

549 This method is mainly used for easy creation of a navdict from strings during unit tests. 

550 

551 Args: 

552 yaml_content: a string containing YAML 

553 label: a label that will be attached to this navdict 

554 

555 Returns: 

556 a navdict that was loaded from the content of the given string. 

557 """ 

558 

559 if not yaml_content: 

560 raise ValueError("Invalid argument to function: No input string or None given.") 

561 

562 yaml = YAML(typ='safe') 

563 try: 

564 data = yaml.load(yaml_content) 

565 except ScannerError as exc: 

566 raise ValueError(f"Invalid YAML string: {exc}") 

567 

568 return NavigableDict(data, label=label) 

569 

570 @staticmethod 

571 def from_yaml_file(filename: Union[str, Path] = None) -> NavigableDict: 

572 """Creates a navigable dictionary from the given YAML file. 

573 

574 Args: 

575 filename (str): the path of the YAML file to be loaded 

576 

577 Returns: 

578 a navdict that was loaded from the given location. 

579 

580 Raises: 

581 ValueError: when no filename is given. 

582 """ 

583 

584 if not filename: 584 ↛ 585line 584 didn't jump to line 585 because the condition on line 584 was never true

585 raise ValueError("Invalid argument to function: No filename or None given.") 

586 

587 data = _load_yaml(str(filename)) 

588 

589 if data == {}: 589 ↛ 590line 589 didn't jump to line 590 because the condition on line 589 was never true

590 warnings.warn(f"Empty YAML file: {filename!s}") 

591 

592 data.set_private_attribute("_filename", Path(filename)) 

593 

594 return data 

595 

596 def to_yaml_file(self, filename: str | Path = None) -> None: 

597 """Saves a NavigableDict to a YAML file. 

598 

599 When no filename is provided, this method will look for a 'private' attribute 

600 `_filename` and use that to save the data. 

601 

602 Args: 

603 filename (str|Path): the path of the YAML file where to save the data 

604 

605 Note: 

606 This method will **overwrite** the original or given YAML file and therefore you might 

607 lose proper formatting and/or comments. 

608 

609 """ 

610 if not filename: 610 ↛ 611line 610 didn't jump to line 611 because the condition on line 610 was never true

611 try: 

612 filename = self.get_private_attribute("_filename") 

613 except KeyError: 

614 raise ValueError("No filename given or known, can not save navdict.") 

615 

616 with Path(filename).open("w") as fd: 

617 fd.write( 

618 textwrap.dedent( 

619 f""" 

620 # This YAML file is generated by: 

621 # 

622 # Setup.to_yaml_file(setup, filename="{filename}') 

623 # 

624 # Created on {datetime.datetime.now(tz=datetime.timezone.utc).isoformat()} 

625  

626 """ 

627 ) 

628 ) 

629 

630 self._save(fd, indent=0) 

631 

632 self.set_private_attribute("_filename", Path(filename)) 

633 

634 def get_filename(self) -> str | None: 

635 """Returns the filename for this navdict or None when no filename could be determined.""" 

636 if self.has_private_attribute("_filename"): 

637 return self.get_private_attribute("_filename") 

638 else: 

639 return None 

640 

641 

642navdict = NavDict = NavigableDict # noqa: ignore typo 

643"""Shortcuts for NavigableDict and more Pythonic.""" 

644 

645 

646def _walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"): 

647 for k, v in dictionary.items(): 

648 if isinstance(v, dict): 

649 branch = tree.add(f"[purple]{k}", style="", guide_style="dim") 

650 _walk_dict_tree(v, branch, text_style=text_style) 

651 else: 

652 text = Text.assemble((str(k), "medium_purple1"), ": ", (str(v), text_style)) 

653 tree.add(text)