Coverage for src/navdict/navdict.py: 60%

279 statements  

« prev     ^ index     » next       coverage.py v7.8.2, created at 2025-06-06 13:57 +0200

1""" 

2NavDict: A navigable dictionary with dot notation access and automatic file loading. 

3 

4NavDict extends Python's built-in dictionary to support convenient dot notation 

5access (data.user.name) alongside traditional key access (data["user"]["name"]). 

6It automatically loads data files and can instantiate classes dynamically based 

7on configuration. 

8 

9Features: 

10 - Dot notation access for nested data structures 

11 - Automatic file loading (CSV, YAML, JSON, etc.) 

12 - Dynamic class instantiation from configuration 

13 - Full backward compatibility with standard dictionaries 

14 

15Example: 

16 >>> from navdict import navdict 

17 >>> data = navdict({"user": {"name": "Alice", "config_file": "yaml//settings.yaml"}}) 

18 >>> data.user.name # "Alice" 

19 >>> data.user.config_file # Automatically loads and parses settings.yaml 

20 >>> data["user"]["name"] # Still works with traditional access 

21 

22Author: Rik Huygen 

23License: MIT 

24""" 

25 

26from __future__ import annotations 

27 

28__all__ = [ 

29 "navdict", # noqa: ignore typo 

30 "NavDict", 

31] 

32 

33import csv 

34import datetime 

35import enum 

36import importlib 

37import logging 

38import textwrap 

39import warnings 

40from enum import Enum 

41from pathlib import Path 

42from typing import Any 

43from typing import Type 

44from typing import Union 

45 

46from _ruamel_yaml import ScannerError 

47from rich.text import Text 

48from rich.tree import Tree 

49from ruamel.yaml import YAML 

50 

51logger = logging.getLogger("navdict") 

52 

53 

54def _load_class(class_name: str): 

55 """ 

56 Find and returns a class based on the fully qualified name. 

57 

58 A class name can be preceded with the string `class//` or `factory//`. This is used in YAML 

59 files where the class is then instantiated on load. 

60 

61 Args: 

62 class_name (str): a fully qualified name for the class 

63 """ 

64 if class_name.startswith("class//"): 64 ↛ 66line 64 didn't jump to line 66 because the condition on line 64 was always true

65 class_name = class_name[7:] 

66 elif class_name.startswith("factory//"): 

67 class_name = class_name[9:] 

68 

69 module_name, class_name = class_name.rsplit(".", 1) 

70 module = importlib.import_module(module_name) 

71 return getattr(module, class_name) 

72 

73 

74def _load_csv(resource_name: str, **kwargs): 

75 """Find and return the content of a CSV file.""" 

76 

77 if resource_name.startswith("csv//"): 77 ↛ 80line 77 didn't jump to line 80 because the condition on line 77 was always true

78 resource_name = resource_name[5:] 

79 

80 parts = resource_name.rsplit("/", 1) 

81 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]] 

82 

83 try: 

84 n_header_rows = int(kwargs["header_rows"]) 

85 except KeyError: 

86 n_header_rows = 0 

87 

88 try: 

89 csv_location = Path(in_dir or '.').expanduser() / fn 

90 with open(csv_location, 'r', encoding='utf-8') as file: 

91 csv_reader = csv.reader(file) 

92 data = list(csv_reader) 

93 except FileNotFoundError: 

94 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True) 

95 raise 

96 

97 return data[:n_header_rows], data[n_header_rows:] 

98 

99 

100def _load_int_enum(enum_name: str, enum_content) -> Type[Enum]: 

101 """Dynamically build (and return) and IntEnum. 

102 

103 In the YAML file this will look like below. 

104 The IntEnum directive (where <name> is the class name): 

105 

106 enum: int_enum//<name> 

107 

108 The IntEnum content: 

109 

110 content: 

111 E: 

112 alias: ['E_SIDE', 'RIGHT_SIDE'] 

113 value: 1 

114 F: 

115 alias: ['F_SIDE', 'LEFT_SIDE'] 

116 value: 0 

117 

118 Args: 

119 - enum_name: Enumeration name (potentially prepended with "int_enum//"). 

120 - enum_content: Content of the enumeration, as read from the navdict field. 

121 """ 

122 if enum_name.startswith("int_enum//"): 122 ↛ 125line 122 didn't jump to line 125 because the condition on line 122 was always true

123 enum_name = enum_name[10:] 

124 

125 definition = {} 

126 for side_name, side_definition in enum_content.items(): 

127 if "alias" in side_definition: 127 ↛ 130line 127 didn't jump to line 130 because the condition on line 127 was always true

128 aliases = side_definition["alias"] 

129 else: 

130 aliases = [] 

131 value = side_definition["value"] 

132 

133 definition[side_name] = value 

134 

135 for alias in aliases: 

136 definition[alias] = value 

137 

138 return enum.IntEnum(enum_name, definition) 

139 

140 

141def _load_yaml(resource_name: str) -> NavigableDict: 

142 """Find and return the content of a YAML file.""" 

143 

144 if resource_name.startswith("yaml//"): 

145 resource_name = resource_name[6:] 

146 

147 parts = resource_name.rsplit("/", 1) 

148 

149 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]] 

150 

151 try: 

152 yaml_location = Path(in_dir or '.').expanduser() 

153 

154 yaml = YAML(typ='safe') 

155 with open(yaml_location / fn, 'r') as file: 

156 data = yaml.load(file) 

157 

158 except FileNotFoundError: 

159 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True) 

160 raise 

161 except IsADirectoryError: 

162 logger.error(f"Couldn't load resource '{resource_name}', file seems to be a directory", exc_info=True) 

163 raise 

164 

165 return navdict(data) 

166 

167 

168def _get_attribute(self, name, default): 

169 try: 

170 attr = object.__getattribute__(self, name) 

171 except AttributeError: 

172 attr = default 

173 return attr 

174 

175 

176class NavigableDict(dict): 

177 """ 

178 A NavigableDict is a dictionary where all keys in the original dictionary are also accessible 

179 as attributes to the class instance. So, if the original dictionary (setup) has a key 

180 "site_id" which is accessible as `setup['site_id']`, it will also be accessible as 

181 `setup.site_id`. 

182 

183 Examples: 

184 >>> setup = NavigableDict({'site_id': 'KU Leuven', 'version': "0.1.0"}) 

185 >>> assert setup['site_id'] == setup.site_id 

186 >>> assert setup['version'] == setup.version 

187 

188 Note: 

189 We always want **all** keys to be accessible as attributes, or none. That means all 

190 keys of the original dictionary shall be of type `str`. 

191 

192 """ 

193 

194 def __init__(self, head: dict = None, label: str = None): 

195 """ 

196 Args: 

197 head (dict): the original dictionary 

198 label (str): a label or name that is used when printing the navdict 

199 """ 

200 

201 head = head or {} 

202 super().__init__(head) 

203 self.__dict__["_memoized"] = {} 

204 self.__dict__["_label"] = label 

205 

206 # By agreement, we only want the keys to be set as attributes if all keys are strings. 

207 # That way we enforce that always all keys are navigable, or none. 

208 

209 if any(True for k in head.keys() if not isinstance(k, str)): 

210 # invalid_keys = list(k for k in head.keys() if not isinstance(k, str)) 

211 # logger.warning(f"Dictionary will not be dot-navigable, not all keys are strings [{invalid_keys=}].") 

212 return 

213 

214 for key, value in head.items(): 

215 if isinstance(value, dict): 

216 setattr(self, key, NavigableDict(head.__getitem__(key))) 

217 else: 

218 setattr(self, key, head.__getitem__(key)) 

219 

220 @property 

221 def label(self) -> str | None: 

222 return self._label 

223 

224 def add(self, key: str, value: Any): 

225 """Set a value for the given key. 

226 

227 If the value is a dictionary, it will be converted into a NavigableDict and the keys 

228 will become available as attributes provided that all the keys are strings. 

229 

230 Args: 

231 key (str): the name of the key / attribute to access the value 

232 value (Any): the value to assign to the key 

233 """ 

234 if isinstance(value, dict) and not isinstance(value, NavigableDict): 

235 value = NavigableDict(value) 

236 setattr(self, key, value) 

237 

238 def clear(self) -> None: 

239 for key in list(self.keys()): 

240 self.__delitem__(key) 

241 

242 def __repr__(self): 

243 return f"{self.__class__.__name__}({super()!r})" 

244 

245 def __delitem__(self, key): 

246 dict.__delitem__(self, key) 

247 object.__delattr__(self, key) 

248 

249 def __setattr__(self, key, value): 

250 # logger.info(f"called __setattr__({self!r}, {key}, {value})") 

251 if isinstance(value, dict) and not isinstance(value, NavigableDict): 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true

252 value = NavigableDict(value) 

253 self.__dict__[key] = value 

254 super().__setitem__(key, value) 

255 try: 

256 del self.__dict__["_memoized"][key] 

257 except KeyError: 

258 pass 

259 

260 def __getattribute__(self, key): 

261 # logger.info(f"called __getattribute__({key})") 

262 value = object.__getattribute__(self, key) 

263 if isinstance(value, str) and value.startswith("class//"): 

264 try: 

265 dev_args = object.__getattribute__(self, f"{key}_args") 

266 except AttributeError: 

267 dev_args = () 

268 return _load_class(value)(*dev_args) 

269 elif isinstance(value, str) and value.startswith("factory//"): 269 ↛ 270line 269 didn't jump to line 270 because the condition on line 269 was never true

270 factory_args = _get_attribute(self, f"{key}_args", {}) 

271 return _load_class(value)().create(**factory_args) 

272 elif isinstance(value, str) and value.startswith("int_enum//"): 

273 content = object.__getattribute__(self, "content") 

274 return _load_int_enum(value, content) 

275 elif isinstance(value, str) and value.startswith("csv//"): 

276 if key in self.__dict__["_memoized"]: 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true

277 return self.__dict__["_memoized"][key] 

278 try: 

279 kwargs = object.__getattribute__(self, "kwargs") 

280 except AttributeError: 

281 kwargs = {} 

282 

283 content = _load_csv(value, **kwargs) 

284 self.__dict__["_memoized"][key] = content 

285 return content 

286 elif isinstance(value, str) and value.startswith("yaml//"): 

287 if key in self.__dict__["_memoized"]: 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true

288 return self.__dict__["_memoized"][key] 

289 content = _load_yaml(value) 

290 self.__dict__["_memoized"][key] = content 

291 return content 

292 else: 

293 return value 

294 

295 def __delattr__(self, item): 

296 # logger.info(f"called __delattr__({self!r}, {item})") 

297 object.__delattr__(self, item) 

298 dict.__delitem__(self, item) 

299 

300 def __setitem__(self, key, value): 

301 # logger.info(f"called __setitem__({self!r}, {key}, {value})") 

302 if isinstance(value, dict) and not isinstance(value, NavigableDict): 

303 value = NavigableDict(value) 

304 super().__setitem__(key, value) 

305 self.__dict__[key] = value 

306 try: 

307 del self.__dict__["_memoized"][key] 

308 except KeyError: 

309 pass 

310 

311 def __getitem__(self, key): 

312 # logger.info(f"called __getitem__({self!r}, {key})") 

313 value = super().__getitem__(key) 

314 if isinstance(value, str) and value.startswith("class//"): 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true

315 try: 

316 dev_args = object.__getattribute__(self, "device_args") 

317 except AttributeError: 

318 dev_args = () 

319 return _load_class(value)(*dev_args) 

320 if isinstance(value, str) and value.startswith("csv//"): 320 ↛ 321line 320 didn't jump to line 321 because the condition on line 320 was never true

321 try: 

322 kwargs = object.__getattribute__(self, "kwargs") 

323 except AttributeError: 

324 kwargs = {} 

325 return _load_csv(value, **kwargs) 

326 if isinstance(value, str) and value.startswith("int_enum//"): 326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true

327 content = object.__getattribute__(self, "content") 

328 return _load_int_enum(value, content) 

329 else: 

330 return value 

331 

332 def set_private_attribute(self, key: str, value: Any) -> None: 

333 """Sets a private attribute for this object. 

334 

335 The name in key will be accessible as an attribute for this object, but the key will not 

336 be added to the dictionary and not be returned by methods like keys(). 

337 

338 The idea behind this private attribute is to have the possibility to add status information 

339 or identifiers to this classes object that can be used by save() or load() methods. 

340 

341 Args: 

342 key (str): the name of the private attribute (must start with an underscore character). 

343 value: the value for this private attribute 

344 

345 Examples: 

346 >>> setup = NavigableDict({'a': 1, 'b': 2, 'c': 3}) 

347 >>> setup.set_private_attribute("_loaded_from_dict", True) 

348 >>> assert "c" in setup 

349 >>> assert "_loaded_from_dict" not in setup 

350 >>> assert setup.get_private_attribute("_loaded_from_dict") == True 

351 

352 """ 

353 if key in self: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true

354 raise ValueError(f"Invalid argument key='{key}', this key already exists in the dictionary.") 

355 if not key.startswith("_"): 355 ↛ 356line 355 didn't jump to line 356 because the condition on line 355 was never true

356 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

357 self.__dict__[key] = value 

358 

359 def get_private_attribute(self, key: str) -> Any: 

360 """Returns the value of the given private attribute. 

361 

362 Args: 

363 key (str): the name of the private attribute (must start with an underscore character). 

364 

365 Returns: 

366 the value of the private attribute given in `key`. 

367 

368 Note: 

369 Because of the implementation, this private attribute can also be accessed as a 'normal' 

370 attribute of the object. This use is however discouraged as it will make your code less 

371 understandable. Use the methods to access these 'private' attributes. 

372 """ 

373 if not key.startswith("_"): 

374 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

375 return self.__dict__[key] 

376 

377 def has_private_attribute(self, key): 

378 """ 

379 Check if the given key is defined as a private attribute. 

380 

381 Args: 

382 key (str): the name of a private attribute (must start with an underscore) 

383 Returns: 

384 True if the given key is a known private attribute. 

385 Raises: 

386 ValueError: when the key doesn't start with an underscore. 

387 """ 

388 if not key.startswith("_"): 

389 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

390 

391 try: 

392 _ = self.__dict__[key] 

393 return True 

394 except KeyError: 

395 return False 

396 

397 def get_raw_value(self, key): 

398 """ 

399 Returns the raw value of the given key. 

400 

401 Some keys have special values that are interpreted by the NavigableDict class. An example is 

402 a value that starts with 'class//'. When you access these values, they are first converted 

403 from their raw value into their expected value, e.g. the instantiated object in the above 

404 example. This method allows you to access the raw value before conversion. 

405 """ 

406 try: 

407 return object.__getattribute__(self, key) 

408 except AttributeError: 

409 raise KeyError(f"The key '{key}' is not defined.") 

410 

411 def __str__(self): 

412 return self._pretty_str() 

413 

414 def _pretty_str(self, indent: int = 0): 

415 msg = "" 

416 

417 for k, v in self.items(): 

418 if isinstance(v, NavigableDict): 

419 msg += f"{' ' * indent}{k}:\n" 

420 msg += v._pretty_str(indent + 1) 

421 else: 

422 msg += f"{' ' * indent}{k}: {v}\n" 

423 

424 return msg 

425 

426 def __rich__(self) -> Tree: 

427 tree = Tree(self.__dict__["_label"] or "NavigableDict", guide_style="dim") 

428 _walk_dict_tree(self, tree, text_style="dark grey") 

429 return tree 

430 

431 def _save(self, fd, indent: int = 0): 

432 """ 

433 Recursive method to write the dictionary to the file descriptor. 

434 

435 Indentation is done in steps of four spaces, i.e. `' '*indent`. 

436 

437 Args: 

438 fd: a file descriptor as returned by the open() function 

439 indent (int): indentation level of each line [default = 0] 

440 

441 """ 

442 

443 # Note that the .items() method returns the actual values of the keys and doesn't use the 

444 # __getattribute__ or __getitem__ methods. So the raw value is returned and not the 

445 # _processed_ value. 

446 

447 for k, v in self.items(): 

448 # history shall be saved last, skip it for now 

449 

450 if k == "history": 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true

451 continue 

452 

453 # make sure to escape a colon in the key name 

454 

455 if isinstance(k, str) and ":" in k: 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true

456 k = '"' + k + '"' 

457 

458 if isinstance(v, NavigableDict): 

459 fd.write(f"{' ' * indent}{k}:\n") 

460 v._save(fd, indent + 1) 

461 fd.flush() 

462 continue 

463 

464 if isinstance(v, float): 464 ↛ 465line 464 didn't jump to line 465 because the condition on line 464 was never true

465 v = f"{v:.6E}" 

466 fd.write(f"{' ' * indent}{k}: {v}\n") 

467 fd.flush() 

468 

469 # now save the history as the last item 

470 

471 if "history" in self: 471 ↛ 472line 471 didn't jump to line 472 because the condition on line 471 was never true

472 fd.write(f"{' ' * indent}history:\n") 

473 self.history._save(fd, indent + 1) # noqa 

474 

475 def get_memoized_keys(self): 

476 return list(self.__dict__["_memoized"].keys()) 

477 

478 @staticmethod 

479 def from_dict(my_dict: dict, label: str = None) -> NavigableDict: 

480 """Create a NavigableDict from a given dictionary. 

481 

482 Remember that all keys in the given dictionary shall be of type 'str' in order to be 

483 accessible as attributes. 

484 

485 Args: 

486 my_dict: a Python dictionary 

487 label: a label that will be attached to this navdict 

488 

489 Examples: 

490 >>> setup = navdict.from_dict({"ID": "my-setup-001", "version": "0.1.0"}, label="Setup") 

491 >>> assert setup["ID"] == setup.ID == "my-setup-001" 

492 

493 """ 

494 return NavigableDict(my_dict, label=label) 

495 

496 @staticmethod 

497 def from_yaml_string(yaml_content: str = None, label: str = None) -> NavigableDict: 

498 """Creates a NavigableDict from the given YAML string. 

499 

500 This method is mainly used for easy creation of a navdict from strings during unit tests. 

501 

502 Args: 

503 yaml_content: a string containing YAML 

504 label: a label that will be attached to this navdict 

505 

506 Returns: 

507 a navdict that was loaded from the content of the given string. 

508 """ 

509 

510 if not yaml_content: 

511 raise ValueError("Invalid argument to function: No input string or None given.") 

512 

513 yaml = YAML(typ='safe') 

514 try: 

515 data = yaml.load(yaml_content) 

516 except ScannerError as exc: 

517 raise ValueError(f"Invalid YAML string: {exc}") 

518 

519 return NavigableDict(data, label=label) 

520 

521 @staticmethod 

522 def from_yaml_file(filename: Union[str, Path] = None) -> NavigableDict: 

523 """Creates a navigable dictionary from the given YAML file. 

524 

525 Args: 

526 filename (str): the path of the YAML file to be loaded 

527 

528 Returns: 

529 a navdict that was loaded from the given location. 

530 

531 Raises: 

532 ValueError: when no filename is given. 

533 """ 

534 

535 if not filename: 535 ↛ 536line 535 didn't jump to line 536 because the condition on line 535 was never true

536 raise ValueError("Invalid argument to function: No filename or None given.") 

537 

538 data = _load_yaml(str(filename)) 

539 

540 if data == {}: 540 ↛ 541line 540 didn't jump to line 541 because the condition on line 540 was never true

541 warnings.warn(f"Empty YAML file: {filename!s}") 

542 

543 data.set_private_attribute("_filename", Path(filename)) 

544 

545 return data 

546 

547 def to_yaml_file(self, filename: str | Path = None) -> None: 

548 """Saves a NavigableDict to a YAML file. 

549 

550 When no filename is provided, this method will look for a 'private' attribute 

551 `_filename` and use that to save the data. 

552 

553 Args: 

554 filename (str|Path): the path of the YAML file where to save the data 

555 

556 Note: 

557 This method will **overwrite** the original or given YAML file and therefore you might 

558 lose proper formatting and/or comments. 

559 

560 """ 

561 if not filename: 561 ↛ 562line 561 didn't jump to line 562 because the condition on line 561 was never true

562 try: 

563 filename = self.get_private_attribute("_filename") 

564 except KeyError: 

565 raise ValueError("No filename given or known, can not save navdict.") 

566 

567 with Path(filename).open("w") as fd: 

568 fd.write( 

569 textwrap.dedent( 

570 f""" 

571 # This YAML file is generated by: 

572 # 

573 # Setup.to_yaml_file(setup, filename="{filename}') 

574 # 

575 # Created on {datetime.datetime.now(tz=datetime.timezone.utc).isoformat()} 

576  

577 """ 

578 ) 

579 ) 

580 

581 self._save(fd, indent=0) 

582 

583 self.set_private_attribute("_filename", Path(filename)) 

584 

585 def get_filename(self) -> str | None: 

586 """Returns the filename for this navdict or None when no filename could be determined.""" 

587 if self.has_private_attribute("_filename"): 

588 return self.get_private_attribute("_filename") 

589 else: 

590 return None 

591 

592 

593navdict = NavDict = NavigableDict # noqa: ignore typo 

594"""Shortcuts for NavigableDict and more Pythonic.""" 

595 

596 

597def _walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"): 

598 for k, v in dictionary.items(): 

599 if isinstance(v, dict): 

600 branch = tree.add(f"[purple]{k}", style="", guide_style="dim") 

601 _walk_dict_tree(v, branch, text_style=text_style) 

602 else: 

603 text = Text.assemble((str(k), "medium_purple1"), ": ", (str(v), text_style)) 

604 tree.add(text)