Coverage for /Users/rik/github/navdict/src/navdict/navdict.py: 36%

267 statements  

« prev     ^ index     » next       coverage.py v7.8.2, created at 2025-06-06 12:04 +0200

1""" 

2NavDict: A navigable dictionary with dot notation access and automatic file loading. 

3 

4NavDict extends Python's built-in dictionary to support convenient dot notation 

5access (data.user.name) alongside traditional key access (data["user"]["name"]). 

6It automatically loads data files and can instantiate classes dynamically based 

7on configuration. 

8 

9Features: 

10 - Dot notation access for nested data structures 

11 - Automatic file loading (CSV, YAML, JSON, etc.) 

12 - Dynamic class instantiation from configuration 

13 - Full backward compatibility with standard dictionaries 

14 

15Example: 

16 >>> from navdict import navdict 

17 >>> data = navdict({"user": {"name": "Alice", "config_file": "yaml//settings.yaml"}}) 

18 >>> data.user.name # "Alice" 

19 >>> data.user.config_file # Automatically loads and parses settings.yaml 

20 >>> data["user"]["name"] # Still works with traditional access 

21 

22Author: Rik Huygen 

23License: MIT 

24""" 

25 

26from __future__ import annotations 

27 

28__all__ = [ 

29 "navdict", # noqa: ignore typo 

30 "NavDict", 

31] 

32 

33import csv 

34import datetime 

35import enum 

36import importlib 

37import logging 

38import textwrap 

39import warnings 

40from enum import Enum 

41from pathlib import Path 

42from typing import Any 

43from typing import Type 

44from typing import Union 

45 

46from _ruamel_yaml import ScannerError 

47from rich.text import Text 

48from rich.tree import Tree 

49from ruamel.yaml import YAML 

50 

51logger = logging.getLogger("navdict") 

52 

53 

54def _load_class(class_name: str): 

55 """ 

56 Find and returns a class based on the fully qualified name. 

57 

58 A class name can be preceded with the string `class//` or `factory//`. This is used in YAML 

59 files where the class is then instantiated on load. 

60 

61 Args: 

62 class_name (str): a fully qualified name for the class 

63 """ 

64 if class_name.startswith("class//"): 

65 class_name = class_name[7:] 

66 elif class_name.startswith("factory//"): 

67 class_name = class_name[9:] 

68 

69 module_name, class_name = class_name.rsplit(".", 1) 

70 module = importlib.import_module(module_name) 

71 return getattr(module, class_name) 

72 

73 

74def _load_csv(resource_name: str): 

75 """Find and return the content of a CSV file.""" 

76 

77 if resource_name.startswith("csv//"): 77 ↛ 80line 77 didn't jump to line 80 because the condition on line 77 was always true

78 resource_name = resource_name[5:] 

79 

80 parts = resource_name.rsplit("/", 1) 

81 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]] 

82 

83 try: 

84 csv_location = Path(in_dir or '.').expanduser() / fn 

85 with open(csv_location, 'r', encoding='utf-8') as file: 

86 csv_reader = csv.reader(file) 

87 data = list(csv_reader) 

88 except FileNotFoundError: 

89 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True) 

90 raise 

91 

92 return data 

93 

94 

95def _load_int_enum(enum_name: str, enum_content) -> Type[Enum]: 

96 """Dynamically build (and return) and IntEnum. 

97 

98 In the YAML file this will look like below. 

99 The IntEnum directive (where <name> is the class name): 

100 

101 enum: int_enum//<name> 

102 

103 The IntEnum content: 

104 

105 content: 

106 E: 

107 alias: ['E_SIDE', 'RIGHT_SIDE'] 

108 value: 1 

109 F: 

110 alias: ['F_SIDE', 'LEFT_SIDE'] 

111 value: 0 

112 

113 Args: 

114 - enum_name: Enumeration name (potentially prepended with "int_enum//"). 

115 - enum_content: Content of the enumeration, as read from the navdict field. 

116 """ 

117 if enum_name.startswith("int_enum//"): 

118 enum_name = enum_name[10:] 

119 

120 definition = {} 

121 for side_name, side_definition in enum_content.items(): 

122 if "alias" in side_definition: 

123 aliases = side_definition["alias"] 

124 else: 

125 aliases = [] 

126 value = side_definition["value"] 

127 

128 definition[side_name] = value 

129 

130 for alias in aliases: 

131 definition[alias] = value 

132 

133 return enum.IntEnum(enum_name, definition) 

134 

135 

136def _load_yaml(resource_name: str) -> NavigableDict: 

137 """Find and return the content of a YAML file.""" 

138 

139 if resource_name.startswith("yaml//"): 139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true

140 resource_name = resource_name[6:] 

141 

142 parts = resource_name.rsplit("/", 1) 

143 

144 in_dir, fn = parts if len(parts) > 1 else [None, parts[0]] 

145 

146 try: 

147 yaml_location = Path(in_dir or '.').expanduser() 

148 

149 yaml = YAML(typ='safe') 

150 with open(yaml_location / fn, 'r') as file: 

151 data = yaml.load(file) 

152 

153 except FileNotFoundError: 

154 logger.error(f"Couldn't load resource '{resource_name}', file not found", exc_info=True) 

155 raise 

156 except IsADirectoryError: 

157 logger.error(f"Couldn't load resource '{resource_name}', file seems to be a directory", exc_info=True) 

158 raise 

159 

160 return navdict(data) 

161 

162 

163def _get_attribute(self, name, default): 

164 try: 

165 attr = object.__getattribute__(self, name) 

166 except AttributeError: 

167 attr = default 

168 return attr 

169 

170 

171class NavigableDict(dict): 

172 """ 

173 A NavigableDict is a dictionary where all keys in the original dictionary are also accessible 

174 as attributes to the class instance. So, if the original dictionary (setup) has a key 

175 "site_id" which is accessible as `setup['site_id']`, it will also be accessible as 

176 `setup.site_id`. 

177 

178 Examples: 

179 >>> setup = NavigableDict({'site_id': 'KU Leuven', 'version': "0.1.0"}) 

180 >>> assert setup['site_id'] == setup.site_id 

181 >>> assert setup['version'] == setup.version 

182 

183 Note: 

184 We always want **all** keys to be accessible as attributes, or none. That means all 

185 keys of the original dictionary shall be of type `str`. 

186 

187 """ 

188 

189 def __init__(self, head: dict = None, label: str = None): 

190 """ 

191 Args: 

192 head (dict): the original dictionary 

193 label (str): a label or name that is used when printing the navdict 

194 """ 

195 

196 head = head or {} 

197 super().__init__(head) 

198 self.__dict__["_memoized"] = {} 

199 self.__dict__["_label"] = label 

200 

201 # By agreement, we only want the keys to be set as attributes if all keys are strings. 

202 # That way we enforce that always all keys are navigable, or none. 

203 

204 if any(True for k in head.keys() if not isinstance(k, str)): 204 ↛ 207line 204 didn't jump to line 207 because the condition on line 204 was never true

205 # invalid_keys = list(k for k in head.keys() if not isinstance(k, str)) 

206 # logger.warning(f"Dictionary will not be dot-navigable, not all keys are strings [{invalid_keys=}].") 

207 return 

208 

209 for key, value in head.items(): 

210 if isinstance(value, dict): 

211 setattr(self, key, NavigableDict(head.__getitem__(key))) 

212 else: 

213 setattr(self, key, head.__getitem__(key)) 

214 

215 @property 

216 def label(self) -> str | None: 

217 return self._label 

218 

219 def add(self, key: str, value: Any): 

220 """Set a value for the given key. 

221 

222 If the value is a dictionary, it will be converted into a NavigableDict and the keys 

223 will become available as attributes provided that all the keys are strings. 

224 

225 Args: 

226 key (str): the name of the key / attribute to access the value 

227 value (Any): the value to assign to the key 

228 """ 

229 if isinstance(value, dict) and not isinstance(value, NavigableDict): 

230 value = NavigableDict(value) 

231 setattr(self, key, value) 

232 

233 def clear(self) -> None: 

234 for key in list(self.keys()): 

235 self.__delitem__(key) 

236 

237 def __repr__(self): 

238 return f"{self.__class__.__name__}({super()!r})" 

239 

240 def __delitem__(self, key): 

241 dict.__delitem__(self, key) 

242 object.__delattr__(self, key) 

243 

244 def __setattr__(self, key, value): 

245 # logger.info(f"called __setattr__({self!r}, {key}, {value})") 

246 if isinstance(value, dict) and not isinstance(value, NavigableDict): 246 ↛ 247line 246 didn't jump to line 247 because the condition on line 246 was never true

247 value = NavigableDict(value) 

248 self.__dict__[key] = value 

249 super().__setitem__(key, value) 

250 try: 

251 del self.__dict__["_memoized"][key] 

252 except KeyError: 

253 pass 

254 

255 def __getattribute__(self, key): 

256 # logger.info(f"called __getattribute__({key})") 

257 value = object.__getattribute__(self, key) 

258 if isinstance(value, str) and value.startswith("class//"): 258 ↛ 259line 258 didn't jump to line 259 because the condition on line 258 was never true

259 try: 

260 dev_args = object.__getattribute__(self, f"{key}_args") 

261 except AttributeError: 

262 dev_args = () 

263 return _load_class(value)(*dev_args) 

264 elif isinstance(value, str) and value.startswith("factory//"): 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true

265 factory_args = _get_attribute(self, f"{key}_args", {}) 

266 return _load_class(value)().create(**factory_args) 

267 elif isinstance(value, str) and value.startswith("int_enum//"): 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true

268 content = object.__getattribute__(self, "content") 

269 return _load_int_enum(value, content) 

270 elif isinstance(value, str) and value.startswith("csv//"): 

271 if key in self.__dict__["_memoized"]: 271 ↛ 272line 271 didn't jump to line 272 because the condition on line 271 was never true

272 return self.__dict__["_memoized"][key] 

273 content = _load_csv(value) 

274 self.__dict__["_memoized"][key] = content 

275 return content 

276 elif isinstance(value, str) and value.startswith("yaml//"): 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true

277 if key in self.__dict__["_memoized"]: 

278 return self.__dict__["_memoized"][key] 

279 content = _load_yaml(value) 

280 self.__dict__["_memoized"][key] = content 

281 return content 

282 else: 

283 return value 

284 

285 def __delattr__(self, item): 

286 # logger.info(f"called __delattr__({self!r}, {item})") 

287 object.__delattr__(self, item) 

288 dict.__delitem__(self, item) 

289 

290 def __setitem__(self, key, value): 

291 # logger.info(f"called __setitem__({self!r}, {key}, {value})") 

292 if isinstance(value, dict) and not isinstance(value, NavigableDict): 

293 value = NavigableDict(value) 

294 super().__setitem__(key, value) 

295 self.__dict__[key] = value 

296 try: 

297 del self.__dict__["_memoized"][key] 

298 except KeyError: 

299 pass 

300 

301 def __getitem__(self, key): 

302 # logger.info(f"called __getitem__({self!r}, {key})") 

303 value = super().__getitem__(key) 

304 if isinstance(value, str) and value.startswith("class//"): 

305 try: 

306 dev_args = object.__getattribute__(self, "device_args") 

307 except AttributeError: 

308 dev_args = () 

309 return _load_class(value)(*dev_args) 

310 if isinstance(value, str) and value.startswith("csv//"): 

311 return _load_csv(value) 

312 if isinstance(value, str) and value.startswith("int_enum//"): 

313 content = object.__getattribute__(self, "content") 

314 return _load_int_enum(value, content) 

315 else: 

316 return value 

317 

318 def set_private_attribute(self, key: str, value: Any) -> None: 

319 """Sets a private attribute for this object. 

320 

321 The name in key will be accessible as an attribute for this object, but the key will not 

322 be added to the dictionary and not be returned by methods like keys(). 

323 

324 The idea behind this private attribute is to have the possibility to add status information 

325 or identifiers to this classes object that can be used by save() or load() methods. 

326 

327 Args: 

328 key (str): the name of the private attribute (must start with an underscore character). 

329 value: the value for this private attribute 

330 

331 Examples: 

332 >>> setup = NavigableDict({'a': 1, 'b': 2, 'c': 3}) 

333 >>> setup.set_private_attribute("_loaded_from_dict", True) 

334 >>> assert "c" in setup 

335 >>> assert "_loaded_from_dict" not in setup 

336 >>> assert setup.get_private_attribute("_loaded_from_dict") == True 

337 

338 """ 

339 if key in self: 339 ↛ 340line 339 didn't jump to line 340 because the condition on line 339 was never true

340 raise ValueError(f"Invalid argument key='{key}', this key already exists in the dictionary.") 

341 if not key.startswith("_"): 341 ↛ 342line 341 didn't jump to line 342 because the condition on line 341 was never true

342 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

343 self.__dict__[key] = value 

344 

345 def get_private_attribute(self, key: str) -> Any: 

346 """Returns the value of the given private attribute. 

347 

348 Args: 

349 key (str): the name of the private attribute (must start with an underscore character). 

350 

351 Returns: 

352 the value of the private attribute given in `key`. 

353 

354 Note: 

355 Because of the implementation, this private attribute can also be accessed as a 'normal' 

356 attribute of the object. This use is however discouraged as it will make your code less 

357 understandable. Use the methods to access these 'private' attributes. 

358 """ 

359 if not key.startswith("_"): 

360 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

361 return self.__dict__[key] 

362 

363 def has_private_attribute(self, key): 

364 """ 

365 Check if the given key is defined as a private attribute. 

366 

367 Args: 

368 key (str): the name of a private attribute (must start with an underscore) 

369 Returns: 

370 True if the given key is a known private attribute. 

371 Raises: 

372 ValueError: when the key doesn't start with an underscore. 

373 """ 

374 if not key.startswith("_"): 

375 raise ValueError(f"Invalid argument key='{key}', must start with underscore character '_'.") 

376 

377 try: 

378 _ = self.__dict__[key] 

379 return True 

380 except KeyError: 

381 return False 

382 

383 def get_raw_value(self, key): 

384 """ 

385 Returns the raw value of the given key. 

386 

387 Some keys have special values that are interpreted by the NavigableDict class. An example is 

388 a value that starts with 'class//'. When you access these values, they are first converted 

389 from their raw value into their expected value, e.g. the instantiated object in the above 

390 example. This method allows you to access the raw value before conversion. 

391 """ 

392 try: 

393 return object.__getattribute__(self, key) 

394 except AttributeError: 

395 raise KeyError(f"The key '{key}' is not defined.") 

396 

397 def __str__(self): 

398 return self._pretty_str() 

399 

400 def _pretty_str(self, indent: int = 0): 

401 msg = "" 

402 

403 for k, v in self.items(): 

404 if isinstance(v, NavigableDict): 

405 msg += f"{' ' * indent}{k}:\n" 

406 msg += v._pretty_str(indent + 1) 

407 else: 

408 msg += f"{' ' * indent}{k}: {v}\n" 

409 

410 return msg 

411 

412 def __rich__(self) -> Tree: 

413 tree = Tree(self.__dict__["_label"] or "NavigableDict", guide_style="dim") 

414 _walk_dict_tree(self, tree, text_style="dark grey") 

415 return tree 

416 

417 def _save(self, fd, indent: int = 0): 

418 """ 

419 Recursive method to write the dictionary to the file descriptor. 

420 

421 Indentation is done in steps of four spaces, i.e. `' '*indent`. 

422 

423 Args: 

424 fd: a file descriptor as returned by the open() function 

425 indent (int): indentation level of each line [default = 0] 

426 

427 """ 

428 

429 # Note that the .items() method returns the actual values of the keys and doesn't use the 

430 # __getattribute__ or __getitem__ methods. So the raw value is returned and not the 

431 # _processed_ value. 

432 

433 for k, v in self.items(): 

434 # history shall be saved last, skip it for now 

435 

436 if k == "history": 

437 continue 

438 

439 # make sure to escape a colon in the key name 

440 

441 if isinstance(k, str) and ":" in k: 

442 k = '"' + k + '"' 

443 

444 if isinstance(v, NavigableDict): 

445 fd.write(f"{' ' * indent}{k}:\n") 

446 v._save(fd, indent + 1) 

447 fd.flush() 

448 continue 

449 

450 if isinstance(v, float): 

451 v = f"{v:.6E}" 

452 fd.write(f"{' ' * indent}{k}: {v}\n") 

453 fd.flush() 

454 

455 # now save the history as the last item 

456 

457 if "history" in self: 

458 fd.write(f"{' ' * indent}history:\n") 

459 self.history._save(fd, indent + 1) # noqa 

460 

461 def get_memoized_keys(self): 

462 return list(self.__dict__["_memoized"].keys()) 

463 

464 @staticmethod 

465 def from_dict(my_dict: dict, label: str = None) -> NavigableDict: 

466 """Create a NavigableDict from a given dictionary. 

467 

468 Remember that all keys in the given dictionary shall be of type 'str' in order to be 

469 accessible as attributes. 

470 

471 Args: 

472 my_dict: a Python dictionary 

473 label: a label that will be attached to this navdict 

474 

475 Examples: 

476 >>> setup = navdict.from_dict({"ID": "my-setup-001", "version": "0.1.0"}, label="Setup") 

477 >>> assert setup["ID"] == setup.ID == "my-setup-001" 

478 

479 """ 

480 return NavigableDict(my_dict, label=label) 

481 

482 @staticmethod 

483 def from_yaml_string(yaml_content: str = None, label: str = None) -> NavigableDict: 

484 """Creates a NavigableDict from the given YAML string. 

485 

486 This method is mainly used for easy creation of a navdict from strings during unit tests. 

487 

488 Args: 

489 yaml_content: a string containing YAML 

490 label: a label that will be attached to this navdict 

491 

492 Returns: 

493 a navdict that was loaded from the content of the given string. 

494 """ 

495 

496 if not yaml_content: 

497 raise ValueError("Invalid argument to function: No input string or None given.") 

498 

499 yaml = YAML(typ='safe') 

500 try: 

501 data = yaml.load(yaml_content) 

502 except ScannerError as exc: 

503 raise ValueError(f"Invalid YAML string: {exc}") 

504 

505 return NavigableDict(data, label=label) 

506 

507 @staticmethod 

508 def from_yaml_file(filename: Union[str, Path] = None) -> NavigableDict: 

509 """Creates a navigable dictionary from the given YAML file. 

510 

511 Args: 

512 filename (str): the path of the YAML file to be loaded 

513 

514 Returns: 

515 a navdict that was loaded from the given location. 

516 

517 Raises: 

518 ValueError: when no filename is given. 

519 """ 

520 

521 if not filename: 521 ↛ 522line 521 didn't jump to line 522 because the condition on line 521 was never true

522 raise ValueError("Invalid argument to function: No filename or None given.") 

523 

524 data = _load_yaml(str(filename)) 

525 

526 if data == {}: 526 ↛ 527line 526 didn't jump to line 527 because the condition on line 526 was never true

527 warnings.warn(f"Empty YAML file: {filename!s}") 

528 

529 data.set_private_attribute("_filename", Path(filename)) 

530 

531 return data 

532 

533 def to_yaml_file(self, filename: str | Path = None) -> None: 

534 """Saves a NavigableDict to a YAML file. 

535 

536 When no filename is provided, this method will look for a 'private' attribute 

537 `_filename` and use that to save the data. 

538 

539 Args: 

540 filename (str|Path): the path of the YAML file where to save the data 

541 

542 Note: 

543 This method will **overwrite** the original or given YAML file and therefore you might 

544 lose proper formatting and/or comments. 

545 

546 """ 

547 if not filename: 

548 try: 

549 filename = self.get_private_attribute("_filename") 

550 except KeyError: 

551 raise ValueError("No filename given or known, can not save navdict.") 

552 

553 with Path(filename).open("w") as fd: 

554 fd.write( 

555 textwrap.dedent( 

556 f""" 

557 # This YAML file is generated by: 

558 # 

559 # Setup.to_yaml_file(setup, filename="{filename}') 

560 # 

561 # Created on {datetime.datetime.now(tz=datetime.timezone.utc).isoformat()} 

562  

563 """ 

564 ) 

565 ) 

566 

567 self._save(fd, indent=0) 

568 

569 self.set_private_attribute("_filename", Path(filename)) 

570 

571 def get_filename(self) -> str | None: 

572 """Returns the filename for this navdict or None when no filename could be determined.""" 

573 if self.has_private_attribute("_filename"): 

574 return self.get_private_attribute("_filename") 

575 else: 

576 return None 

577 

578 

579navdict = NavDict = NavigableDict # noqa: ignore typo 

580"""Shortcuts for NavigableDict and more Pythonic.""" 

581 

582 

583def _walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"): 

584 for k, v in dictionary.items(): 

585 if isinstance(v, dict): 

586 branch = tree.add(f"[purple]{k}", style="", guide_style="dim") 

587 _walk_dict_tree(v, branch, text_style=text_style) 

588 else: 

589 text = Text.assemble((str(k), "medium_purple1"), ": ", (str(v), text_style)) 

590 tree.add(text)