Coverage for src/extratools_core/jsontools.py: 54%
61 statements
« prev ^ index » next coverage.py v7.8.1, created at 2025-05-23 02:47 -0700
« prev ^ index » next coverage.py v7.8.1, created at 2025-05-23 02:47 -0700
1import json
2from csv import DictWriter
3from io import StringIO
4from pathlib import Path
5from typing import Any, TypedDict
7type JsonDict = dict[str, Any]
9type DictOfJsonDicts = dict[str, JsonDict]
10type ListOfJsonDicts = list[JsonDict]
13class DictOfJsonDictsDiffUpdate(TypedDict):
14 old: JsonDict
15 new: JsonDict
18class DictOfJsonDictsDiff(TypedDict):
19 deletes: dict[str, JsonDict]
20 inserts: dict[str, JsonDict]
21 updates: dict[str, DictOfJsonDictsDiffUpdate]
24class ListOfJsonDictsDiff(TypedDict):
25 deletes: list[JsonDict]
26 inserts: list[JsonDict]
29def flatten(data: Any) -> Any:
30 def flatten_rec(data: Any, path: str) -> None:
31 if isinstance(data, dict):
32 for k, v in data.items():
33 flatten_rec(v, path + (f".{k}" if path else k))
34 elif isinstance(data, list):
35 for i, v in enumerate(data):
36 flatten_rec(v, path + f"[{i}]")
37 else:
38 flatten_dict[path or "."] = data
40 flatten_dict: JsonDict = {}
41 flatten_rec(data, "")
42 return flatten_dict
45def json_to_csv(
46 data: DictOfJsonDicts | ListOfJsonDicts,
47 /,
48 csv_path: Path | str | None = None,
49 *,
50 key_field_name: str = "_key",
51) -> str:
52 if isinstance(data, dict):
53 data = [
54 {
55 # In case there is already a key field in each record,
56 # the new key field will be overwritten.
57 # It is okay though as the existing key field is likely
58 # serving the purpose of containing keys.
59 key_field_name: key,
60 **value,
61 }
62 for key, value in data.items()
63 ]
65 fields: set[str] = set()
66 for record in data:
67 fields.update(record.keys())
69 sio = StringIO()
71 writer = DictWriter(sio, fieldnames=fields)
72 writer.writeheader()
73 writer.writerows(data)
75 csv_str: str = sio.getvalue()
77 if csv_path:
78 Path(csv_path).write_text(csv_str)
80 return csv_str
83def dict_of_json_dicts_diff(
84 old: DictOfJsonDicts,
85 new: DictOfJsonDicts,
86) -> DictOfJsonDictsDiff:
87 inserts: dict[str, JsonDict] = {}
88 updates: dict[str, DictOfJsonDictsDiffUpdate] = {}
90 for new_key, new_value in new.items():
91 old_value: dict[str, Any] | None = old.get(new_key, None)
92 if old_value is None:
93 inserts[new_key] = new_value
94 elif json.dumps(old_value) != json.dumps(new_value):
95 updates[new_key] = {
96 "old": old_value,
97 "new": new_value,
98 }
100 deletes: dict[str, JsonDict] = {
101 old_key: old_value
102 for old_key, old_value in old.items()
103 if old_key not in new
104 }
106 return {
107 "deletes": deletes,
108 "inserts": inserts,
109 "updates": updates,
110 }
113def list_of_json_dicts_diff(
114 old: ListOfJsonDicts,
115 new: ListOfJsonDicts,
116) -> ListOfJsonDictsDiff:
117 old_dict: DictOfJsonDicts = {
118 json.dumps(d): d
119 for d in old
120 }
121 new_dict: DictOfJsonDicts = {
122 json.dumps(d): d
123 for d in new
124 }
126 inserts: list[JsonDict] = [
127 new_value
128 for new_key, new_value in new_dict.items()
129 if new_key not in old_dict
130 ]
131 deletes: list[JsonDict] = [
132 old_value
133 for old_key, old_value in old_dict.items()
134 if old_key not in new_dict
135 ]
137 return {
138 "deletes": deletes,
139 "inserts": inserts,
140 }