Coverage for jsanctions/eu.py : 74%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from typing import List, Dict
2from jsanctions.helpers import dict_filter_attributes
3from jutil.parse import parse_datetime
4from jutil.xml import xml_to_dict
5import logging
6import os
7from typing import Any
8from django.db import transaction
9from django.utils.timezone import now
10from jutil.admin import admin_log
11from jsanctions.models import (
12 SanctionsListFile,
13 SanctionEntity,
14 RegulationSummary,
15 BirthDate,
16 Identification,
17 Remark,
18 Address,
19 Citizenship,
20 NameAlias,
21 Regulation,
22 SubjectType,
23)
24from jutil.format import camel_case_to_underscore
26logger = logging.getLogger(__name__)
28EU_LIST_TYPE = "EU"
30EU_XML_ARRAY_TAGS = [
31 "sanctionEntity",
32 "nameAlias",
33 "identification",
34 "address",
35 "birthdate",
36 "citizenship",
37 "regulation",
38 "remark",
39]
41EU_XML_INT_TAGS: List[str] = []
43EU_XML_DATE_ATTRIBUTES = [
44 "@birthdate",
45]
48def eu_sanction_list_xml_attr_filter(k: str, v: Any) -> Any:
49 if k.endswith("Date") or k in EU_XML_DATE_ATTRIBUTES:
50 return parse_datetime(v).date()
51 if k == "@logicalId":
52 return int(v)
53 if v == "false":
54 return False
55 if v == "true":
56 return True
57 return v
60def load_eu_sanction_list_as_dict(filename: str) -> Dict[str, Any]:
61 with open(filename, "rb") as fp:
62 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=EU_XML_ARRAY_TAGS, int_tags=EU_XML_INT_TAGS)
63 data = dict_filter_attributes(data, eu_sanction_list_xml_attr_filter)
64 return data
67def set_eu_object_attr(obj, k: str, v, max_length: int = 512):
68 if v and isinstance(v, str) and len(v) > max_length:
69 logger.warning("'%s' truncated to [%s]: '%s...'", k, max_length, v[:64])
70 v = v[: max_length - 3] + "..."
71 setattr(obj, k, v)
74def set_eu_members( # noqa
75 obj: Any, data: Dict[str, Any], verbose: bool = False, padding: int = 0, **kwargs
76):
77 class_map = {
78 "regulationSummary": RegulationSummary,
79 "subjectType": SubjectType,
80 }
81 array_class_map = {
82 "birthdate": BirthDate,
83 "identification": Identification,
84 "address": Address,
85 "citizenship": Citizenship,
86 "nameAlias": NameAlias,
87 "regulation": Regulation,
88 }
90 padding_str = " " * padding
91 obj.save()
92 obj2: Any
93 for k0, v0 in data.items():
94 if k0[0] == "@":
95 k = camel_case_to_underscore(k0[1:])
96 if k == "birthdate":
97 k = "birth_date" # special case because class name same as attribute
98 if hasattr(obj, k):
99 set_eu_object_attr(obj, k, v0)
100 if verbose:
101 logger.info("%s%s: %s = %s", padding_str, obj, k, v0)
102 elif k0 in class_map:
103 k = camel_case_to_underscore(k0)
104 if hasattr(obj, k):
105 if k == "subject_type":
106 obj2 = SubjectType.objects.get_or_create(
107 code=v0.get("@code", ""), classification_code=v0.get("@classificationCode", "")
108 )[0]
109 elif k == "regulation_summary":
110 obj2 = RegulationSummary.objects.get_or_create(
111 regulation_type=v0.get("@regulationType", ""),
112 publication_date=v0.get("@publicationDate", None),
113 publication_url=v0.get("@publicationUrl", ""),
114 number_title=v0.get("@numberTitle", ""),
115 )[0]
116 else:
117 obj2 = class_map[k0]()
118 kwargs2 = {}
119 kwargs2[k] = obj2
120 for k2, v2 in kwargs.items():
121 set_eu_object_attr(obj2, k2, v2)
122 kwargs2[k2] = v2
124 set_eu_members(obj2, v0, verbose=verbose, padding=padding + 4, **kwargs2)
126 set_eu_object_attr(obj, k, obj2)
127 elif k0 in array_class_map:
128 k = camel_case_to_underscore(k0)
129 cls = array_class_map[k0]
130 for v0_data in v0:
131 obj2 = cls()
132 kwargs2 = {}
133 kwargs2[k] = obj2
134 for k2, v2 in kwargs.items():
135 set_eu_object_attr(obj2, k2, v2)
136 kwargs2[k2] = v2
137 obj2.clean()
138 obj2.save()
139 set_eu_members(obj2, v0_data, verbose=verbose, padding=padding + 4, **kwargs2)
140 elif k0 == "remark":
141 for v0_str in v0:
142 Remark.objects.create(text=v0_str, container=obj)
144 obj.clean()
145 obj.save()
146 if verbose:
147 logger.info("%sSaved %s", padding * ' ', obj)
150def import_eu_sanctions(source: SanctionsListFile, verbose: bool = False):
151 data = load_eu_sanction_list_as_dict(source.full_path)
152 set_eu_members(source, data, verbose=verbose)
153 entities_list = data.get("sanctionEntity", [])
154 logger.info("Importing %s sanction entities from %s", len(entities_list), os.path.basename(source.file.name))
155 t0 = now()
156 for se_data in entities_list:
157 assert isinstance(se_data, dict)
158 if verbose:
159 logger.info(" sanctionEntity")
160 with transaction.atomic():
161 se = SanctionEntity.objects.create(source=source, data=se_data)
162 set_eu_members(se, se_data, verbose=verbose, padding=4, sanction=se)
163 source.imported = now()
164 source.save()
165 msg = "Imported {} sanction entities from {} in {}".format(
166 len(entities_list), source.full_path, source.imported - t0
167 )
168 logger.info(msg)
169 admin_log([source], msg)