Coverage for jsanctions/un.py : 96%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import logging
2from typing import Any, Dict, List, Optional
3from django.core.exceptions import ValidationError
4from django.db import transaction
5from django.utils.dateparse import parse_date
6from django.utils.timezone import now
7from django.utils.translation import gettext as _
8from jutil.admin import admin_log
9from jutil.format import choices_label
10from jutil.parse import parse_datetime
11from jutil.xml import xml_to_dict
13from jsanctions.helpers import get_country_iso2_code
14from jsanctions.models import SanctionsListFile, SanctionEntity, NameAlias, Remark, Address, Identification, \
15 SanctionListObject, SubjectType
17logger = logging.getLogger(__name__)
19UN_LIST_TYPE = "UN"
21UN_XML_ARRAY_TAGS = [
22 'VALUE',
23 'INDIVIDUAL',
24 'INDIVIDUAL_ALIAS',
25 'INDIVIDUAL_ADDRESS',
26 'INDIVIDUAL_DATE_OF_BIRTH',
27 'INDIVIDUAL_PLACE_OF_BIRTH',
28 'INDIVIDUAL_DOCUMENT',
29 'ENTITY',
30 'ENTITY_ALIAS',
31 'ENTITY_ADDRESS',
32]
34UN_NAME_FIELDS = ['FIRST_NAME', 'SECOND_NAME', 'THIRD_NAME', 'FOURTH_NAME', 'FIFTH_NAME', 'SIXTH_NAME']
37def load_un_sanction_list_as_dict(filename: str) -> Dict[str, Any]:
38 with open(filename, "rb") as fp:
39 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=UN_XML_ARRAY_TAGS)
40 return data
43def parse_un_data_id(data: Dict[str, Any]) -> int:
44 uid = data.get('DATAID')
45 if uid is None:
46 raise ValidationError(_('DATAID missing'))
47 return int(uid)
50def create_un_alias(se: SanctionEntity, **kwargs) -> Optional[NameAlias]:
51 names = []
52 for k in UN_NAME_FIELDS:
53 if k in kwargs and kwargs[k]:
54 names.append(kwargs[k])
55 if not names:
56 logger.warning('No names: %s', kwargs)
57 return None
59 alias = NameAlias(sanction=se, logical_id=parse_un_data_id(kwargs))
60 alias.title = kwargs.get('TITLE') or ''
61 alias.last_name = names.pop() or ''
62 alias.first_name = ' '.join(names).strip()
63 alias.full_clean()
64 alias.save()
65 return alias
68def create_un_comments(se: SanctionEntity, **kwargs) -> List[Remark]:
69 out: List[Remark] = []
70 for n in range(1, 10):
71 k = 'COMMENTS{}'.format(n)
72 if k in kwargs and kwargs[k]:
73 obj_out = Remark(container=se, text=kwargs.get(k) or '') # type: ignore
74 obj_out.full_clean()
75 obj_out.save()
76 out.append(obj_out)
77 else:
78 break
79 return out
82def create_un_note(obj: SanctionListObject, note: Any):
83 if note:
84 remark = Remark(container=obj, text=str(note))
85 remark.full_clean()
86 remark.save()
89def create_un_address(se: SanctionEntity, **kwargs) -> Address:
90 # {'STATE_PROVINCE', 'NOTE', 'COUNTRY', 'STREET', 'CITY', 'ZIP_CODE'}
91 address = Address(sanction=se)
92 address.region = kwargs.get('STATE_PROVINCE') or ''
93 address.city = kwargs.get('CITY') or ''
94 address.zip_code = kwargs.get('ZIP_CODE') or ''
95 address.country_description = kwargs.get('COUNTRY') or ''
96 address.street = kwargs.get('STREET') or ''
97 for k, v in kwargs.items():
98 if hasattr(address, k):
99 setattr(address, k, v)
100 address.full_clean()
101 address.save()
102 create_un_note(address, kwargs.get('NOTE'))
103 return address
106def create_un_document(se: SanctionEntity, **kwargs) -> Identification:
107 # {'DATE_OF_ISSUE', 'NUMBER', 'NOTE', 'ISSUING_COUNTRY', 'CITY_OF_ISSUE', 'COUNTRY_OF_ISSUE',
108 # 'TYPE_OF_DOCUMENT', 'TYPE_OF_DOCUMENT2'}
109 id_obj = Identification(sanction=se)
110 id_obj.identification_type_description = kwargs.get('TYPE_OF_DOCUMENT') or kwargs.get('TYPE_OF_DOCUMENT2') or ''
111 id_obj.issue_date = parse_date(str(kwargs.get('DATE_OF_ISSUE'))) if kwargs.get('DATE_OF_ISSUE') else None # type: ignore
112 id_obj.latin_number = kwargs.get('NUMBER') or ''
113 id_obj.issued_by = '{} {} {}'.format(kwargs.get('CITY_OF_ISSUE') or '', kwargs.get('COUNTRY_OF_ISSUE') or '', kwargs.get('ISSUING_COUNTRY') or '').strip()
114 id_obj.country_description = kwargs.get('COUNTRY_OF_ISSUE') or kwargs.get('ISSUING_COUNTRY') or ''
115 id_obj.full_clean()
116 id_obj.save()
117 create_un_note(id_obj, kwargs.get('NOTE'))
118 return id_obj
121def set_un_members( # noqa
122 se: SanctionEntity, data: Dict[str, Any], verbose: bool = False, padding: int = 0,
123):
124 # DATAID
125 se.logical_id = parse_un_data_id(data)
127 # FIRST_NAME, ...
128 create_un_alias(se, **data)
130 # COMMENTSx
131 create_un_comments(se, **data)
133 # INVIDUAL_ADDRESS / ENTITY_ADDRESS
134 address_list = data.get("INVIDUAL_ADDRESS", []) or data.get('ENTITY_ADDRESS', [])
135 addresses: List[Address] = []
136 if address_list:
137 for e_data in address_list:
138 if e_data:
139 addresses.append(create_un_address(se, **e_data))
141 # try to fill address information from UN list name
142 if not addresses:
143 un_list_type = data.get("UN_LIST_TYPE")
144 if un_list_type:
145 country_code = get_country_iso2_code(un_list_type)
146 if country_code:
147 create_un_address(se, country_description=un_list_type, country_code=country_code)
149 # INDIVIDUAL_DOCUMENT
150 docs = data.get('INDIVIDUAL_DOCUMENT')
151 if docs:
152 for e_data in docs:
153 if e_data:
154 create_un_document(se, **e_data)
156 se.full_clean()
157 se.save()
158 if verbose:
159 logger.info("%sSaved %s", padding * ' ', se)
162def import_un_sanctions(source: SanctionsListFile, verbose: bool = False):
163 data = load_un_sanction_list_as_dict(source.full_path)
164 source.generation_date = parse_datetime(data["@dateGenerated"]).date()
166 enterprise, created = SubjectType.objects.get_or_create(classification_code=SubjectType.ENTERPRISE)
167 assert isinstance(enterprise, SubjectType)
168 if created or not enterprise.code:
169 enterprise.code = choices_label(SubjectType.CLASSIFICATION_CODES, enterprise.classification_code)
170 enterprise.save()
171 person, created = SubjectType.objects.get_or_create(classification_code=SubjectType.PERSON)
172 assert isinstance(person, SubjectType)
173 if created or not person.code:
174 person.code = choices_label(SubjectType.CLASSIFICATION_CODES, person.classification_code)
175 person.save()
177 t0 = now()
178 individuals_list = data.get("INDIVIDUALS", {}).get('INDIVIDUAL')
179 for se_data in individuals_list:
180 assert isinstance(se_data, dict)
181 if verbose:
182 logger.info(" sdnEntry uid %s", se_data.get('uid'))
183 with transaction.atomic():
184 se = SanctionEntity.objects.create(source=source, data=se_data, subject_type=person)
185 set_un_members(se, se_data, verbose=verbose, padding=4)
187 entities_list = data.get("ENTITIES", {}).get('ENTITY')
188 for se_data in entities_list:
189 assert isinstance(se_data, dict)
190 if verbose:
191 logger.info(" sdnEntry uid %s", se_data.get('uid'))
192 with transaction.atomic():
193 se = SanctionEntity.objects.create(source=source, data=se_data, subject_type=enterprise)
194 set_un_members(se, se_data, verbose=verbose, padding=4)
196 source.imported = now()
197 source.save()
198 msg = "Imported {} sanction entities and {} individuals from {} in {}".format(
199 len(entities_list), len(individuals_list), source.full_path, source.imported - t0
200 )
201 logger.info(msg)
202 admin_log([source], msg)