Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import logging 

2import re 

3from datetime import date 

4from time import strptime 

5from typing import Dict, Any, Tuple, Optional 

6from django.core.exceptions import ValidationError 

7from django.db import transaction 

8from django.utils import translation 

9from django.utils.timezone import now 

10from django.utils.translation import gettext as _ 

11from jutil.admin import admin_log 

12from jutil.format import choices_label 

13from jutil.xml import xml_to_dict 

14from jsanctions.models import SanctionsListFile, SanctionEntity, NameAlias, SubjectType, BirthDate, Address, \ 

15 Identification, Remark 

16 

17logger = logging.getLogger(__name__) 

18 

19OFAC_LIST_TYPE = "OFAC" 

20 

21OFAC_XML_ARRAY_TAGS = ['sdnEntry', 'program', 'aka', 'dateOfBirthItem', 'placeOfBirthItem', 'address', 'id'] 

22 

23 

24def load_ofac_sanction_list_as_dict(filename: str) -> Dict[str, Any]: 

25 with open(filename, "rb") as fp: 

26 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=OFAC_XML_ARRAY_TAGS) 

27 return data 

28 

29 

30def parse_ofac_date(v: str) -> date: 

31 st = strptime(v, '%m/%d/%Y') 

32 if not st: 

33 raise ValidationError(_('Invalid date value {}').format(v)) 

34 return date(st.tm_year, st.tm_mon, st.tm_mday) 

35 

36 

37def parse_ofac_dob(v: str) -> Tuple[Optional[int], Optional[int], Optional[int]]: 

38 if re.fullmatch(r'\d{4}', v): 

39 return int(v), None, None 

40 if re.fullmatch(r'\d{1,2}/\d{1,2}/\d{4}', v): 

41 st = strptime(v, '%m/%d/%Y') 

42 return st.tm_year, st.tm_mon, st.tm_mday 

43 if re.fullmatch(r'\d{1,2} \w{3} \d{4}', v): 

44 with translation.override('en_US'): 

45 st = strptime(v, '%d %b %Y') 

46 return st.tm_year, st.tm_mon, st.tm_mday 

47 return None, None, None 

48 

49 

50def get_opt_ofac_str(data: Dict[str, Any], key: str) -> str: 

51 return data.get(key, '') or '' 

52 

53 

54def get_ofac_subject_type(data: Dict[str, Any]) -> SubjectType: 

55 sdn_type = data["sdnType"] 

56 if sdn_type == 'Entity': 

57 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.ENTERPRISE) 

58 elif sdn_type == 'Individual': 

59 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.PERSON) 

60 elif sdn_type == 'Vessel': 

61 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.VESSEL) 

62 elif sdn_type == 'Aircraft': 

63 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.AIRCRAFT) 

64 else: 

65 logger.warning('Unknown sdnType: %s', sdn_type) 

66 obj, created = SubjectType.objects.get_or_create(classification_code=sdn_type, code=sdn_type) 

67 assert isinstance(obj, SubjectType) 

68 if created: 

69 obj.code = choices_label(SubjectType.CLASSIFICATION_CODES, obj.classification_code) 

70 obj.save(update_fields=['code']) 

71 return obj 

72 

73 

74def parse_ofac_uid(data: Dict[str, Any]) -> int: 

75 uid = data.get('uid') 

76 if uid is None: 

77 raise ValidationError(_('UID missing')) 

78 return int(uid) 

79 

80 

81def create_ofac_alias(se: SanctionEntity, **kwargs) -> NameAlias: 

82 first_name = kwargs.get('firstName') or '' 

83 last_name = kwargs.get('lastName') or '' 

84 uid = parse_ofac_uid(kwargs) 

85 whole_name = (first_name + ' ' + last_name).strip() 

86 alias = NameAlias(sanction=se, first_name=first_name, last_name=last_name, whole_name=whole_name, logical_id=uid) 

87 alias.full_clean() 

88 alias.save() 

89 return alias 

90 

91 

92def create_ofac_dob(se: SanctionEntity, **kwargs) -> BirthDate: 

93 dob = BirthDate(sanction=se) 

94 dob.logical_id = parse_ofac_uid(kwargs) 

95 dob.birth_date_description = kwargs.get("dateOfBirth") or '' 

96 year, month_of_year, day_of_month = parse_ofac_dob(dob.birth_date_description) 

97 dob.year, dob.month_of_year, dob.day_of_month = year, month_of_year, day_of_month 

98 if year and month_of_year and day_of_month: 

99 dob.birth_date = date(year, month_of_year, day_of_month) 

100 dob.full_clean() 

101 dob.save() 

102 return dob 

103 

104 

105def create_ofac_place_of_birth(se: SanctionEntity, **kwargs) -> BirthDate: 

106 dob = BirthDate.objects.all().filter(sanction=se, place='').order_by('id').first() 

107 if dob is None: 

108 dob = BirthDate(sanction=se, logical_id=parse_ofac_uid(kwargs)) 

109 dob.place = get_opt_ofac_str(kwargs, 'placeOfBirth') 

110 dob.full_clean() 

111 dob.save() 

112 return dob 

113 

114 

115def create_ofac_address(se: SanctionEntity, **kwargs) -> Address: 

116 address = Address(sanction=se) 

117 address.logical_id = parse_ofac_uid(kwargs) 

118 address.region = get_opt_ofac_str(kwargs, 'stateOrProvince') 

119 address.city = get_opt_ofac_str(kwargs, 'city') 

120 address.zip_code = get_opt_ofac_str(kwargs, 'postalCode') 

121 address.country_description = get_opt_ofac_str(kwargs, 'country') 

122 street = '' 

123 for n in range(1, 5): 

124 k = 'address{}'.format(n) 

125 if k in kwargs: 

126 street = street + '\n' + kwargs.get(k) 

127 else: 

128 break 

129 address.street = street.strip() 

130 address.full_clean() 

131 address.save() 

132 return address 

133 

134 

135def create_ofac_id(se: SanctionEntity, **kwargs) -> Identification: 

136 id_obj = Identification(sanction=se) 

137 id_obj.logical_id = parse_ofac_uid(kwargs) 

138 id_obj.number = kwargs.get('idNumber') or '' 

139 id_obj.identification_type_description = kwargs.get('idType') or '' 

140 id_obj.country_description = kwargs.get('idCountry') or '' 

141 id_obj.full_clean() 

142 id_obj.save() 

143 return id_obj 

144 

145 

146def set_ofac_members( # noqa 

147 se: SanctionEntity, data: Dict[str, Any], verbose: bool = False, padding: int = 0, 

148): 

149 # uid 

150 se.logical_id = parse_ofac_uid(data) 

151 

152 # firstName, lastName 

153 first_name, last_name = get_opt_ofac_str(data, 'firstName'), get_opt_ofac_str(data, 'lastName') 

154 if first_name or last_name: 

155 create_ofac_alias(se, **data) 

156 

157 # sdnType 

158 se.subject_type = get_ofac_subject_type(data) 

159 

160 # remarks 

161 remarks = data.get('remarks') or '' 

162 if remarks: 

163 remark_obj = Remark(container=se, text=remarks) 

164 remark_obj.full_clean() 

165 remark_obj.save() 

166 

167 # programList 

168 for program in data.get('programList', {}).get('program', []) or []: 

169 if program: 

170 remark_obj = Remark(container=se, text='program={}'.format(program)) 

171 remark_obj.full_clean() 

172 remark_obj.save() 

173 

174 # akaList 

175 for e_data in data.get('akaList', {}).get('aka', []) or []: 

176 create_ofac_alias(se, **e_data) 

177 

178 # dateOfBirthList 

179 for e_data in data.get('dateOfBirthList', {}).get('dateOfBirthItem', []) or []: 

180 create_ofac_dob(se, **e_data) 

181 

182 # placeOfBirthList 

183 for e_data in data.get('placeOfBirthList', {}).get('placeOfBirthItem', []) or []: 

184 create_ofac_place_of_birth(se, **e_data) 

185 

186 # addressList 

187 for e_data in data.get('addressList', {}).get('address', []) or []: 

188 create_ofac_address(se, **e_data) 

189 

190 # idList 

191 for e_data in data.get('idList', {}).get('id', []) or []: 

192 create_ofac_id(se, **e_data) 

193 

194 se.full_clean() 

195 se.save() 

196 if verbose: 

197 logger.info("%sSaved %s", padding * ' ', se) 

198 

199 

200def import_ofac_sanctions(source: SanctionsListFile, verbose: bool = False): 

201 data = load_ofac_sanction_list_as_dict(source.full_path) 

202 source.generation_date = parse_ofac_date(data['publshInformation']['Publish_Date']) 

203 

204 t0 = now() 

205 entities_list = data.get("sdnEntry", []) 

206 for se_data in entities_list: 

207 assert isinstance(se_data, dict) 

208 if verbose: 

209 logger.info(" sdnEntry uid %s", se_data.get('uid')) 

210 with transaction.atomic(): 

211 se = SanctionEntity.objects.create(source=source, data=se_data) 

212 set_ofac_members(se, se_data, verbose=verbose, padding=4) 

213 

214 source.imported = now() 

215 source.save() 

216 msg = "Imported {} sanction entities from {} in {}".format( 

217 len(entities_list), source.full_path, source.imported - t0 

218 ) 

219 logger.info(msg) 

220 admin_log([source], msg)