Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from typing import List, Dict 

2from jsanctions.helpers import dict_filter_attributes 

3from jutil.parse import parse_datetime 

4from jutil.xml import xml_to_dict 

5import logging 

6import os 

7from typing import Any 

8from django.db import transaction 

9from django.utils.timezone import now 

10from jutil.admin import admin_log 

11from jsanctions.models import ( 

12 SanctionsListFile, 

13 SanctionEntity, 

14 RegulationSummary, 

15 BirthDate, 

16 Identification, 

17 Remark, 

18 Address, 

19 Citizenship, 

20 NameAlias, 

21 Regulation, 

22 SubjectType, 

23) 

24from jutil.format import camel_case_to_underscore 

25 

26logger = logging.getLogger(__name__) 

27 

28EU_LIST_TYPE = "EU" 

29 

30EU_XML_ARRAY_TAGS = [ 

31 "sanctionEntity", 

32 "nameAlias", 

33 "identification", 

34 "address", 

35 "birthdate", 

36 "citizenship", 

37 "regulation", 

38 "remark", 

39] 

40 

41EU_XML_INT_TAGS: List[str] = [] 

42 

43EU_XML_DATE_ATTRIBUTES = [ 

44 "@birthdate", 

45] 

46 

47 

48def eu_sanction_list_xml_attr_filter(k: str, v: Any) -> Any: 

49 if k.endswith("Date") or k in EU_XML_DATE_ATTRIBUTES: 

50 return parse_datetime(v).date() 

51 if k == "@logicalId": 

52 return int(v) 

53 if v == "false": 

54 return False 

55 if v == "true": 

56 return True 

57 return v 

58 

59 

60def load_eu_sanction_list_as_dict(filename: str) -> Dict[str, Any]: 

61 with open(filename, "rb") as fp: 

62 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=EU_XML_ARRAY_TAGS, int_tags=EU_XML_INT_TAGS) 

63 data = dict_filter_attributes(data, eu_sanction_list_xml_attr_filter) 

64 return data 

65 

66 

67def set_eu_object_attr(obj, k: str, v, max_length: int = 512): 

68 if v and isinstance(v, str) and len(v) > max_length: 

69 logger.warning("'%s' truncated to [%s]: '%s...'", k, max_length, v[:64]) 

70 v = v[: max_length - 3] + "..." 

71 setattr(obj, k, v) 

72 

73 

74def set_eu_members( # noqa 

75 obj: Any, data: Dict[str, Any], verbose: bool = False, padding: int = 0, **kwargs 

76): 

77 class_map = { 

78 "regulationSummary": RegulationSummary, 

79 "subjectType": SubjectType, 

80 } 

81 array_class_map = { 

82 "birthdate": BirthDate, 

83 "identification": Identification, 

84 "address": Address, 

85 "citizenship": Citizenship, 

86 "nameAlias": NameAlias, 

87 "regulation": Regulation, 

88 } 

89 

90 padding_str = " " * padding 

91 obj.save() 

92 obj2: Any 

93 for k0, v0 in data.items(): 

94 if k0[0] == "@": 

95 k = camel_case_to_underscore(k0[1:]) 

96 if k == "birthdate": 

97 k = "birth_date" # special case because class name same as attribute 

98 if hasattr(obj, k): 

99 set_eu_object_attr(obj, k, v0) 

100 if verbose: 

101 logger.info("%s%s: %s = %s", padding_str, obj, k, v0) 

102 elif k0 in class_map: 

103 k = camel_case_to_underscore(k0) 

104 if hasattr(obj, k): 

105 if k == "subject_type": 

106 obj2 = SubjectType.objects.get_or_create( 

107 code=v0.get("@code", ""), classification_code=v0.get("@classificationCode", "") 

108 )[0] 

109 elif k == "regulation_summary": 

110 obj2 = RegulationSummary.objects.get_or_create( 

111 regulation_type=v0.get("@regulationType", ""), 

112 publication_date=v0.get("@publicationDate", None), 

113 publication_url=v0.get("@publicationUrl", ""), 

114 number_title=v0.get("@numberTitle", ""), 

115 )[0] 

116 else: 

117 obj2 = class_map[k0]() 

118 kwargs2 = {} 

119 kwargs2[k] = obj2 

120 for k2, v2 in kwargs.items(): 

121 set_eu_object_attr(obj2, k2, v2) 

122 kwargs2[k2] = v2 

123 

124 set_eu_members(obj2, v0, verbose=verbose, padding=padding + 4, **kwargs2) 

125 

126 set_eu_object_attr(obj, k, obj2) 

127 elif k0 in array_class_map: 

128 k = camel_case_to_underscore(k0) 

129 cls = array_class_map[k0] 

130 for v0_data in v0: 

131 obj2 = cls() 

132 kwargs2 = {} 

133 kwargs2[k] = obj2 

134 for k2, v2 in kwargs.items(): 

135 set_eu_object_attr(obj2, k2, v2) 

136 kwargs2[k2] = v2 

137 obj2.clean() 

138 obj2.save() 

139 set_eu_members(obj2, v0_data, verbose=verbose, padding=padding + 4, **kwargs2) 

140 elif k0 == "remark": 

141 for v0_str in v0: 

142 Remark.objects.create(text=v0_str, container=obj) 

143 

144 obj.clean() 

145 obj.save() 

146 if verbose: 

147 logger.info("%sSaved %s", padding * ' ', obj) 

148 

149 

150def import_eu_sanctions(source: SanctionsListFile, verbose: bool = False): 

151 data = load_eu_sanction_list_as_dict(source.full_path) 

152 set_eu_members(source, data, verbose=verbose) 

153 entities_list = data.get("sanctionEntity", []) 

154 logger.info("Importing %s sanction entities from %s", len(entities_list), os.path.basename(source.file.name)) 

155 t0 = now() 

156 for se_data in entities_list: 

157 assert isinstance(se_data, dict) 

158 if verbose: 

159 logger.info(" sanctionEntity") 

160 with transaction.atomic(): 

161 se = SanctionEntity.objects.create(source=source, data=se_data) 

162 set_eu_members(se, se_data, verbose=verbose, padding=4, sanction=se) 

163 source.imported = now() 

164 source.save() 

165 msg = "Imported {} sanction entities from {} in {}".format( 

166 len(entities_list), source.full_path, source.imported - t0 

167 ) 

168 logger.info(msg) 

169 admin_log([source], msg)