Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from datetime import datetime, date 

2from decimal import Decimal 

3from typing import Tuple, Any, Optional, Dict 

4 

5from django.conf import settings 

6from django.core.exceptions import ValidationError 

7from django.db import transaction 

8from django.utils.dateparse import parse_date 

9from django.utils.translation import gettext as _ 

10from jacc.models import Account, EntryType 

11from jutil.format import dec2, dec4 

12from jutil.parse import parse_datetime 

13from jbank.models import StatementFile, Statement, StatementRecord, DELIVERY_FROM_BANK_SYSTEM, \ 

14 StatementRecordDetail, CurrencyExchange, StatementRecordRemittanceInfo, CurrencyExchangeSource 

15from jbank.parsers import parse_filename_suffix 

16from jutil.xml import xml_to_dict 

17 

18 

19CAMT053_STATEMENT_SUFFIXES = ('XML', 'XT', 'CAMT') 

20 

21CAMT053_ARRAY_TAGS = ['Bal', 'Ntry', 'NtryDtls', 'TxDtls', 'Strd'] 

22 

23CAMT053_INT_TAGS = ['NbOfNtries', 'NbOfTxs'] 

24 

25 

26def camt053_get_iban(data: dict) -> str: 

27 return data.get('BkToCstmrStmt', {}).get('Stmt', {}).get('Acct', {}).get('Id', {}).get('IBAN', '') 

28 

29 

30def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = '') -> Any: 

31 if key not in data: 

32 if required: 

33 raise ValidationError(_('camt.053 field {} missing').format(name if name else key)) 

34 return default 

35 return data[key] 

36 

37 

38def camt053_get_str(data: dict, key: str, default: str = '', required: bool = True, name: str = '') -> str: 

39 return str(camt053_get_val(data, key, default, required, name)) 

40 

41 

42def camt053_get_currency(data: dict, key: str, required: bool = True, 

43 name: str = '') -> Tuple[Optional[Decimal], str]: 

44 try: 

45 v = camt053_get_val(data, key, default=None, required=False, name=name) 

46 if v is not None: 

47 amount = dec2(v['@']) 

48 currency_code = v['@Ccy'] 

49 return amount, currency_code 

50 except Exception: 

51 pass 

52 if required: 

53 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'currency')) 

54 return None, '' 

55 

56 

57def camt053_get_dt(data: Dict[str, Any], key: str, name: str = '') -> datetime: 

58 s = camt053_get_val(data, key, None, True, name) 

59 val = parse_datetime(s) 

60 if val is None: 

61 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'datetime') + ': {}'.format(s)) 

62 return val 

63 

64 

65def camt053_get_int(data: Dict[str, Any], key: str, name: str = '') -> int: 

66 s = camt053_get_val(data, key, None, True, name) 

67 try: 

68 return int(s) 

69 except Exception: 

70 pass 

71 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'int')) 

72 

73 

74def camt053_get_date(data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = '') -> date: 

75 s = camt053_get_val(data, key, default, required, name) 

76 try: 

77 val = parse_date(s[:10]) 

78 if val is None: 

79 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'date')) 

80 assert isinstance(val, date) 

81 return val 

82 except Exception: 

83 pass 

84 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'date') + ': {}'.format(s)) 

85 

86 

87def camt053_parse_statement_from_file(filename: str) -> dict: 

88 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES: 

89 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

90 filename=filename, suffixes=', '.join(CAMT053_STATEMENT_SUFFIXES), file_type='camt.053')) 

91 with open(filename, 'rb') as fp: 

92 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS) 

93 return data 

94 

95 

96def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]: 

97 for bal in d_stmt.get('Bal', []): 

98 if bal.get('Tp', {}).get('CdOrPrtry', {}).get('Cd', '') == bal_type: 

99 amt = Decimal(bal.get('Amt', {}).get('@', '')) 

100 dt_data = bal.get('Dt', {}) 

101 dt = None 

102 if 'Dt' in dt_data: 

103 dt = camt053_get_date(dt_data, 'Dt', name='Stmt.Bal[{}].Dt.Dt'.format(bal_type)) 

104 return amt, dt 

105 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format('Stmt.Bal.Tp.CdOrPrty.Cd', bal_type)) 

106 

107 

108def camt053_domain_from_record_code(record_domain: str) -> str: 

109 if record_domain == 'PMNT': 

110 return '700' 

111 if record_domain == 'LDAS': 

112 return '761' 

113 return '' 

114 

115 

116def camt053_get_unified_val(qs, k: str, default: Any) -> Any: 

117 v = default 

118 for e in qs: 

119 v2 = getattr(e, k) 

120 if v == default: 

121 v = v2 

122 elif v and v2 and v2 != v: 

123 return default 

124 return v 

125 

126 

127def camt053_get_unified_str(qs, k: str) -> str: 

128 return camt053_get_unified_val(qs, k, '') 

129 

130 

131@transaction.atomic # noqa 

132def camt053_create_statement(statement_data: dict, # noqa 

133 name: str, file: StatementFile, **kw) -> Statement: 

134 """ 

135 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file() 

136 :param statement_data: XML data in form of dict 

137 :param name: File name of the account statement 

138 :param file: Source statement file 

139 :return: Statement 

140 """ 

141 account_number = camt053_get_iban(statement_data) 

142 if not account_number: 

143 raise ValidationError('{name}: '.format(name=name) + _("account.not.found").format(account_number='')) 

144 accounts = list(Account.objects.filter(name=account_number)) 

145 if len(accounts) != 1: 

146 raise ValidationError('{name}: '.format(name=name) + _("account.not.found").format(account_number=account_number)) 

147 account = accounts[0] 

148 assert isinstance(account, Account) 

149 

150 d_stmt = statement_data.get('BkToCstmrStmt', {}).get('Stmt', {}) 

151 d_acct = d_stmt.get('Acct', {}) 

152 d_ownr = d_acct.get('Ownr', {}) 

153 d_ntry = d_stmt.get('Ntry', []) 

154 d_frto = d_stmt.get('FrToDt', {}) 

155 d_txsummary = d_stmt.get('TxsSummry', {}) 

156 

157 if Statement.objects.filter(name=name, account=account).first(): 

158 raise ValidationError('Bank account {} statement {} of processed already'.format(account_number, name)) 

159 stm = Statement(name=name, account=account, file=file) 

160 stm.account_number = stm.iban = account_number 

161 stm.bic = camt053_get_str(d_acct.get('Svcr', {}).get('FinInstnId', {}), 'BIC', name='Stmt.Acct.Svcr.FinInstnId.BIC') 

162 stm.statement_identifier = camt053_get_str(d_stmt, 'Id', name='Stmt.Id') 

163 stm.statement_number = camt053_get_str(d_stmt, 'LglSeqNb', name='Stmt.LglSeqNb') 

164 stm.record_date = camt053_get_dt(d_stmt, 'CreDtTm', name='Stmt.CreDtTm') 

165 stm.begin_date = camt053_get_dt(d_frto, 'FrDtTm', name='Stmt.FrDtTm').date() 

166 stm.end_date = camt053_get_dt(d_frto, 'ToDtTm', name='Stmt.ToDtTm').date() 

167 stm.currency_code = camt053_get_str(d_acct, 'Ccy', name='Stmt.Acct.Ccy') 

168 if stm.currency_code != account.currency: 

169 raise ValidationError(_('Account currency {account_currency} does not match statement entry currency {statement_currency}'.format( 

170 statement_currency=stm.currency_code, account_currency=account.currency))) 

171 stm.owner_name = camt053_get_str(d_ownr, 'Nm', name='Stm.Acct.Ownr.Nm') 

172 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, 'OPBD') 

173 if stm.begin_balance_date is None: 

174 stm.begin_balance_date = stm.begin_date 

175 stm.record_count = camt053_get_int(d_txsummary.get('TtlNtries', {}), 'NbOfNtries', name='Stmt.TxsSummry.TtlNtries.NbOfNtries') 

176 stm.bank_specific_info_1 = camt053_get_str(d_stmt, 'AddtlStmtInf', required=False)[:1024] 

177 for k, v in kw.items(): 

178 setattr(stm, k, v) 

179 stm.full_clean() 

180 stm.save() 

181 

182 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first() 

183 if not e_deposit: 

184 raise ValidationError(_('entry.type.missing') + ' ({}): {}'.format('settings.E_BANK_DEPOSIT', settings.E_BANK_DEPOSIT)) 

185 assert isinstance(e_deposit, EntryType) 

186 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first() 

187 if not e_withdraw: 

188 raise ValidationError(_('entry.type.missing') + ' ({}): {}'.format('settings.E_BANK_WITHDRAW', settings.E_BANK_WITHDRAW)) 

189 assert isinstance(e_withdraw, EntryType) 

190 e_types = { 

191 'CRDT': e_deposit, 

192 'DBIT': e_withdraw, 

193 } 

194 record_type_map = { 

195 'CRDT': '1', 

196 'DBIT': '2', 

197 } 

198 

199 for ntry in d_ntry: 

200 archive_id = ntry.get('AcctSvcrRef', '') 

201 amount, cur = camt053_get_currency(ntry, 'Amt', name='Stmt.Ntry[{}].Amt'.format(archive_id)) 

202 if cur != account.currency: 

203 raise ValidationError(_('Account currency {account_currency} does not match statement entry currency {statement_currency}'.format( 

204 statement_currency=cur, account_currency=account.currency))) 

205 

206 cdt_dbt_ind = ntry['CdtDbtInd'] 

207 e_type = e_types.get(cdt_dbt_ind, None) 

208 if not e_type: 

209 raise ValidationError(_('Statement entry type {} not supported').format(cdt_dbt_ind)) 

210 

211 rec = StatementRecord(statement=stm, account=account, type=e_type) 

212 rec.amount = amount 

213 rec.archive_identifier = archive_id 

214 rec.entry_type = record_type_map[cdt_dbt_ind] 

215 rec.record_date = record_date = camt053_get_date(ntry.get('BookgDt', {}), 'Dt', name='Stmt.Ntry[{}].BkkgDt.Dt'.format(archive_id)) 

216 rec.value_date = camt053_get_date(ntry.get('ValDt', {}), 'Dt', name='Stmt.Ntry[{}].ValDt.Dt'.format(archive_id)) 

217 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM 

218 

219 d_bktxcd = ntry.get('BkTxCd', {}) 

220 d_domn = d_bktxcd.get('Domn', {}) 

221 d_family = d_domn.get('Fmly', {}) 

222 d_prtry = d_bktxcd.get('Prtry', {}) 

223 rec.record_domain = record_domain = camt053_get_str(d_domn, 'Cd', name='Stmt.Ntry[{}].BkTxCd.Domn.Cd'.format(archive_id)) 

224 rec.record_code = camt053_domain_from_record_code(record_domain) 

225 rec.family_code = camt053_get_str(d_family, 'Cd', name='Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd'.format(archive_id)) 

226 rec.sub_family_code = camt053_get_str(d_family, 'SubFmlyCd', name='Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd'.format(archive_id)) 

227 rec.record_description = camt053_get_str(d_prtry, 'Cd', required=False) 

228 

229 rec.full_clean() 

230 rec.save() 

231 

232 for dtl_batch in ntry.get('NtryDtls', []): 

233 batch_identifier = dtl_batch.get('Btch', {}).get('MsgId', '') 

234 dtl_ix = 0 

235 for dtl in dtl_batch.get('TxDtls', []): 

236 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier) 

237 

238 d_amt_dtl = dtl.get('AmtDtls', {}) 

239 d_txamt = d_amt_dtl.get('TxAmt', {}) 

240 d_xchg = d_txamt.get('CcyXchg', None) 

241 

242 d.amount, d.currency_code = camt053_get_currency(d_txamt, 'Amt', required=False) 

243 d.instructed_amount, source_currency = camt053_get_currency(d_amt_dtl.get('InstdAmt', {}), 'Amt', required=False) 

244 if (not d_xchg and source_currency and source_currency != d.currency_code) or (d_xchg and not source_currency): 

245 raise ValidationError(_('Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls'.format(archive_id, dtl_ix))) 

246 

247 if source_currency and source_currency != d.currency_code: 

248 source_currency = camt053_get_str(d_xchg, 'SrcCcy', default=source_currency, required=False) 

249 target_currency = camt053_get_str(d_xchg, 'TrgCcy', default=d.currency_code, required=False) 

250 unit_currency = camt053_get_str(d_xchg, 'UnitCcy', default='', required=False) 

251 exchange_rate_str = camt053_get_str(d_xchg, 'XchgRate', default='', required=False) 

252 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None 

253 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0] 

254 d.exchange = CurrencyExchange.objects.get_or_create(record_date=record_date, source_currency=source_currency, 

255 target_currency=target_currency, unit_currency=unit_currency, 

256 exchange_rate=exchange_rate, source=exchange_source)[0] 

257 

258 d_refs = dtl.get('Refs', {}) 

259 d.archive_identifier = d_refs.get('AcctSvcrRef', '') 

260 d.end_to_end_identifier = d_refs.get('EndToEndId', '') 

261 

262 d_parties = dtl.get('RltdPties', {}) 

263 d_dbt = d_parties.get('Dbtr', {}) 

264 d.debtor_name = d_dbt.get('Nm', '') 

265 d_udbt = d_parties.get('UltmtDbtr', {}) 

266 d.ultimate_debtor_name = d_udbt.get('Nm', '') 

267 d_cdtr = d_parties.get('Cdtr', {}) 

268 d.creditor_name = d_cdtr.get('Nm', '') 

269 d_cdtr_acct = d_parties.get('CdtrAcct', {}) 

270 d.creditor_account = d_cdtr_acct.get('Id', {}).get('IBAN', '') 

271 

272 d_rmt = dtl.get('RmtInf', {}) 

273 d.unstructured_remittance_info = d_rmt.get('Ustrd', '') 

274 

275 d_rltd_dts = dtl.get('RltdDts', {}) 

276 d.paid_date = camt053_get_dt(d_rltd_dts, 'AccptncDtTm') if 'AccptncDtTm' in d_rltd_dts else None 

277 

278 d.full_clean() 

279 d.save() 

280 

281 st = StatementRecordRemittanceInfo(detail=d) 

282 for strd in d_rmt.get('Strd', []): 

283 additional_info = strd.get('AddtlRmtInf', '') 

284 amount, currency_code = camt053_get_currency(strd.get('RfrdDocAmt', {}), 'RmtdAmt', required=False) 

285 reference = strd.get('CdtrRefInf', {}).get('Ref', '') 

286 

287 # check if new remittance info record is needed 

288 if additional_info and st.additional_info or amount and st.amount or \ 

289 reference and st.reference: # pylint: disable=too-many-boolean-expressions 

290 st = StatementRecordRemittanceInfo(detail=d) 

291 

292 if additional_info: 

293 st.additional_info = additional_info 

294 if amount: 

295 st.amount, st.currency_code = amount, currency_code 

296 if reference: 

297 st.reference = reference 

298 

299 st.full_clean() 

300 st.save() 

301 

302 dtl_ix += 1 

303 

304 # fill record name from details 

305 assert rec.type 

306 if not rec.name: 

307 if rec.type.code == e_withdraw.code: 

308 rec.name = camt053_get_unified_str(rec.detail_set.all(), 'creditor_name') 

309 elif rec.type.code == e_deposit.code: 

310 rec.name = camt053_get_unified_str(rec.detail_set.all(), 'debtor_name') 

311 if not rec.recipient_account_number: 

312 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), 'creditor_account') 

313 if not rec.remittance_info: 

314 rec.remittance_info = camt053_get_unified_str(StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec), 'reference') 

315 if not rec.paid_date: 

316 paid_date = camt053_get_unified_val(rec.detail_set.all(), 'paid_date', default=None) 

317 if paid_date: 

318 assert isinstance(paid_date, datetime) 

319 rec.paid_date = paid_date.date() 

320 

321 rec.full_clean() 

322 rec.save() 

323 

324 return stm