Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from datetime import datetime, date 

2from decimal import Decimal 

3from typing import Tuple, Any, Optional, Dict 

4 

5from django.conf import settings 

6from django.core.exceptions import ValidationError 

7from django.db import transaction 

8from django.utils.dateparse import parse_date 

9from django.utils.translation import gettext as _ 

10from jacc.models import Account, EntryType 

11from jutil.format import dec2, dec4 

12from jutil.parse import parse_datetime 

13from jbank.models import ( 

14 StatementFile, 

15 Statement, 

16 StatementRecord, 

17 DELIVERY_FROM_BANK_SYSTEM, 

18 StatementRecordDetail, 

19 CurrencyExchange, 

20 StatementRecordRemittanceInfo, 

21 CurrencyExchangeSource, 

22) 

23from jbank.parsers import parse_filename_suffix 

24from jutil.xml import xml_to_dict 

25 

26 

27CAMT053_STATEMENT_SUFFIXES = ("XML", "XT", "CAMT") 

28 

29CAMT053_ARRAY_TAGS = ["Bal", "Ntry", "NtryDtls", "TxDtls", "Strd"] 

30 

31CAMT053_INT_TAGS = ["NbOfNtries", "NbOfTxs"] 

32 

33 

34def camt053_get_iban(data: dict) -> str: 

35 return data.get("BkToCstmrStmt", {}).get("Stmt", {}).get("Acct", {}).get("Id", {}).get("IBAN", "") 

36 

37 

38def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = "") -> Any: 

39 if key not in data: 

40 if required: 

41 raise ValidationError(_("camt.053 field {} missing").format(name if name else key)) 

42 return default 

43 return data[key] 

44 

45 

46def camt053_get_str(data: dict, key: str, default: str = "", required: bool = True, name: str = "") -> str: 

47 return str(camt053_get_val(data, key, default, required, name)) 

48 

49 

50def camt053_get_currency(data: dict, key: str, required: bool = True, name: str = "") -> Tuple[Optional[Decimal], str]: 

51 try: 

52 v = camt053_get_val(data, key, default=None, required=False, name=name) 

53 if v is not None: 

54 amount = dec2(v["@"]) 

55 currency_code = v["@Ccy"] 

56 return amount, currency_code 

57 except Exception: 

58 pass 

59 if required: 

60 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "currency")) 

61 return None, "" 

62 

63 

64def camt053_get_dt(data: Dict[str, Any], key: str, name: str = "") -> datetime: 

65 s = camt053_get_val(data, key, None, True, name) 

66 val = parse_datetime(s) 

67 if val is None: 

68 raise ValidationError( 

69 _("camt.053 field {} type {} missing or invalid").format(name, "datetime") + ": {}".format(s) 

70 ) 

71 return val 

72 

73 

74def camt053_get_int(data: Dict[str, Any], key: str, name: str = "") -> int: 

75 s = camt053_get_val(data, key, None, True, name) 

76 try: 

77 return int(s) 

78 except Exception: 

79 pass 

80 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int")) 

81 

82 

83def camt053_get_date( 

84 data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = "" 

85) -> date: 

86 s = camt053_get_val(data, key, default, required, name) 

87 try: 

88 val = parse_date(s[:10]) 

89 if val is None: 

90 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date")) 

91 assert isinstance(val, date) 

92 return val 

93 except Exception: 

94 pass 

95 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date") + ": {}".format(s)) 

96 

97 

98def camt053_parse_statement_from_file(filename: str) -> dict: 

99 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES: 

100 raise ValidationError( 

101 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

102 filename=filename, suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES), file_type="camt.053" 

103 ) 

104 ) 

105 with open(filename, "rb") as fp: 

106 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS) 

107 return data 

108 

109 

110def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]: 

111 for bal in d_stmt.get("Bal", []): 

112 if bal.get("Tp", {}).get("CdOrPrtry", {}).get("Cd", "") == bal_type: 

113 amt = Decimal(bal.get("Amt", {}).get("@", "")) 

114 dt_data = bal.get("Dt", {}) 

115 dt = None 

116 if "Dt" in dt_data: 

117 dt = camt053_get_date(dt_data, "Dt", name="Stmt.Bal[{}].Dt.Dt".format(bal_type)) 

118 return amt, dt 

119 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt.Bal.Tp.CdOrPrty.Cd", bal_type)) 

120 

121 

122def camt053_domain_from_record_code(record_domain: str) -> str: 

123 if record_domain == "PMNT": 

124 return "700" 

125 if record_domain == "LDAS": 

126 return "761" 

127 return "" 

128 

129 

130def camt053_get_unified_val(qs, k: str, default: Any) -> Any: 

131 v = default 

132 for e in qs: 

133 v2 = getattr(e, k) 

134 if v == default: 

135 v = v2 

136 elif v and v2 and v2 != v: 

137 return default 

138 return v 

139 

140 

141def camt053_get_unified_str(qs, k: str) -> str: 

142 return camt053_get_unified_val(qs, k, "") 

143 

144 

145@transaction.atomic # noqa 

146def camt053_create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa 

147 """ 

148 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file() 

149 :param statement_data: XML data in form of dict 

150 :param name: File name of the account statement 

151 :param file: Source statement file 

152 :return: Statement 

153 """ 

154 account_number = camt053_get_iban(statement_data) 

155 if not account_number: 

156 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number="")) 

157 accounts = list(Account.objects.filter(name=account_number)) 

158 if len(accounts) != 1: 

159 raise ValidationError( 

160 "{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number) 

161 ) 

162 account = accounts[0] 

163 assert isinstance(account, Account) 

164 

165 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {}) 

166 d_acct = d_stmt.get("Acct", {}) 

167 d_ownr = d_acct.get("Ownr", {}) 

168 d_ntry = d_stmt.get("Ntry", []) 

169 d_frto = d_stmt.get("FrToDt", {}) 

170 d_txsummary = d_stmt.get("TxsSummry", {}) 

171 

172 if Statement.objects.filter(name=name, account=account).first(): 

173 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name)) 

174 stm = Statement(name=name, account=account, file=file) 

175 stm.account_number = stm.iban = account_number 

176 stm.bic = camt053_get_str(d_acct.get("Svcr", {}).get("FinInstnId", {}), "BIC", name="Stmt.Acct.Svcr.FinInstnId.BIC") 

177 stm.statement_identifier = camt053_get_str(d_stmt, "Id", name="Stmt.Id") 

178 stm.statement_number = camt053_get_str(d_stmt, "LglSeqNb", name="Stmt.LglSeqNb") 

179 stm.record_date = camt053_get_dt(d_stmt, "CreDtTm", name="Stmt.CreDtTm") 

180 stm.begin_date = camt053_get_dt(d_frto, "FrDtTm", name="Stmt.FrDtTm").date() 

181 stm.end_date = camt053_get_dt(d_frto, "ToDtTm", name="Stmt.ToDtTm").date() 

182 stm.currency_code = camt053_get_str(d_acct, "Ccy", name="Stmt.Acct.Ccy") 

183 if stm.currency_code != account.currency: 

184 raise ValidationError( 

185 _( 

186 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format( 

187 statement_currency=stm.currency_code, account_currency=account.currency 

188 ) 

189 ) 

190 ) 

191 stm.owner_name = camt053_get_str(d_ownr, "Nm", name="Stm.Acct.Ownr.Nm") 

192 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, "OPBD") 

193 if stm.begin_balance_date is None: 

194 stm.begin_balance_date = stm.begin_date 

195 stm.record_count = camt053_get_int( 

196 d_txsummary.get("TtlNtries", {}), "NbOfNtries", name="Stmt.TxsSummry.TtlNtries.NbOfNtries" 

197 ) 

198 stm.bank_specific_info_1 = camt053_get_str(d_stmt, "AddtlStmtInf", required=False)[:1024] 

199 for k, v in kw.items(): 

200 setattr(stm, k, v) 

201 stm.full_clean() 

202 stm.save() 

203 

204 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first() 

205 if not e_deposit: 

206 raise ValidationError( 

207 _("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT) 

208 ) 

209 assert isinstance(e_deposit, EntryType) 

210 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first() 

211 if not e_withdraw: 

212 raise ValidationError( 

213 _("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW) 

214 ) 

215 assert isinstance(e_withdraw, EntryType) 

216 e_types = { 

217 "CRDT": e_deposit, 

218 "DBIT": e_withdraw, 

219 } 

220 record_type_map = { 

221 "CRDT": "1", 

222 "DBIT": "2", 

223 } 

224 

225 for ntry in d_ntry: 

226 archive_id = ntry.get("AcctSvcrRef", "") 

227 amount, cur = camt053_get_currency(ntry, "Amt", name="Stmt.Ntry[{}].Amt".format(archive_id)) 

228 if cur != account.currency: 

229 raise ValidationError( 

230 _( 

231 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format( 

232 statement_currency=cur, account_currency=account.currency 

233 ) 

234 ) 

235 ) 

236 

237 cdt_dbt_ind = ntry["CdtDbtInd"] 

238 e_type = e_types.get(cdt_dbt_ind, None) 

239 if not e_type: 

240 raise ValidationError(_("Statement entry type {} not supported").format(cdt_dbt_ind)) 

241 

242 rec = StatementRecord(statement=stm, account=account, type=e_type) 

243 rec.amount = amount 

244 rec.archive_identifier = archive_id 

245 rec.entry_type = record_type_map[cdt_dbt_ind] 

246 rec.record_date = record_date = camt053_get_date( 

247 ntry.get("BookgDt", {}), "Dt", name="Stmt.Ntry[{}].BkkgDt.Dt".format(archive_id) 

248 ) 

249 rec.value_date = camt053_get_date(ntry.get("ValDt", {}), "Dt", name="Stmt.Ntry[{}].ValDt.Dt".format(archive_id)) 

250 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM 

251 

252 d_bktxcd = ntry.get("BkTxCd", {}) 

253 d_domn = d_bktxcd.get("Domn", {}) 

254 d_family = d_domn.get("Fmly", {}) 

255 d_prtry = d_bktxcd.get("Prtry", {}) 

256 rec.record_domain = record_domain = camt053_get_str( 

257 d_domn, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Cd".format(archive_id) 

258 ) 

259 rec.record_code = camt053_domain_from_record_code(record_domain) 

260 rec.family_code = camt053_get_str(d_family, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd".format(archive_id)) 

261 rec.sub_family_code = camt053_get_str( 

262 d_family, "SubFmlyCd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd".format(archive_id) 

263 ) 

264 rec.record_description = camt053_get_str(d_prtry, "Cd", required=False) 

265 

266 rec.full_clean() 

267 rec.save() 

268 

269 for dtl_batch in ntry.get("NtryDtls", []): 

270 batch_identifier = dtl_batch.get("Btch", {}).get("MsgId", "") 

271 dtl_ix = 0 

272 for dtl in dtl_batch.get("TxDtls", []): 

273 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier) 

274 

275 d_amt_dtl = dtl.get("AmtDtls", {}) 

276 d_txamt = d_amt_dtl.get("TxAmt", {}) 

277 d_xchg = d_txamt.get("CcyXchg", None) 

278 

279 d.amount, d.currency_code = camt053_get_currency(d_txamt, "Amt", required=False) 

280 d.instructed_amount, source_currency = camt053_get_currency( 

281 d_amt_dtl.get("InstdAmt", {}), "Amt", required=False 

282 ) 

283 if (not d_xchg and source_currency and source_currency != d.currency_code) or ( 

284 d_xchg and not source_currency 

285 ): 

286 raise ValidationError( 

287 _("Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls".format(archive_id, dtl_ix)) 

288 ) 

289 

290 if source_currency and source_currency != d.currency_code: 

291 source_currency = camt053_get_str(d_xchg, "SrcCcy", default=source_currency, required=False) 

292 target_currency = camt053_get_str(d_xchg, "TrgCcy", default=d.currency_code, required=False) 

293 unit_currency = camt053_get_str(d_xchg, "UnitCcy", default="", required=False) 

294 exchange_rate_str = camt053_get_str(d_xchg, "XchgRate", default="", required=False) 

295 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None 

296 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0] 

297 d.exchange = CurrencyExchange.objects.get_or_create( 

298 record_date=record_date, 

299 source_currency=source_currency, 

300 target_currency=target_currency, 

301 unit_currency=unit_currency, 

302 exchange_rate=exchange_rate, 

303 source=exchange_source, 

304 )[0] 

305 

306 d_refs = dtl.get("Refs", {}) 

307 d.archive_identifier = d_refs.get("AcctSvcrRef", "") 

308 d.end_to_end_identifier = d_refs.get("EndToEndId", "") 

309 

310 d_parties = dtl.get("RltdPties", {}) 

311 d_dbt = d_parties.get("Dbtr", {}) 

312 d.debtor_name = d_dbt.get("Nm", "") 

313 d_udbt = d_parties.get("UltmtDbtr", {}) 

314 d.ultimate_debtor_name = d_udbt.get("Nm", "") 

315 d_cdtr = d_parties.get("Cdtr", {}) 

316 d.creditor_name = d_cdtr.get("Nm", "") 

317 d_cdtr_acct = d_parties.get("CdtrAcct", {}) 

318 d_cdtr_acct_id = d_cdtr_acct.get("Id", {}) 

319 d.creditor_account = d_cdtr_acct_id.get("IBAN", "") 

320 if d.creditor_account: 

321 d.creditor_account_scheme = "IBAN" 

322 else: 

323 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {} 

324 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "") 

325 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or "" 

326 

327 d_rmt = dtl.get("RmtInf", {}) 

328 d.unstructured_remittance_info = d_rmt.get("Ustrd", "") 

329 

330 d_rltd_dts = dtl.get("RltdDts", {}) 

331 d.paid_date = camt053_get_dt(d_rltd_dts, "AccptncDtTm") if "AccptncDtTm" in d_rltd_dts else None 

332 

333 d.full_clean() 

334 d.save() 

335 

336 st = StatementRecordRemittanceInfo(detail=d) 

337 for strd in d_rmt.get("Strd", []): 

338 additional_info = strd.get("AddtlRmtInf", "") 

339 has_additional_info = bool(additional_info and st.additional_info) 

340 amount, currency_code = camt053_get_currency(strd.get("RfrdDocAmt", {}), "RmtdAmt", required=False) 

341 has_amount = bool(amount and st.amount) 

342 reference = strd.get("CdtrRefInf", {}).get("Ref", "") 

343 has_reference = bool(reference and st.reference) 

344 

345 # check if new remittance info record is needed 

346 if has_additional_info or has_amount or has_reference: 

347 st = StatementRecordRemittanceInfo(detail=d) 

348 

349 if additional_info: 

350 st.additional_info = additional_info 

351 if amount: 

352 st.amount, st.currency_code = amount, currency_code 

353 if reference: 

354 st.reference = reference 

355 

356 st.full_clean() 

357 st.save() 

358 

359 dtl_ix += 1 

360 

361 # fill record name from details 

362 assert rec.type 

363 if not rec.name: 

364 if rec.type.code == e_withdraw.code: 

365 rec.name = camt053_get_unified_str(rec.detail_set.all(), "creditor_name") 

366 elif rec.type.code == e_deposit.code: 

367 rec.name = camt053_get_unified_str(rec.detail_set.all(), "debtor_name") 

368 if not rec.recipient_account_number: 

369 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account") 

370 if not rec.remittance_info: 

371 rec.remittance_info = camt053_get_unified_str( 

372 StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec), "reference" 

373 ) 

374 if not rec.paid_date: 

375 paid_date = camt053_get_unified_val(rec.detail_set.all(), "paid_date", default=None) 

376 if paid_date: 

377 assert isinstance(paid_date, datetime) 

378 rec.paid_date = paid_date.date() 

379 

380 rec.full_clean() 

381 rec.save() 

382 

383 return stm