Coverage for jbank/camt.py: 85%
365 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-27 13:36 +0700
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-27 13:36 +0700
1import logging
2from datetime import datetime, date
3from decimal import Decimal
4from typing import Tuple, Any, Optional, Dict
6from django.conf import settings
7from django.core.exceptions import ValidationError
8from django.db import transaction
9from django.utils.dateparse import parse_date
10from django.utils.translation import gettext as _
11from jacc.models import Account, EntryType
12from jutil.format import dec2, dec4
13from jutil.model import clone_model
14from jutil.parse import parse_datetime
15from jbank.models import (
16 StatementFile,
17 Statement,
18 StatementRecord,
19 DELIVERY_FROM_BANK_SYSTEM,
20 StatementRecordDetail,
21 CurrencyExchange,
22 StatementRecordRemittanceInfo,
23 CurrencyExchangeSource, ReferencePaymentBatchFile, ReferencePaymentBatch, ReferencePaymentRecord,
24)
25from jbank.parsers import parse_filename_suffix
26from jutil.xml import xml_to_dict
29CAMT053_STATEMENT_SUFFIXES = ("XML", "XT", "CAMT", "NDCAMT53L", "XML")
31CAMT053_ARRAY_TAGS = ["Bal", "Ntry", "NtryDtls", "TxDtls", "Strd", "Ustrd"]
33CAMT053_INT_TAGS = ["NbOfNtries", "NbOfTxs"]
35CAMT054_STATEMENT_SUFFIXES = ("XE", "XE", "CAMT", "NDCAMT54L", "XML")
37CAMT054_ARRAY_TAGS = ["Ntfctn", "Othr", "Ntry", "NtryDtls", "PrtryAmt", "Chrgs", "AdrLine", "Strd", "Ustrd", "RfrdDocInf", "AddtlRmtInf"]
39CAMT054_INT_TAGS = ["NbOfNtries", "NbOfTxs"]
41logger = logging.getLogger(__name__)
44def camt053_get_iban(data: dict) -> str:
45 return data.get("BkToCstmrStmt", {}).get("Stmt", {}).get("Acct", {}).get("Id", {}).get("IBAN", "")
48def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = "") -> Any:
49 if key not in data:
50 if required:
51 raise ValidationError(_("camt.053 field {} missing").format(name if name else key))
52 return default
53 return data[key]
56def camt053_get_str(data: dict, key: str, default: str = "", required: bool = True, name: str = "") -> str:
57 return str(camt053_get_val(data, key, default, required, name))
60def camt053_get_currency(data: dict, key: str, required: bool = True, name: str = "") -> Tuple[Optional[Decimal], str]:
61 try:
62 v = camt053_get_val(data, key, default=None, required=False, name=name)
63 if v is not None:
64 amount = dec2(v["@"])
65 currency_code = v["@Ccy"]
66 return amount, currency_code
67 except Exception:
68 pass
69 if required:
70 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "currency"))
71 return None, ""
74def camt053_get_dt(data: Dict[str, Any], key: str, name: str = "") -> datetime:
75 s = camt053_get_val(data, key, None, True, name)
76 val = parse_datetime(s)
77 if val is None:
78 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "datetime") + ": {}".format(s))
79 return val
82def camt053_get_int(data: Dict[str, Any], key: str, name: str = "") -> int:
83 s = camt053_get_val(data, key, None, True, name)
84 try:
85 return int(s)
86 except Exception:
87 pass
88 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int"))
91def camt053_get_int_or_none(data: Dict[str, Any], key: str, name: str = "") -> Optional[int]:
92 s = camt053_get_val(data, key, None, False, name)
93 if s is None:
94 return None
95 try:
96 return int(s)
97 except Exception:
98 pass
99 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int"))
102def camt053_get_date(data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = "") -> date:
103 s = camt053_get_val(data, key, default, required, name)
104 try:
105 val = parse_date(s[:10])
106 if val is None:
107 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date"))
108 assert isinstance(val, date)
109 return val
110 except Exception:
111 pass
112 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date") + ": {}".format(s))
115def camt053_parse_statement_from_file(filename: str) -> dict:
116 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
117 raise ValidationError(
118 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(
119 filename=filename, suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES), file_type="camt.053"
120 )
121 )
122 with open(filename, "rb") as fp:
123 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS)
124 return data
127def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]:
128 for bal in d_stmt.get("Bal", []):
129 if bal.get("Tp", {}).get("CdOrPrtry", {}).get("Cd", "") == bal_type:
130 amt = Decimal(bal.get("Amt", {}).get("@", ""))
131 dt_data = bal.get("Dt", {})
132 dt = None
133 if "Dt" in dt_data:
134 dt = camt053_get_date(dt_data, "Dt", name="Stmt.Bal[{}].Dt.Dt".format(bal_type))
135 return amt, dt
136 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt.Bal.Tp.CdOrPrty.Cd", bal_type))
139def camt053_domain_from_record_code(record_domain: str) -> str:
140 if record_domain == "PMNT":
141 return "700"
142 if record_domain == "LDAS":
143 return "761"
144 return ""
147def camt053_get_unified_val(qs, k: str, default: Any) -> Any:
148 v = default
149 for e in qs:
150 v2 = getattr(e, k)
151 if v == default:
152 v = v2
153 elif v and v2 and v2 != v:
154 return default
155 return v
158def camt053_get_unified_str(qs, k: str) -> str:
159 return camt053_get_unified_val(qs, k, "")
162@transaction.atomic # noqa
163def camt053_create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa
164 """
165 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file()
166 :param statement_data: XML data in form of dict
167 :param name: File name of the account statement
168 :param file: Source statement file
169 :return: Statement
170 """
171 account_number = camt053_get_iban(statement_data)
172 if not account_number:
173 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=""))
174 accounts = list(Account.objects.filter(name=account_number))
175 if len(accounts) != 1:
176 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number) + " (" + str(len(accounts)) + ")")
177 account = accounts[0]
178 assert isinstance(account, Account)
180 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {})
181 if not d_stmt:
182 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt", "element"))
183 d_acct = d_stmt.get("Acct", {})
184 d_ownr = d_acct.get("Ownr", {})
185 d_ntry = d_stmt.get("Ntry", [])
186 d_frto = d_stmt.get("FrToDt", {})
187 d_txsummary = d_stmt.get("TxsSummry", {})
189 if Statement.objects.filter(name=name, account=account).first():
190 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name))
191 stm = Statement(name=name, account=account, file=file)
192 stm.account_number = stm.iban = account_number
193 stm.bic = camt053_get_str(d_acct.get("Svcr", {}).get("FinInstnId", {}), "BIC", name="Stmt.Acct.Svcr.FinInstnId.BIC")
194 stm.statement_identifier = camt053_get_str(d_stmt, "Id", name="Stmt.Id")
195 stm.statement_number = camt053_get_str(d_stmt, "LglSeqNb", name="Stmt.LglSeqNb")
196 stm.record_date = camt053_get_dt(d_stmt, "CreDtTm", name="Stmt.CreDtTm")
197 stm.begin_date = camt053_get_dt(d_frto, "FrDtTm", name="Stmt.FrDtTm").date()
198 stm.end_date = camt053_get_dt(d_frto, "ToDtTm", name="Stmt.ToDtTm").date()
199 stm.currency_code = camt053_get_str(d_acct, "Ccy", name="Stmt.Acct.Ccy")
200 if stm.currency_code != account.currency:
201 raise ValidationError(
202 _(
203 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(
204 statement_currency=stm.currency_code, account_currency=account.currency
205 )
206 )
207 )
208 stm.owner_name = camt053_get_str(d_ownr, "Nm", name="Stm.Acct.Ownr.Nm")
209 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, "OPBD")
210 if stm.begin_balance_date is None:
211 stm.begin_balance_date = stm.begin_date
212 stm.record_count = camt053_get_int_or_none(d_txsummary.get("TtlNtries", {}), "NbOfNtries", name="Stmt.TxsSummry.TtlNtries.NbOfNtries") or 0
213 stm.bank_specific_info_1 = camt053_get_str(d_stmt, "AddtlStmtInf", required=False)[:1024]
214 for k, v in kw.items():
215 setattr(stm, k, v)
216 stm.full_clean()
217 stm.save()
219 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first()
220 if not e_deposit:
221 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT))
222 assert isinstance(e_deposit, EntryType)
223 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first()
224 if not e_withdraw:
225 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW))
226 assert isinstance(e_withdraw, EntryType)
227 e_types = {
228 "CRDT": e_deposit,
229 "DBIT": e_withdraw,
230 }
231 record_type_map = {
232 "CRDT": "1",
233 "DBIT": "2",
234 }
236 for ntry in d_ntry:
237 archive_id = ntry.get("AcctSvcrRef", "")
238 amount, cur = camt053_get_currency(ntry, "Amt", name="Stmt.Ntry[{}].Amt".format(archive_id))
239 if cur != account.currency:
240 raise ValidationError(
241 _(
242 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(
243 statement_currency=cur, account_currency=account.currency
244 )
245 )
246 )
248 cdt_dbt_ind = ntry["CdtDbtInd"]
249 e_type = e_types.get(cdt_dbt_ind, None)
250 if not e_type:
251 raise ValidationError(_("Statement entry type {} not supported").format(cdt_dbt_ind))
253 rec = StatementRecord(statement=stm, account=account, type=e_type)
254 rec.amount = amount
255 rec.archive_identifier = archive_id
256 rec.entry_type = record_type_map[cdt_dbt_ind]
257 rec.record_date = record_date = camt053_get_date(ntry.get("BookgDt", {}), "Dt", name="Stmt.Ntry[{}].BkkgDt.Dt".format(archive_id))
258 rec.value_date = camt053_get_date(ntry.get("ValDt", {}), "Dt", name="Stmt.Ntry[{}].ValDt.Dt".format(archive_id))
259 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM
261 d_bktxcd = ntry.get("BkTxCd", {})
262 d_domn = d_bktxcd.get("Domn", {})
263 d_family = d_domn.get("Fmly", {})
264 d_prtry = d_bktxcd.get("Prtry", {})
265 rec.record_domain = record_domain = camt053_get_str(d_domn, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Cd".format(archive_id))
266 rec.record_code = camt053_domain_from_record_code(record_domain)
267 rec.family_code = camt053_get_str(d_family, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd".format(archive_id))
268 rec.sub_family_code = camt053_get_str(d_family, "SubFmlyCd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd".format(archive_id))
269 rec.record_description = camt053_get_str(d_prtry, "Cd", required=False)
271 rec.full_clean()
272 rec.save()
274 for dtl_batch in ntry.get("NtryDtls", []):
275 batch_identifier = dtl_batch.get("Btch", {}).get("MsgId", "")
276 dtl_ix = 0
277 for dtl in dtl_batch.get("TxDtls", []):
278 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier)
280 d_amt_dtl = dtl.get("AmtDtls", {})
281 d_txamt = d_amt_dtl.get("TxAmt", {})
282 d_xchg = d_txamt.get("CcyXchg", None)
284 d.amount, d.currency_code = camt053_get_currency(d_txamt, "Amt", required=False)
285 d.instructed_amount, source_currency = camt053_get_currency(d_amt_dtl.get("InstdAmt", {}), "Amt", required=False)
286 if (not d_xchg and source_currency and source_currency != d.currency_code) or (d_xchg and not source_currency):
287 raise ValidationError(_("Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls".format(archive_id, dtl_ix)))
289 if source_currency and source_currency != d.currency_code:
290 source_currency = camt053_get_str(d_xchg, "SrcCcy", default=source_currency, required=False)
291 target_currency = camt053_get_str(d_xchg, "TrgCcy", default=d.currency_code, required=False)
292 unit_currency = camt053_get_str(d_xchg, "UnitCcy", default="", required=False)
293 exchange_rate_str = camt053_get_str(d_xchg, "XchgRate", default="", required=False)
294 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None
295 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0]
296 d.exchange = CurrencyExchange.objects.get_or_create(
297 record_date=record_date,
298 source_currency=source_currency,
299 target_currency=target_currency,
300 unit_currency=unit_currency,
301 exchange_rate=exchange_rate,
302 source=exchange_source,
303 )[0]
305 d_refs = dtl.get("Refs", {})
306 d.archive_identifier = d_refs.get("AcctSvcrRef", "")
307 d.end_to_end_identifier = d_refs.get("EndToEndId", "")
309 d_parties = dtl.get("RltdPties", {})
310 d_dbt = d_parties.get("Dbtr", {})
311 d.debtor_name = d_dbt.get("Nm", "")
312 d_udbt = d_parties.get("UltmtDbtr", {})
313 d.ultimate_debtor_name = d_udbt.get("Nm", "")
314 d_cdtr = d_parties.get("Cdtr", {})
315 d.creditor_name = d_cdtr.get("Nm", "")
316 d_cdtr_acct = d_parties.get("CdtrAcct", {})
317 d_cdtr_acct_id = d_cdtr_acct.get("Id", {})
318 d.creditor_account = d_cdtr_acct_id.get("IBAN", "")
319 if d.creditor_account:
320 d.creditor_account_scheme = "IBAN"
321 else:
322 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {}
323 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "")
324 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or ""
326 d_rmt = dtl.get("RmtInf", {})
327 for ustrd in d_rmt.get("Ustrd") or []:
328 if d.unstructured_remittance_info:
329 d.unstructured_remittance_info += "\n"
330 d.unstructured_remittance_info = str(ustrd).rstrip()
332 d_rltd_dts = dtl.get("RltdDts", {})
333 d.paid_date = camt053_get_dt(d_rltd_dts, "AccptncDtTm") if "AccptncDtTm" in d_rltd_dts else None
335 d.full_clean()
336 d.save()
338 st = StatementRecordRemittanceInfo(detail=d)
339 for strd in d_rmt.get("Strd", []):
340 additional_info = strd.get("AddtlRmtInf", "")
341 has_additional_info = bool(additional_info and st.additional_info)
342 amount, currency_code = camt053_get_currency(strd.get("RfrdDocAmt", {}), "RmtdAmt", required=False)
343 has_amount = bool(amount and st.amount)
344 reference = strd.get("CdtrRefInf", {}).get("Ref", "")
345 has_reference = bool(reference and st.reference)
347 # check if new remittance info record is needed
348 if has_additional_info or has_amount or has_reference:
349 st = StatementRecordRemittanceInfo(detail=d)
351 if additional_info:
352 st.additional_info = additional_info
353 if amount:
354 st.amount, st.currency_code = amount, currency_code
355 if reference:
356 st.reference = reference
358 st.full_clean()
359 st.save()
361 dtl_ix += 1
363 # fill record name from details
364 assert rec.type
365 if not rec.name:
366 if rec.type.code == e_withdraw.code:
367 rec.name = camt053_get_unified_str(rec.detail_set.all(), "creditor_name")
368 elif rec.type.code == e_deposit.code:
369 rec.name = camt053_get_unified_str(rec.detail_set.all(), "debtor_name")
370 if not rec.recipient_account_number:
371 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account")
372 if not rec.remittance_info:
373 rec.remittance_info = camt053_get_unified_str(StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec).order_by("id").distinct(), "reference")
374 if not rec.paid_date:
375 paid_date = camt053_get_unified_val(rec.detail_set.all(), "paid_date", default=None)
376 if paid_date:
377 assert isinstance(paid_date, datetime)
378 rec.paid_date = paid_date.date()
380 rec.full_clean()
381 rec.save()
383 return stm
386def camt054_parse_file(filename: str) -> dict:
387 if parse_filename_suffix(filename).upper() not in CAMT054_STATEMENT_SUFFIXES:
388 raise ValidationError(
389 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(
390 filename=filename, suffixes=", ".join(CAMT054_STATEMENT_SUFFIXES), file_type="camt.054"
391 )
392 )
393 with open(filename, "rb") as fp:
394 data = xml_to_dict(fp.read(), array_tags=CAMT054_ARRAY_TAGS, int_tags=CAMT054_INT_TAGS)
395 return data
398def camt054_parse_ntfctn_acct(ntfctn: dict, default_currency: str = "EUR") -> Tuple[str, str]:
399 """
400 Returns account_number, currency from Ntfctn
401 """
402 acc_data = ntfctn["Acct"]
403 currency = acc_data.get("Ccy") or default_currency
404 account_number = acc_data["Id"].get("IBAN") or acc_data["Id"].get("BBAN")
405 return account_number, currency
408def camt054_parse_date(data: dict, key: str) -> date:
409 val = data.get(key)
410 if not val or "Dt" not in val:
411 raise Exception(_("Failed to parse date '{}'").format(key))
412 return parse_date(val["Dt"])
415def camt054_parse_amt(data: dict, key: str) -> Tuple[Decimal, str]:
416 val = data.get(key) or {}
417 amt = val.get("Amt")
418 if not amt or "@" not in amt or "@Ccy" not in amt:
419 raise Exception(_("Failed to parse amount '{}'").format(key))
420 return Decimal(amt["@"]), amt["@Ccy"]
423def camt054_parse_rmtinf(data: dict, key: str) -> str:
424 out = ""
425 val = data.get(key) or {}
426 ustrd_list = val.get("Ustrd")
427 if ustrd_list:
428 out = "\n".join(str(ustrd).rstrip() for ustrd in ustrd_list)
429 strd_list = val.get("Strd") or []
430 for strd in strd_list:
431 cdtrrefinf = strd.get("CdtrRefInf") or {}
432 ref_val = cdtrrefinf.get("Ref") or ""
433 if ref_val:
434 if out:
435 out += "\n"
436 out += ref_val
437 return out
440def camt054_parse_dbtr(data: dict, key: str) -> str:
441 val = data.get(key) or {}
442 if not val:
443 return ""
444 dbtr = val.get("Dbtr")
445 if not dbtr or "Nm" not in dbtr:
446 raise Exception(_("Failed to parse debtor '{}'").format(key))
447 return dbtr["Nm"]
450def camt054_parse_rltdagts_cdtragt_fininstnid_bic(data: dict, key: str) -> str:
451 rltdagts = data.get(key) or {}
452 cdtragt = rltdagts.get("CdtrAgt") or {}
453 fininstnid = cdtragt.get("FinInstnId") or {}
454 return fininstnid.get("BIC") or ""
457def camt054_parse_refs_endtoendid(data: dict, key: str) -> str:
458 refs = data.get(key) or {}
459 return refs.get("EndToEndId") or ""
462@transaction.atomic
463def camt054_create_reference_payment_batch(ntfctn: dict, name: str, file: ReferencePaymentBatchFile) -> ReferencePaymentBatch:
464 account_number, account_currency = camt054_parse_ntfctn_acct(ntfctn)
465 account = Account.objects.get(name=account_number, currency=account_currency)
466 assert isinstance(account, Account)
467 created_datetime = parse_datetime(ntfctn["CreDtTm"])
468 identifier = ntfctn["Id"]
469 batch = ReferencePaymentBatch(record_date=created_datetime, file=file, identifier=identifier, name=name)
470 batch.clean()
471 batch.save()
472 e_deposit = EntryType.objects.get(code=settings.E_BANK_DEPOSIT)
473 e_withdraw = EntryType.objects.get(code=settings.E_BANK_WITHDRAW)
474 for ntry in ntfctn["Ntry"]:
475 rec = ReferencePaymentRecord(batch=batch, account_number=account_number, account=account)
476 rec.record_date = camt054_parse_date(ntry, "BookgDt")
477 rec.record_type = cdtdbtind = ntry.get("CdtDbtInd") or ""
478 if cdtdbtind == "CRDT":
479 rec.type = e_deposit
480 elif cdtdbtind == "DBIT":
481 rec.type = e_withdraw
482 else:
483 raise Exception(_("Unknown credit/debit indicator '{}'").format(cdtdbtind))
484 rec.value_date = camt054_parse_date(ntry, "ValDt")
485 ntrydtls_list = ntry.get("NtryDtls") or []
486 for ntrydtls0 in ntrydtls_list:
487 rec_tx = clone_model(rec)
488 assert isinstance(rec_tx, ReferencePaymentRecord)
489 txdtls = ntrydtls0["TxDtls"]
490 amtdtls = txdtls["AmtDtls"]
491 rec_tx.instructed_amount, rec_tx.instructed_currency = camt054_parse_amt(amtdtls, "InstdAmt")
492 rec_tx.amount, rec_currency = camt054_parse_amt(amtdtls, "TxAmt")
493 if rec_currency != account_currency:
494 raise Exception(_("Account currency {} does not match record currency {}").format(account_currency, rec_currency))
495 rec_tx.remittance_info = camt054_parse_rmtinf(txdtls, "RmtInf")
496 rec_tx.payer_name = camt054_parse_dbtr(txdtls, "RltdPties")
497 rec_tx.creditor_bank_bic = camt054_parse_rltdagts_cdtragt_fininstnid_bic(txdtls, "RltdAgts")
498 rec_tx.end_to_end_identifier = camt054_parse_refs_endtoendid(txdtls, "Refs")
499 rec_tx.clean()
500 rec_tx.save()
501 return batch