Coverage for jbank/camt.py : 84%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from datetime import datetime, date
2from decimal import Decimal
3from typing import Tuple, Any, Optional, Dict
5from django.conf import settings
6from django.core.exceptions import ValidationError
7from django.db import transaction
8from django.utils.dateparse import parse_date
9from django.utils.translation import gettext as _
10from jacc.models import Account, EntryType
11from jutil.format import dec2, dec4
12from jutil.parse import parse_datetime
13from jbank.models import StatementFile, Statement, StatementRecord, DELIVERY_FROM_BANK_SYSTEM, \
14 StatementRecordDetail, CurrencyExchange, StatementRecordRemittanceInfo, CurrencyExchangeSource
15from jbank.parsers import parse_filename_suffix
16from jutil.xml import xml_to_dict
19CAMT053_STATEMENT_SUFFIXES = ('XML', 'XT', 'CAMT')
21CAMT053_ARRAY_TAGS = ['Bal', 'Ntry', 'NtryDtls', 'TxDtls', 'Strd']
23CAMT053_INT_TAGS = ['NbOfNtries', 'NbOfTxs']
26def camt053_get_iban(data: dict) -> str:
27 return data.get('BkToCstmrStmt', {}).get('Stmt', {}).get('Acct', {}).get('Id', {}).get('IBAN', '')
30def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = '') -> Any:
31 if key not in data:
32 if required:
33 raise ValidationError(_('camt.053 field {} missing').format(name if name else key))
34 return default
35 return data[key]
38def camt053_get_str(data: dict, key: str, default: str = '', required: bool = True, name: str = '') -> str:
39 return str(camt053_get_val(data, key, default, required, name))
42def camt053_get_currency(data: dict, key: str, required: bool = True,
43 name: str = '') -> Tuple[Optional[Decimal], str]:
44 try:
45 v = camt053_get_val(data, key, default=None, required=False, name=name)
46 if v is not None:
47 amount = dec2(v['@'])
48 currency_code = v['@Ccy']
49 return amount, currency_code
50 except Exception:
51 pass
52 if required:
53 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'currency'))
54 return None, ''
57def camt053_get_dt(data: Dict[str, Any], key: str, name: str = '') -> datetime:
58 s = camt053_get_val(data, key, None, True, name)
59 val = parse_datetime(s)
60 if val is None:
61 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'datetime') + ': {}'.format(s))
62 return val
65def camt053_get_int(data: Dict[str, Any], key: str, name: str = '') -> int:
66 s = camt053_get_val(data, key, None, True, name)
67 try:
68 return int(s)
69 except Exception:
70 pass
71 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'int'))
74def camt053_get_date(data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = '') -> date:
75 s = camt053_get_val(data, key, default, required, name)
76 try:
77 val = parse_date(s[:10])
78 if val is None:
79 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'date'))
80 assert isinstance(val, date)
81 return val
82 except Exception:
83 pass
84 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'date') + ': {}'.format(s))
87def camt053_parse_statement_from_file(filename: str) -> dict:
88 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
89 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(
90 filename=filename, suffixes=', '.join(CAMT053_STATEMENT_SUFFIXES), file_type='camt.053'))
91 with open(filename, 'rb') as fp:
92 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS)
93 return data
96def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]:
97 for bal in d_stmt.get('Bal', []):
98 if bal.get('Tp', {}).get('CdOrPrtry', {}).get('Cd', '') == bal_type:
99 amt = Decimal(bal.get('Amt', {}).get('@', ''))
100 dt_data = bal.get('Dt', {})
101 dt = None
102 if 'Dt' in dt_data:
103 dt = camt053_get_date(dt_data, 'Dt', name='Stmt.Bal[{}].Dt.Dt'.format(bal_type))
104 return amt, dt
105 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format('Stmt.Bal.Tp.CdOrPrty.Cd', bal_type))
108def camt053_domain_from_record_code(record_domain: str) -> str:
109 if record_domain == 'PMNT':
110 return '700'
111 if record_domain == 'LDAS':
112 return '761'
113 return ''
116def camt053_get_unified_val(qs, k: str, default: Any) -> Any:
117 v = default
118 for e in qs:
119 v2 = getattr(e, k)
120 if v == default:
121 v = v2
122 elif v and v2 and v2 != v:
123 return default
124 return v
127def camt053_get_unified_str(qs, k: str) -> str:
128 return camt053_get_unified_val(qs, k, '')
131@transaction.atomic # noqa
132def camt053_create_statement(statement_data: dict, # noqa
133 name: str, file: StatementFile, **kw) -> Statement:
134 """
135 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file()
136 :param statement_data: XML data in form of dict
137 :param name: File name of the account statement
138 :param file: Source statement file
139 :return: Statement
140 """
141 account_number = camt053_get_iban(statement_data)
142 if not account_number:
143 raise ValidationError('{name}: '.format(name=name) + _("account.not.found").format(account_number=''))
144 accounts = list(Account.objects.filter(name=account_number))
145 if len(accounts) != 1:
146 raise ValidationError('{name}: '.format(name=name) + _("account.not.found").format(account_number=account_number))
147 account = accounts[0]
148 assert isinstance(account, Account)
150 d_stmt = statement_data.get('BkToCstmrStmt', {}).get('Stmt', {})
151 d_acct = d_stmt.get('Acct', {})
152 d_ownr = d_acct.get('Ownr', {})
153 d_ntry = d_stmt.get('Ntry', [])
154 d_frto = d_stmt.get('FrToDt', {})
155 d_txsummary = d_stmt.get('TxsSummry', {})
157 if Statement.objects.filter(name=name, account=account).first():
158 raise ValidationError('Bank account {} statement {} of processed already'.format(account_number, name))
159 stm = Statement(name=name, account=account, file=file)
160 stm.account_number = stm.iban = account_number
161 stm.bic = camt053_get_str(d_acct.get('Svcr', {}).get('FinInstnId', {}), 'BIC', name='Stmt.Acct.Svcr.FinInstnId.BIC')
162 stm.statement_identifier = camt053_get_str(d_stmt, 'Id', name='Stmt.Id')
163 stm.statement_number = camt053_get_str(d_stmt, 'LglSeqNb', name='Stmt.LglSeqNb')
164 stm.record_date = camt053_get_dt(d_stmt, 'CreDtTm', name='Stmt.CreDtTm')
165 stm.begin_date = camt053_get_dt(d_frto, 'FrDtTm', name='Stmt.FrDtTm').date()
166 stm.end_date = camt053_get_dt(d_frto, 'ToDtTm', name='Stmt.ToDtTm').date()
167 stm.currency_code = camt053_get_str(d_acct, 'Ccy', name='Stmt.Acct.Ccy')
168 if stm.currency_code != account.currency:
169 raise ValidationError(_('Account currency {account_currency} does not match statement entry currency {statement_currency}'.format(
170 statement_currency=stm.currency_code, account_currency=account.currency)))
171 stm.owner_name = camt053_get_str(d_ownr, 'Nm', name='Stm.Acct.Ownr.Nm')
172 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, 'OPBD')
173 if stm.begin_balance_date is None:
174 stm.begin_balance_date = stm.begin_date
175 stm.record_count = camt053_get_int(d_txsummary.get('TtlNtries', {}), 'NbOfNtries', name='Stmt.TxsSummry.TtlNtries.NbOfNtries')
176 stm.bank_specific_info_1 = camt053_get_str(d_stmt, 'AddtlStmtInf', required=False)[:1024]
177 for k, v in kw.items():
178 setattr(stm, k, v)
179 stm.full_clean()
180 stm.save()
182 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first()
183 if not e_deposit:
184 raise ValidationError(_('entry.type.missing') + ' ({}): {}'.format('settings.E_BANK_DEPOSIT', settings.E_BANK_DEPOSIT))
185 assert isinstance(e_deposit, EntryType)
186 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first()
187 if not e_withdraw:
188 raise ValidationError(_('entry.type.missing') + ' ({}): {}'.format('settings.E_BANK_WITHDRAW', settings.E_BANK_WITHDRAW))
189 assert isinstance(e_withdraw, EntryType)
190 e_types = {
191 'CRDT': e_deposit,
192 'DBIT': e_withdraw,
193 }
194 record_type_map = {
195 'CRDT': '1',
196 'DBIT': '2',
197 }
199 for ntry in d_ntry:
200 archive_id = ntry.get('AcctSvcrRef', '')
201 amount, cur = camt053_get_currency(ntry, 'Amt', name='Stmt.Ntry[{}].Amt'.format(archive_id))
202 if cur != account.currency:
203 raise ValidationError(_('Account currency {account_currency} does not match statement entry currency {statement_currency}'.format(
204 statement_currency=cur, account_currency=account.currency)))
206 cdt_dbt_ind = ntry['CdtDbtInd']
207 e_type = e_types.get(cdt_dbt_ind, None)
208 if not e_type:
209 raise ValidationError(_('Statement entry type {} not supported').format(cdt_dbt_ind))
211 rec = StatementRecord(statement=stm, account=account, type=e_type)
212 rec.amount = amount
213 rec.archive_identifier = archive_id
214 rec.entry_type = record_type_map[cdt_dbt_ind]
215 rec.record_date = record_date = camt053_get_date(ntry.get('BookgDt', {}), 'Dt', name='Stmt.Ntry[{}].BkkgDt.Dt'.format(archive_id))
216 rec.value_date = camt053_get_date(ntry.get('ValDt', {}), 'Dt', name='Stmt.Ntry[{}].ValDt.Dt'.format(archive_id))
217 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM
219 d_bktxcd = ntry.get('BkTxCd', {})
220 d_domn = d_bktxcd.get('Domn', {})
221 d_family = d_domn.get('Fmly', {})
222 d_prtry = d_bktxcd.get('Prtry', {})
223 rec.record_domain = record_domain = camt053_get_str(d_domn, 'Cd', name='Stmt.Ntry[{}].BkTxCd.Domn.Cd'.format(archive_id))
224 rec.record_code = camt053_domain_from_record_code(record_domain)
225 rec.family_code = camt053_get_str(d_family, 'Cd', name='Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd'.format(archive_id))
226 rec.sub_family_code = camt053_get_str(d_family, 'SubFmlyCd', name='Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd'.format(archive_id))
227 rec.record_description = camt053_get_str(d_prtry, 'Cd', required=False)
229 rec.full_clean()
230 rec.save()
232 for dtl_batch in ntry.get('NtryDtls', []):
233 batch_identifier = dtl_batch.get('Btch', {}).get('MsgId', '')
234 dtl_ix = 0
235 for dtl in dtl_batch.get('TxDtls', []):
236 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier)
238 d_amt_dtl = dtl.get('AmtDtls', {})
239 d_txamt = d_amt_dtl.get('TxAmt', {})
240 d_xchg = d_txamt.get('CcyXchg', None)
242 d.amount, d.currency_code = camt053_get_currency(d_txamt, 'Amt', required=False)
243 d.instructed_amount, source_currency = camt053_get_currency(d_amt_dtl.get('InstdAmt', {}), 'Amt', required=False)
244 if (not d_xchg and source_currency and source_currency != d.currency_code) or (d_xchg and not source_currency):
245 raise ValidationError(_('Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls'.format(archive_id, dtl_ix)))
247 if source_currency and source_currency != d.currency_code:
248 source_currency = camt053_get_str(d_xchg, 'SrcCcy', default=source_currency, required=False)
249 target_currency = camt053_get_str(d_xchg, 'TrgCcy', default=d.currency_code, required=False)
250 unit_currency = camt053_get_str(d_xchg, 'UnitCcy', default='', required=False)
251 exchange_rate_str = camt053_get_str(d_xchg, 'XchgRate', default='', required=False)
252 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None
253 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0]
254 d.exchange = CurrencyExchange.objects.get_or_create(record_date=record_date, source_currency=source_currency,
255 target_currency=target_currency, unit_currency=unit_currency,
256 exchange_rate=exchange_rate, source=exchange_source)[0]
258 d_refs = dtl.get('Refs', {})
259 d.archive_identifier = d_refs.get('AcctSvcrRef', '')
260 d.end_to_end_identifier = d_refs.get('EndToEndId', '')
262 d_parties = dtl.get('RltdPties', {})
263 d_dbt = d_parties.get('Dbtr', {})
264 d.debtor_name = d_dbt.get('Nm', '')
265 d_udbt = d_parties.get('UltmtDbtr', {})
266 d.ultimate_debtor_name = d_udbt.get('Nm', '')
267 d_cdtr = d_parties.get('Cdtr', {})
268 d.creditor_name = d_cdtr.get('Nm', '')
269 d_cdtr_acct = d_parties.get('CdtrAcct', {})
270 d.creditor_account = d_cdtr_acct.get('Id', {}).get('IBAN', '')
272 d_rmt = dtl.get('RmtInf', {})
273 d.unstructured_remittance_info = d_rmt.get('Ustrd', '')
275 d_rltd_dts = dtl.get('RltdDts', {})
276 d.paid_date = camt053_get_dt(d_rltd_dts, 'AccptncDtTm') if 'AccptncDtTm' in d_rltd_dts else None
278 d.full_clean()
279 d.save()
281 st = StatementRecordRemittanceInfo(detail=d)
282 for strd in d_rmt.get('Strd', []):
283 additional_info = strd.get('AddtlRmtInf', '')
284 amount, currency_code = camt053_get_currency(strd.get('RfrdDocAmt', {}), 'RmtdAmt', required=False)
285 reference = strd.get('CdtrRefInf', {}).get('Ref', '')
287 # check if new remittance info record is needed
288 if additional_info and st.additional_info or amount and st.amount or \
289 reference and st.reference: # pylint: disable=too-many-boolean-expressions
290 st = StatementRecordRemittanceInfo(detail=d)
292 if additional_info:
293 st.additional_info = additional_info
294 if amount:
295 st.amount, st.currency_code = amount, currency_code
296 if reference:
297 st.reference = reference
299 st.full_clean()
300 st.save()
302 dtl_ix += 1
304 # fill record name from details
305 assert rec.type
306 if not rec.name:
307 if rec.type.code == e_withdraw.code:
308 rec.name = camt053_get_unified_str(rec.detail_set.all(), 'creditor_name')
309 elif rec.type.code == e_deposit.code:
310 rec.name = camt053_get_unified_str(rec.detail_set.all(), 'debtor_name')
311 if not rec.recipient_account_number:
312 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), 'creditor_account')
313 if not rec.remittance_info:
314 rec.remittance_info = camt053_get_unified_str(StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec), 'reference')
315 if not rec.paid_date:
316 paid_date = camt053_get_unified_val(rec.detail_set.all(), 'paid_date', default=None)
317 if paid_date:
318 assert isinstance(paid_date, datetime)
319 rec.paid_date = paid_date.date()
321 rec.full_clean()
322 rec.save()
324 return stm