Coverage for jbank/tito.py : 83%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from copy import copy
2from os.path import basename
3from typing import Dict, Any, List
4from django.core.exceptions import ValidationError
5from django.utils.translation import gettext as _
6from pytz import timezone
7from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields
9TO_STATEMENT_SUFFIXES = ("TO", "TXT", "TITO")
11TO_FILE_HEADER_TYPES = ("T00",)
13TO_FILE_HEADER_DATES = (
14 "begin_date",
15 "end_date",
16 ("record_date", "record_time"),
17 "begin_balance_date",
18)
20TO_FILE_HEADER_DECIMALS = (
21 ("begin_balance", "begin_balance_sign"),
22 "account_limit",
23)
25TO_FILE_HEADER = (
26 ("statement_type", "X", "P"),
27 ("record_type", "XX", "P"),
28 ("record_length", "9(3)", "P"),
29 ("version", "X(3)", "P"),
30 ("account_number", "X(14)", "P"),
31 ("statement_number", "9(3)", "P"),
32 ("begin_date", "9(6)", "P"),
33 ("end_date", "9(6)", "P"),
34 ("record_date", "9(6)", "P"),
35 ("record_time", "9(4)", "P"),
36 ("customer_identifier", "X(17)", "P"),
37 ("begin_balance_date", "9(6)", "P"),
38 ("begin_balance_sign", "X", "P"),
39 ("begin_balance", "9(18)", "P"),
40 ("record_count", "9(6)", "P"),
41 ("currency_code", "X(3)", "P"),
42 ("account_name", "X(30)", "V"),
43 ("account_limit", "9(18)", "P"),
44 ("owner_name", "X(35)", "P"),
45 ("contact_info_1", "X(40)", "P"),
46 ("contact_info_2", "X(40)", "V"),
47 ("bank_specific_info_1", "X(30)", "V"),
48 ("iban_and_bic", "X(30)", "V"),
49)
51TO_FILE_RECORD_TYPES = ("T10", "T80")
53TO_FILE_RECORD_DATES = (
54 "record_date",
55 "value_date",
56 "paid_date",
57)
59TO_FILE_RECORD_DECIMALS = (("amount", "amount_sign"),)
61TO_FILE_RECORD = (
62 ("statement_type", "X", "P"),
63 ("record_type", "XX", "P"),
64 ("record_length", "9(3)", "P"),
65 ("record_number", "9(6)", "P"),
66 ("archive_identifier", "X(18)", "V"),
67 ("record_date", "9(6)", "P"),
68 ("value_date", "9(6)", "V"),
69 ("paid_date", "9(6)", "V"),
70 ("entry_type", "X", "P"), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma
71 ("record_code", "X(3)", "P"),
72 ("record_description", "X(35)", "P"),
73 ("amount_sign", "X", "P"),
74 ("amount", "9(18)", "P"),
75 ("receipt_code", "X", "P"),
76 ("delivery_method", "X", "P"),
77 ("name", "X(35)", "V"),
78 ("name_source", "X", "V"),
79 ("recipient_account_number", "X(14)", "V"),
80 ("recipient_account_number_changed", "X", "V"),
81 ("remittance_info", "X(20)", "V"),
82 ("form_number", "X(8)", "V"),
83 ("level_identifier", "X", "P"),
84)
86TO_FILE_RECORD_EXTRA_INFO_TYPES = ("T11", "T81")
88TO_FILE_RECORD_EXTRA_INFO_HEADER = (
89 ("statement_type", "X", "P"),
90 ("record_type", "XX", "P"),
91 ("record_length", "9(3)", "P"),
92 ("extra_info_type", "9(2)", "P"),
93)
95TO_FILE_RECORD_EXTRA_INFO_COUNTS = (("entry_count", "9(8)", "P"),)
97TO_FILE_RECORD_EXTRA_INFO_INVOICE = (
98 ("customer_number", "X(10)", "P"),
99 ("pad01", "X", "P"),
100 ("invoice_number", "X(15)", "P"),
101 ("pad02", "X", "P"),
102 ("invoice_date", "X(6)", "P"),
103)
105TO_FILE_RECORD_EXTRA_INFO_CARD = (
106 ("card_number", "X(19)", "P"),
107 ("pad01", "X", "P"),
108 ("merchant_reference", "X(14)", "P"),
109)
111TO_FILE_RECORD_EXTRA_INFO_CORRECTION = (("original_archive_identifier", "X(18)", "P"),)
113TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = (("amount", "amount_sign"),)
115TO_FILE_RECORD_EXTRA_INFO_CURRENCY = (
116 ("amount_sign", "X", "P"),
117 ("amount", "9(18)", "P"),
118 ("pad01", "X", "P"),
119 ("currency_code", "X(3)", "P"),
120 ("pad02", "X", "P"),
121 ("exchange_rate", "9(11)", "P"),
122 ("rate_reference", "X(6)", "V"),
123)
125TO_FILE_RECORD_EXTRA_INFO_REASON = (
126 ("reason_code", "9(3)", "P"),
127 ("pad01", "X", "P"),
128 ("reason_description", "X(31)", "P"),
129)
131TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = (("name_detail", "X(35)", "P"),)
133TO_FILE_RECORD_EXTRA_INFO_SEPA = (
134 ("reference", "X(35)", "V"),
135 ("iban_account_number", "X(35)", "V"),
136 ("bic_code", "X(35)", "V"),
137 ("recipient_name_detail", "X(70)", "V"),
138 ("payer_name_detail", "X(70)", "V"),
139 ("identifier", "X(35)", "V"),
140 ("archive_identifier", "X(35)", "V"),
141)
143TO_BALANCE_RECORD_DATES = ("record_date",)
145TO_BALANCE_RECORD_DECIMALS = (
146 ("end_balance", "end_balance_sign"),
147 ("available_balance", "available_balance_sign"),
148)
150TO_BALANCE_RECORD = (
151 ("statement_type", "X", "P"),
152 ("record_type", "XX", "P"),
153 ("record_length", "9(3)", "P"),
154 ("record_date", "9(6)", "P"),
155 ("end_balance_sign", "X", "P"),
156 ("end_balance", "9(18)", "P"),
157 ("available_balance_sign", "X", "P"),
158 ("available_balance", "9(18)", "P"),
159)
161TO_CUMULATIVE_RECORD_DATES = ("period_date",)
163TO_CUMULATIVE_RECORD_DECIMALS = (
164 ("deposits_amount", "deposits_sign"),
165 ("withdrawals_amount", "withdrawals_sign"),
166)
168TO_CUMULATIVE_RECORD = (
169 ("statement_type", "X", "P"),
170 ("record_type", "XX", "P"),
171 ("record_length", "9(3)", "P"),
172 ("period_identifier", "X", "P"), # 1=day, 2=term, 3=month, 4=year
173 ("period_date", "9(6)", "P"),
174 ("deposits_count", "9(8)", "P"),
175 ("deposits_sign", "X", "P"),
176 ("deposits_amount", "9(18)", "P"),
177 ("withdrawals_count", "9(8)", "P"),
178 ("withdrawals_sign", "X", "P"),
179 ("withdrawals_amount", "9(18)", "P"),
180)
182TO_SPECIAL_RECORD = (
183 ("statement_type", "X", "P"),
184 ("record_type", "XX", "P"),
185 ("record_length", "9(3)", "P"),
186 ("bank_group_identifier", "X(3)", "P"),
187)
190def parse_tiliote_statements_from_file(filename: str) -> list:
191 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES:
192 raise ValidationError(
193 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(
194 filename=filename, suffixes=", ".join(TO_STATEMENT_SUFFIXES), file_type="tiliote"
195 )
196 )
197 with open(filename, "rt", encoding="ISO-8859-1") as fp:
198 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore
201def parse_tiliote_statements(content: str, filename: str) -> List[dict]: # pylint: disable=too-many-locals
202 lines = content.split("\n")
203 nlines = len(lines)
205 line_number = 1
206 tz = timezone("Europe/Helsinki")
208 header = None
209 records = []
210 balance = None
211 cumulative = None
212 cumulative_adjustment = None
213 special_records = []
214 statements = []
216 while line_number <= nlines:
217 line = lines[line_number - 1]
218 if line.strip() == "":
219 line_number += 1
220 continue
221 # print('parsing line {}: {}'.format(line_number, line))
222 record_type = line[:3]
224 if record_type in TO_FILE_HEADER_TYPES:
225 if header:
226 statements.append(
227 combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records)
228 )
229 header, records, balance, cumulative, cumulative_adjustment, special_records = (
230 None,
231 [],
232 None,
233 None,
234 None,
235 [],
236 )
238 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number)
239 convert_date_fields(header, TO_FILE_HEADER_DATES, tz)
240 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS)
241 iban_and_bic = str(header.get("iban_and_bic", "")).split(" ")
242 if len(iban_and_bic) == 2:
243 header["iban"], header["bic"] = iban_and_bic
244 line_number += 1
246 elif record_type in TO_FILE_RECORD_TYPES:
247 record = parse_records(line, TO_FILE_RECORD, line_number=line_number)
248 convert_date_fields(record, TO_FILE_RECORD_DATES, tz)
249 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS)
251 line_number += 1
252 # check for record extra info
253 if line_number <= nlines:
254 line = lines[line_number - 1]
255 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES:
256 parse_record_extra_info(record, line, line_number=line_number)
257 line_number += 1
258 line = lines[line_number - 1]
260 records.append(record)
261 elif record_type == "T40":
262 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number)
263 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz)
264 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS)
265 line_number += 1
266 elif record_type == "T50":
267 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)
268 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz)
269 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS)
270 line_number += 1
271 elif record_type == "T51":
272 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)
273 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz)
274 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS)
275 line_number += 1
276 elif record_type in ("T60", "T70"):
277 special_records.append(
278 parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False)
279 )
280 line_number += 1
281 else:
282 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type))
284 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))
285 return statements
288def combine_statement( # pylint: disable=too-many-arguments
289 header, records, balance, cumulative, cumulative_adjustment, special_records
290) -> Dict[str, Any]:
291 data = {
292 "header": header,
293 "records": records,
294 }
295 if balance is not None:
296 data["balance"] = balance
297 if cumulative is not None:
298 data["cumulative"] = cumulative
299 if cumulative_adjustment is not None:
300 data["cumulative_adjustment"] = cumulative_adjustment
301 if special_records:
302 data["special_records"] = special_records
303 return data
306def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int):
307 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES:
308 raise ValidationError("SVM record extra info validation error on line {}".format(line_number))
310 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False)
311 extra_info_type = header["extra_info_type"]
312 # print(line)
313 extra_data = copy(header["extra_data"])
314 assert isinstance(extra_data, str)
315 if extra_info_type == "00":
316 record["messages"] = parse_record_messages(extra_data)
317 elif extra_info_type == "01":
318 record["counts"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8)
319 elif extra_info_type == "02":
320 record["invoice"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33)
321 elif extra_info_type == "03":
322 record["card"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34)
323 elif extra_info_type == "04":
324 record["correction"] = parse_records(
325 extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18
326 )
327 elif extra_info_type == "05":
328 record["currency"] = parse_records(
329 extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41
330 )
331 convert_decimal_fields(record["currency"], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS)
332 elif extra_info_type == "06":
333 record["client_messages"] = parse_record_messages(extra_data)
334 elif extra_info_type == "07":
335 record["bank_messages"] = parse_record_messages(extra_data)
336 elif extra_info_type == "08":
337 record["reason"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35)
338 elif extra_info_type == "09":
339 record["name_detail"] = parse_records(
340 extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35
341 )
342 elif extra_info_type == "11":
343 record["sepa"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323)
344 else:
345 raise ValidationError(
346 _('Line {line}: Invalid record extra info type "{extra_info_type}"').format(
347 line=line_number, extra_info_type=extra_info_type
348 )
349 )
352def parse_record_messages(extra_data: str) -> List[str]:
353 msg = []
354 while extra_data:
355 msg.append(extra_data[:35])
356 extra_data = extra_data[35:]
357 return msg