Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import logging 

2import re 

3from copy import copy 

4from os.path import basename 

5from datetime import time, datetime, date 

6from decimal import Decimal 

7from typing import List, Any, Tuple, Optional, Dict, Sequence, Union 

8from django.core.exceptions import ValidationError 

9from django.utils.translation import gettext as _ 

10from pytz import timezone 

11 

12 

13REGEX_SIMPLE_FIELD = re.compile(r'^(X|9)+$') 

14 

15REGEX_VARIABLE_FIELD = re.compile(r'^(X|9)\((\d+)\)$') 

16 

17TO_STATEMENT_SUFFIXES = ('TO', 'TXT') 

18 

19TO_FILE_HEADER_TYPES = ('T00', ) 

20 

21TO_FILE_HEADER_DATES = ( 

22 'begin_date', 

23 'end_date', 

24 ('record_date', 'record_time'), 

25 'begin_balance_date', 

26) 

27 

28TO_FILE_HEADER_DECIMALS = ( 

29 ('begin_balance', 'begin_balance_sign'), 

30 'account_limit', 

31) 

32 

33TO_FILE_HEADER = ( 

34 ('statement_type', 'X', 'P'), 

35 ('record_type', 'XX', 'P'), 

36 ('record_length', '9(3)', 'P'), 

37 ('version', 'X(3)', 'P'), 

38 ('account_number', 'X(14)', 'P'), 

39 ('statement_number', '9(3)', 'P'), 

40 ('begin_date', '9(6)', 'P'), 

41 ('end_date', '9(6)', 'P'), 

42 ('record_date', '9(6)', 'P'), 

43 ('record_time', '9(4)', 'P'), 

44 ('customer_identifier', 'X(17)', 'P'), 

45 ('begin_balance_date', '9(6)', 'P'), 

46 ('begin_balance_sign', 'X', 'P'), 

47 ('begin_balance', '9(18)', 'P'), 

48 ('record_count', '9(6)', 'P'), 

49 ('currency_code', 'X(3)', 'P'), 

50 ('account_name', 'X(30)', 'V'), 

51 ('account_limit', '9(18)', 'P'), 

52 ('owner_name', 'X(35)', 'P'), 

53 ('contact_info_1', 'X(40)', 'P'), 

54 ('contact_info_2', 'X(40)', 'V'), 

55 ('bank_specific_info_1', 'X(30)', 'V'), 

56 ('iban_and_bic', 'X(30)', 'V'), 

57) 

58 

59TO_FILE_RECORD_TYPES = ('T10', 'T80') 

60 

61TO_FILE_RECORD_DATES = ( 

62 'record_date', 

63 'value_date', 

64 'paid_date', 

65) 

66 

67TO_FILE_RECORD_DECIMALS = ( 

68 ('amount', 'amount_sign'), 

69) 

70 

71TO_FILE_RECORD = ( 

72 ('statement_type', 'X', 'P'), 

73 ('record_type', 'XX', 'P'), 

74 ('record_length', '9(3)', 'P'), 

75 ('record_number', '9(6)', 'P'), 

76 ('archive_identifier', 'X(18)', 'V'), 

77 ('record_date', '9(6)', 'P'), 

78 ('value_date', '9(6)', 'V'), 

79 ('paid_date', '9(6)', 'V'), 

80 ('entry_type', 'X', 'P'), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma 

81 ('record_code', 'X(3)', 'P'), 

82 ('record_description', 'X(35)', 'P'), 

83 ('amount_sign', 'X', 'P'), 

84 ('amount', '9(18)', 'P'), 

85 ('receipt_code', 'X', 'P'), 

86 ('delivery_method', 'X', 'P'), 

87 ('name', 'X(35)', 'V'), 

88 ('name_source', 'X', 'V'), 

89 ('recipient_account_number', 'X(14)', 'V'), 

90 ('recipient_account_number_changed', 'X', 'V'), 

91 ('remittance_info', 'X(20)', 'V'), 

92 ('form_number', 'X(8)', 'V'), 

93 ('level_identifier', 'X', 'P'), 

94) 

95 

96TO_FILE_RECORD_EXTRA_INFO_TYPES = ('T11', 'T81') 

97 

98TO_FILE_RECORD_EXTRA_INFO_HEADER = ( 

99 ('statement_type', 'X', 'P'), 

100 ('record_type', 'XX', 'P'), 

101 ('record_length', '9(3)', 'P'), 

102 ('extra_info_type', '9(2)', 'P'), 

103) 

104 

105TO_FILE_RECORD_EXTRA_INFO_COUNTS = ( 

106 ('entry_count', '9(8)', 'P'), 

107) 

108 

109TO_FILE_RECORD_EXTRA_INFO_INVOICE = ( 

110 ('customer_number', 'X(10)', 'P'), 

111 ('pad01', 'X', 'P'), 

112 ('invoice_number', 'X(15)', 'P'), 

113 ('pad02', 'X', 'P'), 

114 ('invoice_date', 'X(6)', 'P'), 

115) 

116 

117TO_FILE_RECORD_EXTRA_INFO_CARD = ( 

118 ('card_number', 'X(19)', 'P'), 

119 ('pad01', 'X', 'P'), 

120 ('merchant_reference', 'X(14)', 'P'), 

121) 

122 

123TO_FILE_RECORD_EXTRA_INFO_CORRECTION = ( 

124 ('original_archive_identifier', 'X(18)', 'P'), 

125) 

126 

127TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = ( 

128 ('amount', 'amount_sign'), 

129) 

130 

131TO_FILE_RECORD_EXTRA_INFO_CURRENCY = ( 

132 ('amount_sign', 'X', 'P'), 

133 ('amount', '9(18)', 'P'), 

134 ('pad01', 'X', 'P'), 

135 ('currency_code', 'X(3)', 'P'), 

136 ('pad02', 'X', 'P'), 

137 ('exchange_rate', '9(11)', 'P'), 

138 ('rate_reference', 'X(6)', 'V'), 

139) 

140 

141TO_FILE_RECORD_EXTRA_INFO_REASON = ( 

142 ('reason_code', '9(3)', 'P'), 

143 ('pad01', 'X', 'P'), 

144 ('reason_description', 'X(31)', 'P'), 

145) 

146 

147TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = ( 

148 ('name_detail', 'X(35)', 'P'), 

149) 

150 

151TO_FILE_RECORD_EXTRA_INFO_SEPA = ( 

152 ('reference', 'X(35)', 'V'), 

153 ('iban_account_number', 'X(35)', 'V'), 

154 ('bic_code', 'X(35)', 'V'), 

155 ('recipient_name_detail', 'X(70)', 'V'), 

156 ('payer_name_detail', 'X(70)', 'V'), 

157 ('identifier', 'X(35)', 'V'), 

158 ('archive_identifier', 'X(35)', 'V'), 

159) 

160 

161TO_BALANCE_RECORD_DATES = ( 

162 'record_date', 

163) 

164 

165TO_BALANCE_RECORD_DECIMALS = ( 

166 ('end_balance', 'end_balance_sign'), 

167 ('available_balance', 'available_balance_sign'), 

168) 

169 

170TO_BALANCE_RECORD = ( 

171 ('statement_type', 'X', 'P'), 

172 ('record_type', 'XX', 'P'), 

173 ('record_length', '9(3)', 'P'), 

174 ('record_date', '9(6)', 'P'), 

175 ('end_balance_sign', 'X', 'P'), 

176 ('end_balance', '9(18)', 'P'), 

177 ('available_balance_sign', 'X', 'P'), 

178 ('available_balance', '9(18)', 'P'), 

179) 

180 

181TO_CUMULATIVE_RECORD_DATES = ( 

182 'period_date', 

183) 

184 

185TO_CUMULATIVE_RECORD_DECIMALS = ( 

186 ('deposits_amount', 'deposits_sign'), 

187 ('withdrawals_amount', 'withdrawals_sign'), 

188) 

189 

190TO_CUMULATIVE_RECORD = ( 

191 ('statement_type', 'X', 'P'), 

192 ('record_type', 'XX', 'P'), 

193 ('record_length', '9(3)', 'P'), 

194 ('period_identifier', 'X', 'P'), # 1=day, 2=term, 3=month, 4=year 

195 ('period_date', '9(6)', 'P'), 

196 ('deposits_count', '9(8)', 'P'), 

197 ('deposits_sign', 'X', 'P'), 

198 ('deposits_amount', '9(18)', 'P'), 

199 ('withdrawals_count', '9(8)', 'P'), 

200 ('withdrawals_sign', 'X', 'P'), 

201 ('withdrawals_amount', '9(18)', 'P'), 

202) 

203 

204TO_SPECIAL_RECORD = ( 

205 ('statement_type', 'X', 'P'), 

206 ('record_type', 'XX', 'P'), 

207 ('record_length', '9(3)', 'P'), 

208 ('bank_group_identifier', 'X(3)', 'P'), 

209) 

210 

211SVM_STATEMENT_SUFFIXES = ('SVM', 'TXT', 'KTL') 

212 

213SVM_FILE_HEADER_DATES = ( 

214 ('record_date', 'record_time'), 

215) 

216 

217SVM_FILE_HEADER_TYPES = ( 

218 '0', 

219) 

220 

221SVM_FILE_HEADER = ( 

222 ('statement_type', '9(1)', 'P'), 

223 ('record_date', '9(6)', 'P'), 

224 ('record_time', '9(4)', 'P'), 

225 ('institution_identifier', 'X(2)', 'P'), 

226 ('service_identifier', 'X(9)', 'P'), 

227 ('currency_identifier', 'X(1)', 'P'), 

228 ('pad01', 'X(67)', 'P'), 

229) 

230 

231SVM_FILE_RECORD_TYPES = ('3', '5') 

232 

233SVM_FILE_RECORD_DECIMALS = ( 

234 'amount', 

235) 

236 

237SVM_FILE_RECORD_DATES = ( 

238 'record_date', 

239 'paid_date', 

240) 

241 

242SVM_FILE_RECORD = ( 

243 ('record_type', '9(1)', 'P'), # 3=viitesiirto, 5=suoraveloitus 

244 ('account_number', '9(14)', 'P'), 

245 ('record_date', '9(6)', 'P'), 

246 ('paid_date', '9(6)', 'P'), 

247 ('archive_identifier', 'X(16)', 'P'), 

248 ('remittance_info', '9(20)', 'P'), 

249 ('payer_name', 'X(12)', 'P'), 

250 ('currency_identifier', 'X(1)', 'P'), # 1=eur 

251 ('name_source', 'X', 'V'), 

252 ('amount', '9(10)', 'P'), 

253 ('correction_identifier', 'X', 'V'), # 0=normal, 1=correction 

254 ('delivery_method', 'X', 'P'), # A=asiakkaalta, K=konttorista, J=pankin jarjestelmasta 

255 ('receipt_code', 'X', 'P'), 

256) 

257 

258SVM_FILE_SUMMARY_TYPES = ( 

259 '9', 

260) 

261 

262SVM_FILE_SUMMARY_DECIMALS = ( 

263 'record_amount', 

264 'correction_amount', 

265) 

266 

267SVM_FILE_SUMMARY = ( 

268 ('record_type', '9(1)', 'P'), # 9 

269 ('record_count', '9(6)', 'P'), 

270 ('record_amount', '9(11)', 'P'), 

271 ('correction_count', '9(6)', 'P'), 

272 ('correction_amount', '9(11)', 'P'), 

273 ('pad01', 'X(5)', 'P'), 

274) 

275 

276 

277logger = logging.getLogger(__name__) 

278 

279 

280def parse_record_format(fmt: str) -> tuple: 

281 """ 

282 :param fmt: Data format used in .TO files 

283 :return: Data type ('X' or '9'), data length (number of characters) 

284 """ 

285 res = REGEX_SIMPLE_FIELD.match(fmt) 

286 data_type, data_len = None, None 

287 if res: 

288 data_type = res.group(1) 

289 data_len = len(fmt) 

290 else: 

291 res = REGEX_VARIABLE_FIELD.match(fmt) 

292 if res: 

293 data_type = res.group(1) 

294 data_len = int(res.group(2)) 

295 if not data_type or not data_len: 

296 raise Exception('Failed to parse data format {}'.format(fmt)) 

297 return data_type, data_len 

298 

299 

300def parse_record_value(data_type, data_len, data, name: str, line_number: int) -> str: 

301 value = data[:data_len] 

302 if len(value) != data_len: 

303 raise ValidationError(_('Line {line}: Invalid field "{field}" value "{value}"').format(line=line_number, field=name, value=value)) 

304 if data_type == 'X': 

305 pass 

306 elif data_type == '9': 

307 charset = '0123456789' 

308 for ch in value: 

309 if ch not in charset: 

310 raise ValidationError(_('Line {line}: Invalid field "{field}" value "{value}"').format(line=line_number, field=name, value=value)) 

311 # logger.info('jbank.parsers.parse_record_value: {} = {}'.format(name, value)) 

312 else: 

313 raise ValidationError(_('Line {line}: Invalid field "{field}" value "{value}"').format(line=line_number, field=name, value=value)) 

314 return value 

315 

316 

317def parse_records(line: str, specs: Sequence[Tuple[str, str, str]], line_number: int, check_record_length: bool = True, 

318 record_length: Optional[int] = None) -> Dict[str, Union[int, str]]: 

319 i = 0 

320 data: Dict[str, Union[int, str]] = dict() 

321 data['line_number'] = line_number 

322 for name, fmt, req in specs: # pylint: disable=unused-variable 

323 data_type, data_len = parse_record_format(fmt) 

324 value = parse_record_value(data_type, data_len, line[i:], name=name, line_number=line_number) 

325 # print('[{}:{}] {}="{}"'.format(i, i+data_len, name, value)) 

326 data[name] = str(value).strip() 

327 i += data_len 

328 data['extra_data'] = line[i:] 

329 

330 rec_len = data.get('record_length', record_length) 

331 if check_record_length and rec_len: 

332 data['extra_data'] = str(data['extra_data']).strip() 

333 if i != rec_len and data['extra_data'] != '': 

334 raise ValidationError(_('Line {line}: Record length ({record_length}) does not match length of ' 

335 'parsed data ({data_length}). Extra data: "{extra_data}"').format( 

336 line=line_number, data_length=i+len(str(data['extra_data'])), 

337 record_length=rec_len, extra_data=data['extra_data'])) 

338 return data 

339 

340 

341def parse_record_messages(extra_data: str) -> List[str]: 

342 msg = [] 

343 while extra_data: 

344 msg.append(extra_data[:35]) 

345 extra_data = extra_data[35:] 

346 return msg 

347 

348 

349def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int): 

350 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES: 

351 raise ValidationError('SVM record extra info validation error on line {}'.format(line_number)) 

352 

353 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False) 

354 extra_info_type = header['extra_info_type'] 

355 # print(line) 

356 extra_data = copy(header['extra_data']) 

357 assert isinstance(extra_data, str) 

358 if extra_info_type == '00': 

359 record['messages'] = parse_record_messages(extra_data) 

360 elif extra_info_type == '01': 

361 record['counts'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8) 

362 elif extra_info_type == '02': 

363 record['invoice'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33) 

364 elif extra_info_type == '03': 

365 record['card'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34) 

366 elif extra_info_type == '04': 

367 record['correction'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18) 

368 elif extra_info_type == '05': 

369 record['currency'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41) 

370 convert_decimal_fields(record['currency'], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS) 

371 elif extra_info_type == '06': 

372 record['client_messages'] = parse_record_messages(extra_data) 

373 elif extra_info_type == '07': 

374 record['bank_messages'] = parse_record_messages(extra_data) 

375 elif extra_info_type == '08': 

376 record['reason'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35) 

377 elif extra_info_type == '09': 

378 record['name_detail'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35) 

379 elif extra_info_type == '11': 

380 record['sepa'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323) 

381 else: 

382 raise ValidationError(_('Line {line}: Invalid record extra info type "{extra_info_type}"').format(line=line_number, extra_info_type=extra_info_type)) 

383 

384 

385def convert_date(v: Optional[str], field_name: str) -> date: 

386 if v is None: 

387 raise ValidationError(_("Date field missing: {}").format(field_name)) 

388 if len(v) != 6: 

389 raise ValidationError(_("Date format error in field {}: {}").format(field_name, v)) 

390 year = int(v[0:2]) + 2000 

391 month = int(v[2:4]) 

392 day = int(v[4:6]) 

393 return date(year=year, month=month, day=day) 

394 

395 

396def convert_time(v: Optional[str], field_name: str) -> time: 

397 if v is None: 

398 raise ValidationError(_("Time field missing: {}").format(field_name)) 

399 if not re.match(r'^\d\d\d\d$', v): 

400 raise ValidationError(_("Time format error in field {}: {}").format(field_name, v)) 

401 return time(int(v[0:2]), int(v[2:4])) 

402 

403 

404def convert_date_fields(data: dict, date_fields: tuple, tz: Any): 

405 for k in date_fields: 

406 if isinstance(k, str): 

407 v_date = data.get(k) 

408 if v_date: 

409 data[k] = convert_date(v_date, k) 

410 elif isinstance(k, tuple): 

411 if len(k) != 2: 

412 raise ValidationError(_("Date format error in field {}").format(k)) 

413 k_date, k_time = k 

414 v_date, v_time = data.get(k_date), data.get(k_time) 

415 # print('Converting {}'.format(k)) 

416 # pprint(data) 

417 if v_date or v_time: 

418 assert v_date is None or isinstance(v_date, str) 

419 assert v_time is None or isinstance(v_time, str) 

420 v_date = convert_date(v_date, k_date) 

421 v_time = convert_time(v_time, k_time) 

422 v_datetime = datetime.combine(v_date, v_time) 

423 data[k_date] = tz.localize(v_datetime) 

424 del data[k_time] 

425 

426 

427def convert_decimal_fields(data: dict, decimal_fields: tuple): 

428 for field in decimal_fields: 

429 if isinstance(field, str): 

430 v_number = data.get(field) 

431 if v_number is not None: 

432 v = Decimal(v_number) * Decimal('0.01') 

433 # logger.info('jbank.parsers.convert_decimal_fields: {} = {}'.format(field, v)) 

434 data[field] = v 

435 elif isinstance(field, tuple) and len(field) == 2: 

436 k_number, k_sign = field 

437 v_number, v_sign = data.get(k_number), data.get(k_sign) 

438 if v_number is not None: 

439 v = Decimal(v_number) * Decimal('0.01') 

440 if v_sign == '-': 

441 v = -v 

442 data[k_number] = v 

443 # logger.info('jbank.parsers.convert_decimal_fields: {} = {}'.format(k_number, v)) 

444 del data[k_sign] 

445 else: 

446 raise ValidationError(_('Invalid decimal field format: {}').format(field)) 

447 

448 

449def combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records) -> dict: # pylint: disable=too-many-arguments 

450 data = { 

451 'header': header, 

452 'records': records, 

453 } 

454 if balance is not None: 

455 data['balance'] = balance 

456 if cumulative is not None: 

457 data['cumulative'] = cumulative 

458 if cumulative_adjustment is not None: 

459 data['cumulative_adjustment'] = cumulative_adjustment 

460 if special_records: 

461 data['special_records'] = special_records 

462 return data 

463 

464 

465def parse_tiliote_statements(content: str, filename: str) -> list: # pylint: disable=too-many-locals 

466 lines = content.split('\n') 

467 nlines = len(lines) 

468 

469 line_number = 1 

470 tz = timezone('Europe/Helsinki') 

471 

472 header = None 

473 records = [] 

474 balance = None 

475 cumulative = None 

476 cumulative_adjustment = None 

477 special_records = [] 

478 statements = [] 

479 

480 while line_number <= nlines: 

481 line = lines[line_number-1] 

482 if line.strip() == '': 

483 line_number += 1 

484 continue 

485 # print('parsing line {}: {}'.format(line_number, line)) 

486 record_type = line[:3] 

487 

488 if record_type in TO_FILE_HEADER_TYPES: 

489 if header: 

490 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records)) 

491 header, records, balance, cumulative, cumulative_adjustment, special_records = None, [], None, None, None, [] 

492 

493 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number) 

494 convert_date_fields(header, TO_FILE_HEADER_DATES, tz) 

495 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS) 

496 iban_and_bic = str(header.get('iban_and_bic', '')).split(' ') 

497 if len(iban_and_bic) == 2: 

498 header['iban'], header['bic'] = iban_and_bic 

499 line_number += 1 

500 

501 elif record_type in TO_FILE_RECORD_TYPES: 

502 record = parse_records(line, TO_FILE_RECORD, line_number=line_number) 

503 convert_date_fields(record, TO_FILE_RECORD_DATES, tz) 

504 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS) 

505 

506 line_number += 1 

507 # check for record extra info 

508 if line_number <= nlines: 

509 line = lines[line_number-1] 

510 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES: 

511 parse_record_extra_info(record, line, line_number=line_number) 

512 line_number += 1 

513 line = lines[line_number-1] 

514 

515 records.append(record) 

516 elif record_type == 'T40': 

517 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number) 

518 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz) 

519 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS) 

520 line_number += 1 

521 elif record_type == 'T50': 

522 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number) 

523 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz) 

524 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS) 

525 line_number += 1 

526 elif record_type == 'T51': 

527 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number) 

528 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz) 

529 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS) 

530 line_number += 1 

531 elif record_type in ('T60', 'T70'): 

532 special_records.append(parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False)) 

533 line_number += 1 

534 else: 

535 raise ValidationError(_('Unknown record type on {}({}): {}').format(filename, line_number, record_type)) 

536 

537 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records)) 

538 return statements 

539 

540 

541def parse_filename_suffix(filename: str) -> str: 

542 a = filename.rsplit('.', 1) 

543 return a[len(a)-1] 

544 

545 

546def parse_tiliote_statements_from_file(filename: str) -> list: 

547 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES: 

548 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

549 filename=filename, suffixes=', '.join(TO_STATEMENT_SUFFIXES), file_type='tiliote')) 

550 with open(filename, 'rt', encoding='ISO-8859-1') as fp: 

551 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore 

552 

553 

554def combine_svm_batch(header: Optional[Dict[str, Any]], records: List[Dict[str, Union[int, str]]], 

555 summary: Optional[Dict[str, Any]]) -> Dict[str, Any]: 

556 data = {'header': header, 'records': records} 

557 if summary is not None: 

558 data['summary'] = summary 

559 return data 

560 

561 

562def parse_svm_batches(content: str, filename: str) -> list: 

563 lines = content.split('\n') 

564 nlines = len(lines) 

565 

566 line_number = 1 

567 tz = timezone('Europe/Helsinki') 

568 

569 batches = [] 

570 header: Union[Dict[str, Union[int, str]], None] = None 

571 summary: Union[Dict[str, Union[int, str]], None] = None 

572 records: List[Dict[str, Union[int, str]]] = [] 

573 

574 while line_number <= nlines: 

575 line = lines[line_number-1] 

576 if line.strip() == '': 

577 line_number += 1 

578 continue 

579 record_type = line[:1] 

580 

581 if record_type in SVM_FILE_HEADER_TYPES: 

582 if header: 

583 batches.append(combine_svm_batch(header, records, summary)) 

584 header, records, summary = None, [], None 

585 header = parse_records(lines[line_number - 1], SVM_FILE_HEADER, line_number=line_number) 

586 convert_date_fields(header, SVM_FILE_HEADER_DATES, tz) 

587 line_number += 1 

588 elif record_type in SVM_FILE_RECORD_TYPES: 

589 record = parse_records(line, SVM_FILE_RECORD, line_number=line_number) 

590 convert_date_fields(record, SVM_FILE_RECORD_DATES, tz) 

591 convert_decimal_fields(record, SVM_FILE_RECORD_DECIMALS) 

592 line_number += 1 

593 records.append(record) 

594 elif record_type in SVM_FILE_SUMMARY_TYPES: 

595 summary = parse_records(line, SVM_FILE_SUMMARY, line_number=line_number) 

596 convert_decimal_fields(summary, SVM_FILE_SUMMARY_DECIMALS) 

597 line_number += 1 

598 else: 

599 raise ValidationError(_('Unknown record type on {}({}): {}').format(filename, line_number, record_type)) 

600 

601 batches.append(combine_svm_batch(header, records, summary)) 

602 return batches 

603 

604 

605def parse_svm_batches_from_file(filename: str) -> list: 

606 if parse_filename_suffix(filename).upper() not in SVM_STATEMENT_SUFFIXES: 

607 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

608 filename=filename, suffixes=', '.join(SVM_STATEMENT_SUFFIXES), file_type='saapuvat viitemaksut')) 

609 with open(filename, 'rt', encoding='ISO-8859-1') as fp: 

610 return parse_svm_batches(fp.read(), filename=basename(filename)) # type: ignore