Coverage for jbank/svm.py: 90%

154 statements  

« prev     ^ index     » next       coverage.py v7.2.2, created at 2023-03-27 13:36 +0700

1from os.path import basename 

2from typing import Union, Dict, List, Optional, Any 

3 

4from django.conf import settings 

5from django.core.exceptions import ValidationError 

6from django.db import transaction 

7from django.utils.translation import gettext as _, gettext_lazy as _ 

8from jacc.models import Account, EntryType 

9from pytz import timezone 

10 

11from jbank.helpers import MESSAGE_STATEMENT_RECORD_FIELDS 

12from jbank.models import StatementFile, Statement, StatementRecord, StatementRecordSepaInfo, ReferencePaymentBatchFile, ReferencePaymentBatch, \ 

13 ReferencePaymentRecord 

14from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields 

15 

16SVM_STATEMENT_SUFFIXES = ("SVM", "TXT", "KTL") 

17 

18SVM_FILE_HEADER_DATES = (("record_date", "record_time"),) 

19 

20SVM_FILE_HEADER_TYPES = ("0",) 

21 

22SVM_FILE_HEADER = ( 

23 ("statement_type", "9(1)", "P"), 

24 ("record_date", "9(6)", "P"), 

25 ("record_time", "9(4)", "P"), 

26 ("institution_identifier", "X(2)", "P"), 

27 ("service_identifier", "X(9)", "P"), 

28 ("currency_identifier", "X(1)", "P"), 

29 ("pad01", "X(67)", "P"), 

30) 

31 

32SVM_FILE_RECORD_TYPES = ("3", "5") 

33 

34SVM_FILE_RECORD_DECIMALS = ("amount",) 

35 

36SVM_FILE_RECORD_DATES = ( 

37 "record_date", 

38 "paid_date", 

39) 

40 

41SVM_FILE_RECORD = ( 

42 ("record_type", "9(1)", "P"), # 3=viitesiirto, 5=suoraveloitus 

43 ("account_number", "9(14)", "P"), 

44 ("record_date", "9(6)", "P"), 

45 ("paid_date", "9(6)", "P"), 

46 ("archive_identifier", "X(16)", "P"), 

47 ("remittance_info", "X(20)", "P"), 

48 ("payer_name", "X(12)", "P"), 

49 ("currency_identifier", "X(1)", "P"), # 1=eur 

50 ("name_source", "X", "V"), 

51 ("amount", "9(10)", "P"), 

52 ("correction_identifier", "X", "V"), # 0=normal, 1=correction 

53 ("delivery_method", "X", "P"), # A=asiakkaalta, K=konttorista, J=pankin jarjestelmasta 

54 ("receipt_code", "X", "P"), 

55) 

56 

57SVM_FILE_SUMMARY_TYPES = ("9",) 

58 

59SVM_FILE_SUMMARY_DECIMALS = ( 

60 "record_amount", 

61 "correction_amount", 

62) 

63 

64SVM_FILE_SUMMARY = ( 

65 ("record_type", "9(1)", "P"), # 9 

66 ("record_count", "9(6)", "P"), 

67 ("record_amount", "9(11)", "P"), 

68 ("correction_count", "9(6)", "P"), 

69 ("correction_amount", "9(11)", "P"), 

70 ("pad01", "X(5)", "P"), 

71) 

72 

73 

74def parse_svm_batches_from_file(filename: str) -> list: 

75 if parse_filename_suffix(filename).upper() not in SVM_STATEMENT_SUFFIXES: 

76 raise ValidationError( 

77 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

78 filename=filename, suffixes=", ".join(SVM_STATEMENT_SUFFIXES), file_type="saapuvat viitemaksut" 

79 ) 

80 ) 

81 with open(filename, "rt", encoding="ISO-8859-1") as fp: 

82 return parse_svm_batches(fp.read(), filename=basename(filename)) # type: ignore 

83 

84 

85def parse_svm_batches(content: str, filename: str) -> list: 

86 lines = content.split("\n") 

87 nlines = len(lines) 

88 line_number = 1 

89 tz = timezone("Europe/Helsinki") 

90 batches = [] 

91 header: Optional[Dict[str, Union[int, str]]] = None 

92 records: List[Dict[str, Union[int, str]]] = [] 

93 summary: Optional[Dict[str, Union[int, str]]] = None 

94 

95 while line_number <= nlines: 

96 line = lines[line_number - 1] 

97 if line.strip() == "": 

98 line_number += 1 

99 continue 

100 record_type = line[:1] 

101 

102 if record_type in SVM_FILE_HEADER_TYPES: 

103 if header: 

104 batches.append(combine_svm_batch(header, records, summary)) 

105 header, records, summary = None, [], None 

106 header = parse_records(lines[line_number - 1], SVM_FILE_HEADER, line_number=line_number) 

107 convert_date_fields(header, SVM_FILE_HEADER_DATES, tz) 

108 line_number += 1 

109 elif record_type in SVM_FILE_RECORD_TYPES: 

110 record = parse_records(line, SVM_FILE_RECORD, line_number=line_number) 

111 convert_date_fields(record, SVM_FILE_RECORD_DATES, tz) 

112 convert_decimal_fields(record, SVM_FILE_RECORD_DECIMALS) 

113 line_number += 1 

114 records.append(record) 

115 elif record_type in SVM_FILE_SUMMARY_TYPES: 

116 summary = parse_records(line, SVM_FILE_SUMMARY, line_number=line_number) 

117 convert_decimal_fields(summary, SVM_FILE_SUMMARY_DECIMALS) 

118 line_number += 1 

119 else: 

120 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type)) 

121 

122 batches.append(combine_svm_batch(header, records, summary)) 

123 return batches 

124 

125 

126def combine_svm_batch(header: Optional[Dict[str, Any]], records: List[Dict[str, Union[int, str]]], summary: Optional[Dict[str, Any]]) -> Dict[str, Any]: 

127 data = {"header": header, "records": records} 

128 if summary is not None: 

129 data["summary"] = summary 

130 return data 

131 

132 

133@transaction.atomic # noqa 

134def create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa 

135 """ 

136 Creates Statement from statement data parsed by parse_tiliote_statements() 

137 :param statement_data: See parse_tiliote_statements 

138 :param name: File name of the account statement 

139 :param file: Source statement file 

140 :return: Statement 

141 """ 

142 if "header" not in statement_data or not statement_data["header"]: 

143 raise ValidationError("Invalid header field in statement data {}: {}".format(name, statement_data.get("header"))) 

144 header = statement_data["header"] 

145 

146 account_number = header["account_number"] 

147 if not account_number: 

148 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number="")) 

149 accounts = list(Account.objects.filter(name=account_number)) 

150 if len(accounts) != 1: 

151 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number)) 

152 account = accounts[0] 

153 assert isinstance(account, Account) 

154 

155 if Statement.objects.filter(name=name, account=account).first(): 

156 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name)) 

157 stm = Statement(name=name, account=account, file=file) 

158 for k in ASSIGNABLE_STATEMENT_HEADER_FIELDS: 

159 if k in header: 

160 setattr(stm, k, header[k]) 

161 # pprint(statement_data['header']) 

162 for k, v in kw.items(): 

163 setattr(stm, k, v) 

164 stm.full_clean() 

165 stm.save() 

166 

167 if EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).count() == 0: 

168 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT)) 

169 if EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).count() == 0: 

170 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW)) 

171 entry_types = { 

172 "1": EntryType.objects.get(code=settings.E_BANK_DEPOSIT), 

173 "2": EntryType.objects.get(code=settings.E_BANK_WITHDRAW), 

174 } 

175 

176 for rec_data in statement_data["records"]: 

177 line_number = rec_data["line_number"] 

178 e_type = entry_types.get(rec_data["entry_type"]) 

179 rec = StatementRecord(statement=stm, account=account, type=e_type, line_number=line_number) 

180 for k in ASSIGNABLE_STATEMENT_RECORD_FIELDS: 

181 if k in rec_data: 

182 setattr(rec, k, rec_data[k]) 

183 for k in MESSAGE_STATEMENT_RECORD_FIELDS: 

184 if k in rec_data: 

185 setattr(rec, k, "\n".join(rec_data[k])) 

186 rec.full_clean() 

187 rec.save() 

188 

189 if "sepa" in rec_data: 

190 sepa_info_data = rec_data["sepa"] 

191 sepa_info = StatementRecordSepaInfo(record=rec) 

192 for k in ASSIGNABLE_STATEMENT_RECORD_SEPA_INFO_FIELDS: 

193 if k in sepa_info_data: 

194 setattr(sepa_info, k, sepa_info_data[k]) 

195 # pprint(rec_data['sepa']) 

196 sepa_info.full_clean() 

197 sepa_info.save() 

198 

199 return stm 

200 

201 

202@transaction.atomic 

203def create_reference_payment_batch(batch_data: dict, name: str, file: ReferencePaymentBatchFile, **kw) -> ReferencePaymentBatch: 

204 """ 

205 Creates ReferencePaymentBatch from data parsed by parse_svm_batches() 

206 :param batch_data: See parse_svm_batches 

207 :param name: File name of the batch file 

208 :return: ReferencePaymentBatch 

209 """ 

210 if ReferencePaymentBatch.objects.exclude(file=file).filter(name=name).first(): 

211 raise ValidationError("Reference payment batch file {} already exists".format(name)) 

212 

213 if "header" not in batch_data or not batch_data["header"]: 

214 raise ValidationError("Invalid header field in reference payment batch data {}: {}".format(name, batch_data.get("header"))) 

215 header = batch_data["header"] 

216 

217 batch = ReferencePaymentBatch(name=name, file=file) 

218 for k in ASSIGNABLE_REFERENCE_PAYMENT_BATCH_HEADER_FIELDS: 

219 if k in header: 

220 setattr(batch, k, header[k]) 

221 # pprint(statement_data['header']) 

222 for k, v in kw.items(): 

223 setattr(batch, k, v) 

224 batch.full_clean() 

225 batch.save() 

226 e_type = EntryType.objects.get(code=settings.E_BANK_REFERENCE_PAYMENT) 

227 

228 for rec_data in batch_data["records"]: 

229 line_number = rec_data["line_number"] 

230 account_number = rec_data["account_number"] 

231 if not account_number: 

232 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number="")) 

233 accounts = list(Account.objects.filter(name=account_number)) 

234 if len(accounts) != 1: 

235 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number)) 

236 account = accounts[0] 

237 assert isinstance(account, Account) 

238 

239 rec = ReferencePaymentRecord(batch=batch, account=account, type=e_type, line_number=line_number) 

240 for k in ASSIGNABLE_REFERENCE_PAYMENT_RECORD_FIELDS: 

241 if k in rec_data: 

242 setattr(rec, k, rec_data[k]) 

243 # pprint(rec_data) 

244 rec.full_clean() 

245 rec.save() 

246 

247 return batch 

248 

249 

250ASSIGNABLE_REFERENCE_PAYMENT_RECORD_FIELDS = ( 

251 "record_type", 

252 "account_number", 

253 "record_date", 

254 "paid_date", 

255 "archive_identifier", 

256 "remittance_info", 

257 "payer_name", 

258 "currency_identifier", 

259 "name_source", 

260 "amount", 

261 "correction_identifier", 

262 "delivery_method", 

263 "receipt_code", 

264) 

265ASSIGNABLE_REFERENCE_PAYMENT_BATCH_HEADER_FIELDS = ( 

266 "record_date", 

267 "institution_identifier", 

268 "service_identifier", 

269 "currency_identifier", 

270) 

271ASSIGNABLE_STATEMENT_RECORD_SEPA_INFO_FIELDS = ( 

272 "reference", 

273 "iban_account_number", 

274 "bic_code", 

275 "recipient_name_detail", 

276 "payer_name_detail", 

277 "identifier", 

278 "archive_identifier", 

279) 

280ASSIGNABLE_STATEMENT_RECORD_FIELDS = ( 

281 "record_date", 

282 "value_date", 

283 "paid_date", 

284 "record_number", 

285 "archive_identifier", 

286 "entry_type", 

287 "record_code", 

288 "record_description", 

289 "amount", 

290 "receipt_code", 

291 "delivery_method", 

292 "name", 

293 "name_source", 

294 "recipient_account_number", 

295 "recipient_account_number_changed", 

296 "remittance_info", 

297) 

298ASSIGNABLE_STATEMENT_HEADER_FIELDS = ( 

299 "account_number", 

300 "statement_number", 

301 "begin_date", 

302 "end_date", 

303 "record_date", 

304 "customer_identifier", 

305 "begin_balance_date", 

306 "begin_balance", 

307 "record_count", 

308 "currency_code", 

309 "account_name", 

310 "account_limit", 

311 "owner_name", 

312 "contact_info_1", 

313 "contact_info_2", 

314 "bank_specific_info_1", 

315 "iban", 

316 "bic", 

317)