Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import csv 

2import html 

3import json 

4import logging 

5import os 

6import re 

7import tempfile 

8from collections import OrderedDict 

9from datetime import timedelta 

10from decimal import Decimal 

11import subprocess 

12from io import StringIO 

13from typing import List, Any, Optional, Union, Dict, Sequence, Tuple, TypeVar 

14from django.conf import settings 

15from django.core.exceptions import ValidationError 

16from django.utils.functional import lazy 

17import xml.dom.minidom # type: ignore 

18from django.utils.safestring import mark_safe 

19from django.utils.text import capfirst 

20 

21logger = logging.getLogger(__name__) 

22 

23S = TypeVar("S") 

24 

25 

26def format_full_name(first_name: str, last_name: str, max_length: int = 20) -> str: 

27 """ 

28 Limits name length to specified length. Tries to keep name as human-readable an natural as possible. 

29 :param first_name: First name 

30 :param last_name: Last name 

31 :param max_length: Maximum length 

32 :return: Full name of shortened version depending on length 

33 """ 

34 # dont allow commas in limited names 

35 first_name = first_name.replace(",", " ") 

36 last_name = last_name.replace(",", " ") 

37 

38 # accept short full names as is 

39 original_full_name = first_name + " " + last_name 

40 if len(original_full_name) <= max_length: 

41 return original_full_name 

42 

43 # drop middle names 

44 first_name = first_name.split(" ")[0] 

45 full_name = first_name + " " + last_name 

46 if len(full_name) <= max_length: 

47 return full_name 

48 

49 # drop latter parts of combined first names 

50 first_name = re.split(r"[\s\-]", first_name)[0] 

51 full_name = first_name + " " + last_name 

52 if len(full_name) <= max_length: 

53 return full_name 

54 

55 # drop latter parts of multi part last names 

56 last_name = re.split(r"[\s\-]", last_name)[0] 

57 full_name = first_name + " " + last_name 

58 if len(full_name) <= max_length: 

59 return full_name 

60 

61 # shorten last name to one letter 

62 last_name = last_name[:1] 

63 

64 full_name = first_name + " " + last_name 

65 if len(full_name) > max_length: 

66 raise Exception("Failed to shorten name {}".format(original_full_name)) 

67 return full_name 

68 

69 

70def format_timedelta( 

71 dt: timedelta, days_label: str = "d", hours_label: str = "h", minutes_label: str = "min", seconds_label: str = "s" 

72) -> str: 

73 """ 

74 Formats timedelta to readable format, e.g. 1h30min15s. 

75 :param dt: timedelta 

76 :param days_label: Label for days. Leave empty '' if value should be skipped / ignored. 

77 :param hours_label: Label for hours. Leave empty '' if value should be skipped / ignored. 

78 :param minutes_label: Label for minutes. Leave empty '' if value should be skipped / ignored. 

79 :param seconds_label: Label for seconds. Leave empty '' if value should be skipped / ignored. 

80 :return: str 

81 """ 

82 parts = ( 

83 (86400, days_label), 

84 (3600, hours_label), 

85 (60, minutes_label), 

86 (1, seconds_label), 

87 ) 

88 out = "" 

89 seconds_f = dt.total_seconds() 

90 seconds = int(seconds_f) 

91 for n_secs, label in parts: 

92 n, remainder = divmod(seconds, n_secs) 

93 if n > 0 and label: 

94 out += str(n) + label 

95 seconds = remainder 

96 out_str = out.strip() 

97 if not out_str: 

98 if seconds_f >= 0.001: 98 ↛ 101line 98 didn't jump to line 101, because the condition on line 98 was never false

99 out_str = "{:0.3f}".format(int(seconds_f * 1000.0) * 0.001) + seconds_label 

100 else: 

101 out_str = "0" + seconds_label 

102 return out_str.strip() 

103 

104 

105def format_xml(content: str, encoding: str = "UTF-8", exceptions: bool = False) -> str: 

106 """ 

107 Formats XML document as human-readable plain text. 

108 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used. 

109 :param content: XML data as str 

110 :param encoding: XML file encoding 

111 :param exceptions: Raise exceptions on error 

112 :return: str (Formatted XML str) 

113 """ 

114 assert isinstance(content, str) 

115 try: 

116 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 116 ↛ 122line 116 didn't jump to line 122, because the condition on line 116 was never false

117 with tempfile.NamedTemporaryFile() as fp: 

118 fp.write(content.encode(encoding=encoding)) 

119 fp.flush() 

120 out = subprocess.check_output([settings.XMLLINT_PATH, "--format", fp.name]) 

121 return out.decode(encoding=encoding) 

122 return xml.dom.minidom.parseString(content).toprettyxml() 

123 except Exception as e: 

124 logger.error("format_xml failed: %s", e) 

125 if exceptions: 

126 raise 

127 return content 

128 

129 

130def format_xml_bytes(content: bytes, encoding: str = "UTF-8", exceptions: bool = False) -> bytes: 

131 """ 

132 Formats XML document as human-readable plain text and returns result in bytes. 

133 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used. 

134 :param content: XML data as bytes 

135 :param encoding: XML file encoding 

136 :param exceptions: Raise exceptions on error 

137 :return: bytes (Formatted XML as bytes) 

138 """ 

139 assert isinstance(content, bytes) 

140 try: 

141 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 141 ↛ 147line 141 didn't jump to line 147, because the condition on line 141 was never false

142 with tempfile.NamedTemporaryFile() as fp: 

143 fp.write(content) 

144 fp.flush() 

145 out = subprocess.check_output([settings.XMLLINT_PATH, "--format", fp.name]) 

146 return out 

147 return xml.dom.minidom.parseString(content.decode(encoding=encoding)).toprettyxml(encoding=encoding) 

148 except Exception as e: 

149 logger.error("format_xml_bytes failed: %s", e) 

150 if exceptions: 

151 raise 

152 return content 

153 

154 

155def format_xml_file(full_path: str, encoding: str = "UTF-8", exceptions: bool = False) -> bytes: 

156 """ 

157 Formats XML file as human-readable plain text and returns result in bytes. 

158 Tries to format XML file first, if formatting fails the file content is returned as is. 

159 If the file does not exist empty bytes is returned. 

160 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used. 

161 :param full_path: Full path to XML file 

162 :param encoding: XML file encoding 

163 :param exceptions: Raise exceptions on error 

164 :return: bytes 

165 """ 

166 try: 

167 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 

168 return subprocess.check_output([settings.XMLLINT_PATH, "--format", full_path]) 

169 with open(full_path, "rb") as fp: 

170 return xml.dom.minidom.parse(fp).toprettyxml(encoding=encoding) # type: ignore 

171 except Exception as e: 

172 logger.error("format_xml_file failed (1): %s", e) 

173 if exceptions: 

174 raise 

175 try: 

176 with open(full_path, "rb") as fp: 

177 return fp.read() 

178 except Exception as e: 

179 logger.error("format_xml_file failed (2): %s", e) 

180 return b"" 

181 

182 

183def format_as_html_json(value: Any) -> str: 

184 """ 

185 Returns value as JSON-formatted value in HTML. 

186 :param value: Any value which can be converted to JSON by json.dumps 

187 :return: str 

188 """ 

189 return mark_safe(html.escape(json.dumps(value, indent=4)).replace("\n", "<br/>").replace(" ", "&nbsp;")) 

190 

191 

192def _format_dict_as_html_key(k: str) -> str: 

193 if k.startswith("@"): 

194 k = k[1:] 

195 k = k.replace("_", " ") 

196 k = re.sub(r"((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))", r" \1", k) 

197 parts = k.split(" ") 

198 out: List[str] = [str(capfirst(parts[0].strip()))] 

199 for p in parts[1:]: 

200 p2 = p.strip().lower() 

201 if p2: 201 ↛ 199line 201 didn't jump to line 199, because the condition on line 201 was never false

202 out.append(p2) 

203 return " ".join(out) 

204 

205 

206def _format_dict_as_html_r(data: Dict[str, Any], margin: str = "", format_keys: bool = True) -> str: 

207 if not isinstance(data, dict): 207 ↛ 208line 207 didn't jump to line 208, because the condition on line 207 was never true

208 return "{}{}\n".format(margin, data) 

209 out = "" 

210 for k, v in OrderedDict(sorted(data.items())).items(): 

211 if isinstance(v, dict): 

212 out += "{}{}:\n".format(margin, _format_dict_as_html_key(k) if format_keys else k) 

213 out += _format_dict_as_html_r(v, margin + " ", format_keys=format_keys) 

214 out += "\n" 

215 elif isinstance(v, list): 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true

216 for v2 in v: 

217 out += "{}{}:\n".format(margin, _format_dict_as_html_key(k) if format_keys else k) 

218 out += _format_dict_as_html_r(v2, margin + " ", format_keys=format_keys) 

219 out += "\n" 

220 else: 

221 out += "{}{}: {}\n".format(margin, _format_dict_as_html_key(k) if format_keys else k, v) 

222 return out 

223 

224 

225def format_dict_as_html(data: Dict[str, Any], format_keys: bool = True) -> str: 

226 """ 

227 Formats dict to simple human readable pre-formatted html (<pre> tag). 

228 :param data: dict 

229 :param format_keys: Re-format 'additionalInfo' and 'additional_info' type of keys as 'Additional info' 

230 :return: str (html) 

231 """ 

232 return "<pre>" + _format_dict_as_html_r(data, format_keys=format_keys) + "</pre>" 

233 

234 

235def format_csv(rows: List[List[Any]], dialect: str = "excel") -> str: 

236 """ 

237 Formats rows to CSV string content. 

238 :param rows: List[List[Any]] 

239 :param dialect: See csv.writer dialect 

240 :return: str 

241 """ 

242 f = StringIO() 

243 writer = csv.writer(f, dialect=dialect) 

244 for row in rows: 

245 writer.writerow(row) 

246 return f.getvalue() 

247 

248 

249def format_table( # noqa 

250 rows: List[List[Any]], 

251 max_col: Optional[int] = None, 

252 max_line: Optional[int] = 200, 

253 col_sep: str = "|", 

254 row_sep: str = "-", 

255 row_begin: str = "|", 

256 row_end: str = "|", 

257 has_label_row: bool = False, 

258 left_align: Optional[List[int]] = None, 

259 center_align: Optional[List[int]] = None, 

260) -> str: 

261 """ 

262 Formats "ASCII-table" rows by padding column widths to longest column value, optionally limiting column widths. 

263 Optionally separates colums with ' | ' character and header row with '-' characters. 

264 Supports left, right and center alignment. Useful for console apps / debugging. 

265 

266 :param rows: List[List[Any]] 

267 :param max_col: Max column value width. Pass None for unlimited length. 

268 :param max_line: Maximum single line length. Exceeding columns truncated. Pass None for unlimited length. 

269 :param col_sep: Column separator string. 

270 :param row_sep: Row separator character used before first row, end, after first row (if has_label_row). 

271 :param row_begin: Row begin string, inserted before each row. 

272 :param row_end: Row end string, appended after each row. 

273 :param has_label_row: Set to True if table starts with column label row. 

274 :param left_align: Indexes of left-aligned columns. By default all are right aligned. 

275 :param center_align: Indexes of center-aligned columns. By default all are right aligned. 

276 :return: str 

277 """ 

278 # validate parameters 

279 assert max_col is None or max_col > 2 

280 if left_align is None: 

281 left_align = [] 

282 if center_align is None: 

283 center_align = [] 

284 if left_align: 

285 if set(left_align) & set(center_align): 285 ↛ 286line 285 didn't jump to line 286, because the condition on line 285 was never true

286 raise ValidationError("Left align columns {} overlap with center align {}".format(left_align, center_align)) 

287 

288 # find out number of columns 

289 ncols = 0 

290 for row in rows: 

291 ncols = max(ncols, len(row)) 

292 

293 # find out full-width column lengths 

294 col_lens0: List[int] = [0] * ncols 

295 for row in rows: 

296 for ix, v in enumerate(row): 

297 v = str(v) 

298 col_lens0[ix] = max(col_lens0[ix], len(v)) 

299 

300 # adjust max_col if needed 

301 if max_line and (not max_col or sum(col_lens0) > max_line): 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true

302 max_col = max_line // ncols 

303 

304 # length limited lines and final column lengths 

305 col_lens = [0] * ncols 

306 lines: List[List[str]] = [] 

307 for row in rows: 

308 line = [] 

309 for ix, v in enumerate(row): 

310 v = str(v) 

311 if max_col and len(v) > max_col: 

312 v = v[: max_col - 2] + ".." 

313 line.append(v) 

314 col_lens[ix] = max(col_lens[ix], len(v)) 

315 while len(line) < ncols: 315 ↛ 316line 315 didn't jump to line 316, because the condition on line 315 was never true

316 line.append("") 

317 lines.append(line) 

318 

319 # padded lines 

320 lines2: List[List[str]] = [] 

321 for line in lines: 

322 line2 = [] 

323 for ix, v in enumerate(line): 

324 col_len = col_lens[ix] 

325 if len(v) < col_len: 

326 if ix in left_align: 

327 v = v + " " * (col_len - len(v)) 

328 elif ix in center_align: 

329 pad = col_len - len(v) 

330 lpad = int(pad / 2) 

331 rpad = pad - lpad 

332 v = " " * lpad + v + " " * rpad 

333 else: 

334 v = " " * (col_len - len(v)) + v 

335 line2.append(v) 

336 lines2.append(line2) 

337 

338 # calculate max number of columns and max line length 

339 max_line_len = 0 

340 col_sep_len = len(col_sep) 

341 ncols0 = ncols 

342 for line in lines2: 

343 if max_line is not None: 343 ↛ 342line 343 didn't jump to line 342, because the condition on line 343 was never false

344 line_len = len(row_begin) + sum(len(v) + col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end) 

345 while line_len > max_line: 

346 ncols -= 1 

347 line_len = len(row_begin) + sum(len(v) + col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end) 

348 max_line_len = max(max_line_len, line_len) 

349 

350 # find out how we should terminate lines/rows 

351 line_term = "" 

352 row_sep_term = "" 

353 if ncols0 > ncols: 

354 line_term = ".." 

355 row_sep_term = row_sep * int(2 / len(row_sep)) 

356 

357 # final output with row and column separators 

358 lines3 = [] 

359 if row_sep: 359 ↛ 361line 359 didn't jump to line 361, because the condition on line 359 was never false

360 lines3.append(row_sep * max_line_len + row_sep_term) 

361 for line_ix, line in enumerate(lines2): 

362 while len(line) > ncols: 

363 line.pop() 

364 line_out = col_sep.join(line) 

365 lines3.append(row_begin + line_out + row_end + line_term) 

366 if line_ix == 0 and row_sep and has_label_row: 

367 lines3.append(row_sep * max_line_len + row_sep_term) 

368 if row_sep: 368 ↛ 370line 368 didn't jump to line 370, because the condition on line 368 was never false

369 lines3.append(row_sep * max_line_len + row_sep_term) 

370 return "\n".join(lines3) 

371 

372 

373def ucfirst(v: str) -> str: 

374 """ 

375 Converts first character of the string to uppercase. 

376 :param v: str 

377 :return: str 

378 """ 

379 return v[0:1].upper() + v[1:] 

380 

381 

382ucfirst_lazy = lazy(ucfirst, str) 

383 

384 

385def dec1(a: Union[float, int, Decimal, str]) -> Decimal: 

386 """ 

387 Converts number to Decimal with 1 decimal digits. 

388 :param a: Number 

389 :return: Decimal with 1 decimal digits 

390 """ 

391 return Decimal(a).quantize(Decimal("1.0")) 

392 

393 

394def dec2(a: Union[float, int, Decimal, str]) -> Decimal: 

395 """ 

396 Converts number to Decimal with 2 decimal digits. 

397 :param a: Number 

398 :return: Decimal with 2 decimal digits 

399 """ 

400 return Decimal(a).quantize(Decimal("1.00")) 

401 

402 

403def dec3(a: Union[float, int, Decimal, str]) -> Decimal: 

404 """ 

405 Converts number to Decimal with 3 decimal digits. 

406 :param a: Number 

407 :return: Decimal with 3 decimal digits 

408 """ 

409 return Decimal(a).quantize(Decimal("1.000")) 

410 

411 

412def dec4(a: Union[float, int, Decimal, str]) -> Decimal: 

413 """ 

414 Converts number to Decimal with 4 decimal digits. 

415 :param a: Number 

416 :return: Decimal with 4 decimal digits 

417 """ 

418 return Decimal(a).quantize(Decimal("1.0000")) 

419 

420 

421def dec5(a: Union[float, int, Decimal, str]) -> Decimal: 

422 """ 

423 Converts number to Decimal with 5 decimal digits. 

424 :param a: Number 

425 :return: Decimal with 4 decimal digits 

426 """ 

427 return Decimal(a).quantize(Decimal("1.00000")) 

428 

429 

430def dec6(a: Union[float, int, Decimal, str]) -> Decimal: 

431 """ 

432 Converts number to Decimal with 6 decimal digits. 

433 :param a: Number 

434 :return: Decimal with 4 decimal digits 

435 """ 

436 return Decimal(a).quantize(Decimal("1.000000")) 

437 

438 

439def is_media_full_path(file_path: str) -> bool: 

440 """ 

441 Checks if file path is under (settings) MEDIA_ROOT. 

442 """ 

443 return ( 

444 hasattr(settings, "MEDIA_ROOT") 

445 and settings.MEDIA_ROOT 

446 and os.path.isabs(file_path) 

447 and os.path.realpath(file_path).startswith(str(settings.MEDIA_ROOT)) 

448 ) 

449 

450 

451def strip_media_root(file_path: str) -> str: 

452 """ 

453 If file path starts with (settings) MEDIA_ROOT, 

454 the MEDIA_ROOT part gets stripped and only relative path is returned. 

455 Otherwise file path is returned as is. This enabled stored file names in more 

456 portable format for different environment / storage. 

457 If MEDIA_ROOT is missing or empty, the filename is returned as is. 

458 Reverse operation of this is get_media_full_path(). 

459 :param file_path: str 

460 :return: str 

461 """ 

462 full_path = os.path.realpath(file_path) 

463 if not is_media_full_path(full_path): 463 ↛ 464line 463 didn't jump to line 464, because the condition on line 463 was never true

464 logger.error("strip_media_root() expects absolute path under MEDIA_ROOT, got %s (%s)", file_path, full_path) 

465 raise ValueError("strip_media_root() expects absolute path under MEDIA_ROOT") 

466 file_path = full_path[len(settings.MEDIA_ROOT) :] 

467 if file_path.startswith("/"): 467 ↛ 469line 467 didn't jump to line 469, because the condition on line 467 was never false

468 return file_path[1:] 

469 return file_path 

470 

471 

472def get_media_full_path(file_path: str) -> str: 

473 """ 

474 Returns the absolute path from a (relative) path to (settings) MEDIA_ROOT. 

475 This enabled stored file names in more portable format for different environment / storage. 

476 If MEDIA_ROOT is missing or non-media path is passed to function, exception is raised. 

477 Reverse operation of this is strip_media_root(). 

478 :param file_path: str 

479 :return: str 

480 """ 

481 full_path = ( 

482 os.path.realpath(file_path) if os.path.isabs(file_path) else os.path.join(settings.MEDIA_ROOT, file_path) 

483 ) 

484 if not is_media_full_path(full_path): 484 ↛ 485line 484 didn't jump to line 485, because the condition on line 484 was never true

485 logger.error("get_media_full_path() expects relative path to MEDIA_ROOT, got %s (%s)", file_path, full_path) 

486 raise ValueError("get_media_full_path() expects relative path to MEDIA_ROOT") 

487 return full_path 

488 

489 

490def camel_case_to_underscore(s: str) -> str: 

491 """ 

492 Converts camelCaseWord to camel_case_word. 

493 :param s: str 

494 :return: str 

495 """ 

496 if s: 496 ↛ 500line 496 didn't jump to line 500, because the condition on line 496 was never false

497 s = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", s) 

498 s = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", s) 

499 s = s.replace("-", "_") 

500 return s.lower() 

501 

502 

503def underscore_to_camel_case(s: str) -> str: 

504 """ 

505 Converts under_score_word to underScoreWord. 

506 :param s: str 

507 :return: str 

508 """ 

509 if s: 509 ↛ 512line 509 didn't jump to line 512, because the condition on line 509 was never false

510 p = s.split("_") 

511 s = p[0] + "".join([ucfirst(w) for w in p[1:]]) 

512 return s 

513 

514 

515def choices_label(choices: Sequence[Tuple[S, str]], value: S) -> str: 

516 """ 

517 Iterates (value,label) list and returns label matching the choice 

518 :param choices: [(choice1, label1), (choice2, label2), ...] 

519 :param value: Value to find 

520 :return: label or None 

521 """ 

522 for key, label in choices: 522 ↛ 525line 522 didn't jump to line 525, because the loop on line 522 didn't complete

523 if key == value: 523 ↛ 522line 523 didn't jump to line 522, because the condition on line 523 was never false

524 return label 

525 return ""