Coverage for jutil/format.py : 82%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import csv
2import html
3import json
4import logging
5import os
6import re
7import tempfile
8from collections import OrderedDict
9from datetime import timedelta
10from decimal import Decimal
11import subprocess
12from io import StringIO
13from typing import List, Any, Optional, Union, Dict, Sequence, Tuple, TypeVar
14from django.conf import settings
15from django.core.exceptions import ValidationError
16from django.utils.functional import lazy
17import xml.dom.minidom # type: ignore
18from django.utils.safestring import mark_safe
19from django.utils.text import capfirst
21logger = logging.getLogger(__name__)
23S = TypeVar("S")
26def format_full_name(first_name: str, last_name: str, max_length: int = 20) -> str:
27 """
28 Limits name length to specified length. Tries to keep name as human-readable an natural as possible.
29 :param first_name: First name
30 :param last_name: Last name
31 :param max_length: Maximum length
32 :return: Full name of shortened version depending on length
33 """
34 # dont allow commas in limited names
35 first_name = first_name.replace(",", " ")
36 last_name = last_name.replace(",", " ")
38 # accept short full names as is
39 original_full_name = first_name + " " + last_name
40 if len(original_full_name) <= max_length:
41 return original_full_name
43 # drop middle names
44 first_name = first_name.split(" ")[0]
45 full_name = first_name + " " + last_name
46 if len(full_name) <= max_length:
47 return full_name
49 # drop latter parts of combined first names
50 first_name = re.split(r"[\s\-]", first_name)[0]
51 full_name = first_name + " " + last_name
52 if len(full_name) <= max_length:
53 return full_name
55 # drop latter parts of multi part last names
56 last_name = re.split(r"[\s\-]", last_name)[0]
57 full_name = first_name + " " + last_name
58 if len(full_name) <= max_length:
59 return full_name
61 # shorten last name to one letter
62 last_name = last_name[:1]
64 full_name = first_name + " " + last_name
65 if len(full_name) > max_length:
66 raise Exception("Failed to shorten name {}".format(original_full_name))
67 return full_name
70def format_timedelta(
71 dt: timedelta, days_label: str = "d", hours_label: str = "h", minutes_label: str = "min", seconds_label: str = "s"
72) -> str:
73 """
74 Formats timedelta to readable format, e.g. 1h30min15s.
75 :param dt: timedelta
76 :param days_label: Label for days. Leave empty '' if value should be skipped / ignored.
77 :param hours_label: Label for hours. Leave empty '' if value should be skipped / ignored.
78 :param minutes_label: Label for minutes. Leave empty '' if value should be skipped / ignored.
79 :param seconds_label: Label for seconds. Leave empty '' if value should be skipped / ignored.
80 :return: str
81 """
82 parts = (
83 (86400, days_label),
84 (3600, hours_label),
85 (60, minutes_label),
86 (1, seconds_label),
87 )
88 out = ""
89 seconds_f = dt.total_seconds()
90 seconds = int(seconds_f)
91 for n_secs, label in parts:
92 n, remainder = divmod(seconds, n_secs)
93 if n > 0 and label:
94 out += str(n) + label
95 seconds = remainder
96 out_str = out.strip()
97 if not out_str:
98 if seconds_f >= 0.001: 98 ↛ 101line 98 didn't jump to line 101, because the condition on line 98 was never false
99 out_str = "{:0.3f}".format(int(seconds_f * 1000.0) * 0.001) + seconds_label
100 else:
101 out_str = "0" + seconds_label
102 return out_str.strip()
105def format_xml(content: str, encoding: str = "UTF-8", exceptions: bool = False) -> str:
106 """
107 Formats XML document as human-readable plain text.
108 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.
109 :param content: XML data as str
110 :param encoding: XML file encoding
111 :param exceptions: Raise exceptions on error
112 :return: str (Formatted XML str)
113 """
114 assert isinstance(content, str)
115 try:
116 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 116 ↛ 122line 116 didn't jump to line 122, because the condition on line 116 was never false
117 with tempfile.NamedTemporaryFile() as fp:
118 fp.write(content.encode(encoding=encoding))
119 fp.flush()
120 out = subprocess.check_output([settings.XMLLINT_PATH, "--format", fp.name])
121 return out.decode(encoding=encoding)
122 return xml.dom.minidom.parseString(content).toprettyxml()
123 except Exception as e:
124 logger.error("format_xml failed: %s", e)
125 if exceptions:
126 raise
127 return content
130def format_xml_bytes(content: bytes, encoding: str = "UTF-8", exceptions: bool = False) -> bytes:
131 """
132 Formats XML document as human-readable plain text and returns result in bytes.
133 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.
134 :param content: XML data as bytes
135 :param encoding: XML file encoding
136 :param exceptions: Raise exceptions on error
137 :return: bytes (Formatted XML as bytes)
138 """
139 assert isinstance(content, bytes)
140 try:
141 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 141 ↛ 147line 141 didn't jump to line 147, because the condition on line 141 was never false
142 with tempfile.NamedTemporaryFile() as fp:
143 fp.write(content)
144 fp.flush()
145 out = subprocess.check_output([settings.XMLLINT_PATH, "--format", fp.name])
146 return out
147 return xml.dom.minidom.parseString(content.decode(encoding=encoding)).toprettyxml(encoding=encoding)
148 except Exception as e:
149 logger.error("format_xml_bytes failed: %s", e)
150 if exceptions:
151 raise
152 return content
155def format_xml_file(full_path: str, encoding: str = "UTF-8", exceptions: bool = False) -> bytes:
156 """
157 Formats XML file as human-readable plain text and returns result in bytes.
158 Tries to format XML file first, if formatting fails the file content is returned as is.
159 If the file does not exist empty bytes is returned.
160 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.
161 :param full_path: Full path to XML file
162 :param encoding: XML file encoding
163 :param exceptions: Raise exceptions on error
164 :return: bytes
165 """
166 try:
167 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH:
168 return subprocess.check_output([settings.XMLLINT_PATH, "--format", full_path])
169 with open(full_path, "rb") as fp:
170 return xml.dom.minidom.parse(fp).toprettyxml(encoding=encoding) # type: ignore
171 except Exception as e:
172 logger.error("format_xml_file failed (1): %s", e)
173 if exceptions:
174 raise
175 try:
176 with open(full_path, "rb") as fp:
177 return fp.read()
178 except Exception as e:
179 logger.error("format_xml_file failed (2): %s", e)
180 return b""
183def format_as_html_json(value: Any) -> str:
184 """
185 Returns value as JSON-formatted value in HTML.
186 :param value: Any value which can be converted to JSON by json.dumps
187 :return: str
188 """
189 return mark_safe(html.escape(json.dumps(value, indent=4)).replace("\n", "<br/>").replace(" ", " "))
192def _format_dict_as_html_key(k: str) -> str:
193 if k.startswith("@"):
194 k = k[1:]
195 k = k.replace("_", " ")
196 k = re.sub(r"((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))", r" \1", k)
197 parts = k.split(" ")
198 out: List[str] = [str(capfirst(parts[0].strip()))]
199 for p in parts[1:]:
200 p2 = p.strip().lower()
201 if p2: 201 ↛ 199line 201 didn't jump to line 199, because the condition on line 201 was never false
202 out.append(p2)
203 return " ".join(out)
206def _format_dict_as_html_r(data: Dict[str, Any], margin: str = "", format_keys: bool = True) -> str:
207 if not isinstance(data, dict): 207 ↛ 208line 207 didn't jump to line 208, because the condition on line 207 was never true
208 return "{}{}\n".format(margin, data)
209 out = ""
210 for k, v in OrderedDict(sorted(data.items())).items():
211 if isinstance(v, dict):
212 out += "{}{}:\n".format(margin, _format_dict_as_html_key(k) if format_keys else k)
213 out += _format_dict_as_html_r(v, margin + " ", format_keys=format_keys)
214 out += "\n"
215 elif isinstance(v, list): 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 for v2 in v:
217 out += "{}{}:\n".format(margin, _format_dict_as_html_key(k) if format_keys else k)
218 out += _format_dict_as_html_r(v2, margin + " ", format_keys=format_keys)
219 out += "\n"
220 else:
221 out += "{}{}: {}\n".format(margin, _format_dict_as_html_key(k) if format_keys else k, v)
222 return out
225def format_dict_as_html(data: Dict[str, Any], format_keys: bool = True) -> str:
226 """
227 Formats dict to simple human readable pre-formatted html (<pre> tag).
228 :param data: dict
229 :param format_keys: Re-format 'additionalInfo' and 'additional_info' type of keys as 'Additional info'
230 :return: str (html)
231 """
232 return "<pre>" + _format_dict_as_html_r(data, format_keys=format_keys) + "</pre>"
235def format_csv(rows: List[List[Any]], dialect: str = "excel") -> str:
236 """
237 Formats rows to CSV string content.
238 :param rows: List[List[Any]]
239 :param dialect: See csv.writer dialect
240 :return: str
241 """
242 f = StringIO()
243 writer = csv.writer(f, dialect=dialect)
244 for row in rows:
245 writer.writerow(row)
246 return f.getvalue()
249def format_table( # noqa
250 rows: List[List[Any]],
251 max_col: Optional[int] = None,
252 max_line: Optional[int] = 200,
253 col_sep: str = "|",
254 row_sep: str = "-",
255 row_begin: str = "|",
256 row_end: str = "|",
257 has_label_row: bool = False,
258 left_align: Optional[List[int]] = None,
259 center_align: Optional[List[int]] = None,
260) -> str:
261 """
262 Formats "ASCII-table" rows by padding column widths to longest column value, optionally limiting column widths.
263 Optionally separates colums with ' | ' character and header row with '-' characters.
264 Supports left, right and center alignment. Useful for console apps / debugging.
266 :param rows: List[List[Any]]
267 :param max_col: Max column value width. Pass None for unlimited length.
268 :param max_line: Maximum single line length. Exceeding columns truncated. Pass None for unlimited length.
269 :param col_sep: Column separator string.
270 :param row_sep: Row separator character used before first row, end, after first row (if has_label_row).
271 :param row_begin: Row begin string, inserted before each row.
272 :param row_end: Row end string, appended after each row.
273 :param has_label_row: Set to True if table starts with column label row.
274 :param left_align: Indexes of left-aligned columns. By default all are right aligned.
275 :param center_align: Indexes of center-aligned columns. By default all are right aligned.
276 :return: str
277 """
278 # validate parameters
279 assert max_col is None or max_col > 2
280 if left_align is None:
281 left_align = []
282 if center_align is None:
283 center_align = []
284 if left_align:
285 if set(left_align) & set(center_align): 285 ↛ 286line 285 didn't jump to line 286, because the condition on line 285 was never true
286 raise ValidationError("Left align columns {} overlap with center align {}".format(left_align, center_align))
288 # find out number of columns
289 ncols = 0
290 for row in rows:
291 ncols = max(ncols, len(row))
293 # find out full-width column lengths
294 col_lens0: List[int] = [0] * ncols
295 for row in rows:
296 for ix, v in enumerate(row):
297 v = str(v)
298 col_lens0[ix] = max(col_lens0[ix], len(v))
300 # adjust max_col if needed
301 if max_line and (not max_col or sum(col_lens0) > max_line): 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true
302 max_col = max_line // ncols
304 # length limited lines and final column lengths
305 col_lens = [0] * ncols
306 lines: List[List[str]] = []
307 for row in rows:
308 line = []
309 for ix, v in enumerate(row):
310 v = str(v)
311 if max_col and len(v) > max_col:
312 v = v[: max_col - 2] + ".."
313 line.append(v)
314 col_lens[ix] = max(col_lens[ix], len(v))
315 while len(line) < ncols: 315 ↛ 316line 315 didn't jump to line 316, because the condition on line 315 was never true
316 line.append("")
317 lines.append(line)
319 # padded lines
320 lines2: List[List[str]] = []
321 for line in lines:
322 line2 = []
323 for ix, v in enumerate(line):
324 col_len = col_lens[ix]
325 if len(v) < col_len:
326 if ix in left_align:
327 v = v + " " * (col_len - len(v))
328 elif ix in center_align:
329 pad = col_len - len(v)
330 lpad = int(pad / 2)
331 rpad = pad - lpad
332 v = " " * lpad + v + " " * rpad
333 else:
334 v = " " * (col_len - len(v)) + v
335 line2.append(v)
336 lines2.append(line2)
338 # calculate max number of columns and max line length
339 max_line_len = 0
340 col_sep_len = len(col_sep)
341 ncols0 = ncols
342 for line in lines2:
343 if max_line is not None: 343 ↛ 342line 343 didn't jump to line 342, because the condition on line 343 was never false
344 line_len = len(row_begin) + sum(len(v) + col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)
345 while line_len > max_line:
346 ncols -= 1
347 line_len = len(row_begin) + sum(len(v) + col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)
348 max_line_len = max(max_line_len, line_len)
350 # find out how we should terminate lines/rows
351 line_term = ""
352 row_sep_term = ""
353 if ncols0 > ncols:
354 line_term = ".."
355 row_sep_term = row_sep * int(2 / len(row_sep))
357 # final output with row and column separators
358 lines3 = []
359 if row_sep: 359 ↛ 361line 359 didn't jump to line 361, because the condition on line 359 was never false
360 lines3.append(row_sep * max_line_len + row_sep_term)
361 for line_ix, line in enumerate(lines2):
362 while len(line) > ncols:
363 line.pop()
364 line_out = col_sep.join(line)
365 lines3.append(row_begin + line_out + row_end + line_term)
366 if line_ix == 0 and row_sep and has_label_row:
367 lines3.append(row_sep * max_line_len + row_sep_term)
368 if row_sep: 368 ↛ 370line 368 didn't jump to line 370, because the condition on line 368 was never false
369 lines3.append(row_sep * max_line_len + row_sep_term)
370 return "\n".join(lines3)
373def ucfirst(v: str) -> str:
374 """
375 Converts first character of the string to uppercase.
376 :param v: str
377 :return: str
378 """
379 return v[0:1].upper() + v[1:]
382ucfirst_lazy = lazy(ucfirst, str)
385def dec1(a: Union[float, int, Decimal, str]) -> Decimal:
386 """
387 Converts number to Decimal with 1 decimal digits.
388 :param a: Number
389 :return: Decimal with 1 decimal digits
390 """
391 return Decimal(a).quantize(Decimal("1.0"))
394def dec2(a: Union[float, int, Decimal, str]) -> Decimal:
395 """
396 Converts number to Decimal with 2 decimal digits.
397 :param a: Number
398 :return: Decimal with 2 decimal digits
399 """
400 return Decimal(a).quantize(Decimal("1.00"))
403def dec3(a: Union[float, int, Decimal, str]) -> Decimal:
404 """
405 Converts number to Decimal with 3 decimal digits.
406 :param a: Number
407 :return: Decimal with 3 decimal digits
408 """
409 return Decimal(a).quantize(Decimal("1.000"))
412def dec4(a: Union[float, int, Decimal, str]) -> Decimal:
413 """
414 Converts number to Decimal with 4 decimal digits.
415 :param a: Number
416 :return: Decimal with 4 decimal digits
417 """
418 return Decimal(a).quantize(Decimal("1.0000"))
421def dec5(a: Union[float, int, Decimal, str]) -> Decimal:
422 """
423 Converts number to Decimal with 5 decimal digits.
424 :param a: Number
425 :return: Decimal with 4 decimal digits
426 """
427 return Decimal(a).quantize(Decimal("1.00000"))
430def dec6(a: Union[float, int, Decimal, str]) -> Decimal:
431 """
432 Converts number to Decimal with 6 decimal digits.
433 :param a: Number
434 :return: Decimal with 4 decimal digits
435 """
436 return Decimal(a).quantize(Decimal("1.000000"))
439def is_media_full_path(file_path: str) -> bool:
440 """
441 Checks if file path is under (settings) MEDIA_ROOT.
442 """
443 return (
444 hasattr(settings, "MEDIA_ROOT")
445 and settings.MEDIA_ROOT
446 and os.path.isabs(file_path)
447 and os.path.realpath(file_path).startswith(str(settings.MEDIA_ROOT))
448 )
451def strip_media_root(file_path: str) -> str:
452 """
453 If file path starts with (settings) MEDIA_ROOT,
454 the MEDIA_ROOT part gets stripped and only relative path is returned.
455 Otherwise file path is returned as is. This enabled stored file names in more
456 portable format for different environment / storage.
457 If MEDIA_ROOT is missing or empty, the filename is returned as is.
458 Reverse operation of this is get_media_full_path().
459 :param file_path: str
460 :return: str
461 """
462 full_path = os.path.realpath(file_path)
463 if not is_media_full_path(full_path): 463 ↛ 464line 463 didn't jump to line 464, because the condition on line 463 was never true
464 logger.error("strip_media_root() expects absolute path under MEDIA_ROOT, got %s (%s)", file_path, full_path)
465 raise ValueError("strip_media_root() expects absolute path under MEDIA_ROOT")
466 file_path = full_path[len(settings.MEDIA_ROOT) :]
467 if file_path.startswith("/"): 467 ↛ 469line 467 didn't jump to line 469, because the condition on line 467 was never false
468 return file_path[1:]
469 return file_path
472def get_media_full_path(file_path: str) -> str:
473 """
474 Returns the absolute path from a (relative) path to (settings) MEDIA_ROOT.
475 This enabled stored file names in more portable format for different environment / storage.
476 If MEDIA_ROOT is missing or non-media path is passed to function, exception is raised.
477 Reverse operation of this is strip_media_root().
478 :param file_path: str
479 :return: str
480 """
481 full_path = (
482 os.path.realpath(file_path) if os.path.isabs(file_path) else os.path.join(settings.MEDIA_ROOT, file_path)
483 )
484 if not is_media_full_path(full_path): 484 ↛ 485line 484 didn't jump to line 485, because the condition on line 484 was never true
485 logger.error("get_media_full_path() expects relative path to MEDIA_ROOT, got %s (%s)", file_path, full_path)
486 raise ValueError("get_media_full_path() expects relative path to MEDIA_ROOT")
487 return full_path
490def camel_case_to_underscore(s: str) -> str:
491 """
492 Converts camelCaseWord to camel_case_word.
493 :param s: str
494 :return: str
495 """
496 if s: 496 ↛ 500line 496 didn't jump to line 500, because the condition on line 496 was never false
497 s = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", s)
498 s = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", s)
499 s = s.replace("-", "_")
500 return s.lower()
503def underscore_to_camel_case(s: str) -> str:
504 """
505 Converts under_score_word to underScoreWord.
506 :param s: str
507 :return: str
508 """
509 if s: 509 ↛ 512line 509 didn't jump to line 512, because the condition on line 509 was never false
510 p = s.split("_")
511 s = p[0] + "".join([ucfirst(w) for w in p[1:]])
512 return s
515def choices_label(choices: Sequence[Tuple[S, str]], value: S) -> str:
516 """
517 Iterates (value,label) list and returns label matching the choice
518 :param choices: [(choice1, label1), (choice2, label2), ...]
519 :param value: Value to find
520 :return: label or None
521 """
522 for key, label in choices: 522 ↛ 525line 522 didn't jump to line 525, because the loop on line 522 didn't complete
523 if key == value: 523 ↛ 522line 523 didn't jump to line 522, because the condition on line 523 was never false
524 return label
525 return ""