Coverage for cc_modules/cc_convert.py : 33%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
3"""
4camcops_server/cc_modules/cc_convert.py
6===============================================================================
8 Copyright (C) 2012-2020 Rudolf Cardinal (rudolf@pobox.com).
10 This file is part of CamCOPS.
12 CamCOPS is free software: you can redistribute it and/or modify
13 it under the terms of the GNU General Public License as published by
14 the Free Software Foundation, either version 3 of the License, or
15 (at your option) any later version.
17 CamCOPS is distributed in the hope that it will be useful,
18 but WITHOUT ANY WARRANTY; without even the implied warranty of
19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 GNU General Public License for more details.
22 You should have received a copy of the GNU General Public License
23 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
25===============================================================================
27**Miscellaneous conversion functions.**
29"""
31import logging
32import re
33from typing import Any, List
35from cardinal_pythonlib.convert import (
36 base64_64format_decode,
37 base64_64format_encode,
38 hex_xformat_decode,
39 REGEX_BASE64_64FORMAT,
40 REGEX_HEX_XFORMAT,
41)
42from cardinal_pythonlib.logs import BraceStyleAdapter
43from cardinal_pythonlib.sql.literals import (
44 gen_items_from_sql_csv,
45 SQUOTE,
46 sql_dequote_string,
47 sql_quote_string,
48)
49from cardinal_pythonlib.text import escape_newlines, unescape_newlines
50from markupsafe import escape, Markup
52log = BraceStyleAdapter(logging.getLogger(__name__))
54REGEX_WHITESPACE = re.compile(r"\s")
57# =============================================================================
58# Conversion to/from quoted SQL values
59# =============================================================================
61def encode_single_value(v: Any, is_blob=False) -> str:
62 """
63 Encodes a value for incorporation into an SQL CSV value string.
65 Note that this also escapes newlines. That is not necessary when receiving
66 data from tablets, because those data arrive in CGI forms, but necessary
67 for the return journey to the tablet/webclient, because those data get sent
68 in a one-record-one-line format.
70 In the old Titanium client, the client-side counterpart to this function
71 was ``decode_single_sql_literal()`` in ``lib/conversion.js``.
73 In the newer C++ client, the client-side counterpart is
74 ``fromSqlLiteral()`` in ``lib/convert.cpp``.
76 """
77 if v is None:
78 return "NULL"
79 if is_blob:
80 return base64_64format_encode(v)
81 if isinstance(v, str):
82 return sql_quote_string(escape_newlines(v))
83 # for int, float, etc.:
84 return str(v)
87def decode_single_value(v: str) -> Any:
88 """
89 Takes a string representing an SQL value. Returns the value. Value
90 types/examples:
92 ========== ===========================================================
93 int ``35``, ``-12``
94 float ``7.23``
95 str ``'hello, here''s an apostrophe'``
96 (starts and ends with a quote)
97 NULL ``NULL``
98 (case-insensitive)
99 BLOB ``X'4D7953514C'``
100 (hex-encoded; matches MySQL method;
101 https://dev.mysql.com/doc/refman/5.0/en/hexadecimal-literals.html)
102 BLOB ``64'TXlTUUw='``
103 (base-64-encoded; this notation is my invention)
104 ========== ===========================================================
106 But
108 - we use ISO-8601 text for dates/times
110 In the old Titanium client, the client-side counterpart to this function
111 was SQLite's ``QUOTE()`` function (see ``getRecordByPK_lowmem()`` in
112 ``lib/dbsqlite.js``), except in the case of BLOBs (when it was
113 ``getEncodedBlob()`` in ``table/Blob.js``); see ``lib/dbupload.js``.
115 In the newer C++ client, the client-side counterpart is
116 ``toSqlLiteral()`` in ``lib/convert.cpp``.
118 """
120 if not v:
121 # shouldn't happen; treat it as a NULL
122 return None
123 if v.upper() == "NULL":
124 return None
126 # special BLOB encoding here
127 t = REGEX_WHITESPACE.sub("", v)
128 # t is a copy of v with all whitespace removed. We remove whitespace in
129 # some cases because some base-64 encoders insert newline characters
130 # (e.g. Titanium iOS).
131 if REGEX_HEX_XFORMAT.match(t):
132 # log.debug("MATCHES HEX-ENCODED BLOB")
133 return hex_xformat_decode(t)
134 if REGEX_BASE64_64FORMAT.match(t):
135 # log.debug("MATCHES BASE64-ENCODED BLOB")
136 return base64_64format_decode(t)
138 if len(v) >= 2 and v[0] == SQUOTE and v[-1] == SQUOTE:
139 # v is a quoted string
140 s = unescape_newlines(sql_dequote_string(v))
141 # s is the underlying string that the source started with
142 # log.debug("UNDERLYING STRING: {}", s)
143 return s
145 # Not a quoted string.
146 # int?
147 try:
148 return int(v)
149 except (TypeError, ValueError):
150 pass
151 # float?
152 try:
153 return float(v)
154 except (TypeError, ValueError):
155 pass
156 # Who knows; something odd. Allow it as a string. "Be conservative in what
157 # you send, liberal in what you accept", and all that.
158 return v
161def decode_values(valuelist: str) -> List[Any]:
162 """
163 Takes a SQL CSV value list and returns the corresponding list of decoded
164 values.
165 """
166 # log.debug("decode_values: valuelist={}", valuelist)
167 v = [decode_single_value(v) for v in gen_items_from_sql_csv(valuelist)]
168 # log.debug("decode_values: values={}", v)
169 return v
172# =============================================================================
173# Escape for HTML/XML
174# =============================================================================
176def br_html(text: str) -> str:
177 r"""
178 Filter that escapes text safely whilst also converting \n to <br>.
179 """
180 # https://stackoverflow.com/questions/2285507/converting-n-to-br-in-mako-files
181 # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/br
182 return escape(text).replace('\n', Markup('<br>'))