Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/renderer/list_of_dictionaries.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*Render a python list of dictionaries in various list and markup formats*
6:Author:
7 David Young
9:Date Created:
10 September 14, 2016
11"""
12################# GLOBAL IMPORTS ####################
13from builtins import str
14from builtins import range
15from builtins import object
16import sys
17import os
18import io
19import re
20import unicodecsv as csv
21import codecs
22import copy
23import json
24import yaml
25from decimal import Decimal
26from datetime import datetime
27os.environ['TERM'] = 'vt100'
28from fundamentals import tools
29from fundamentals.mysql import convert_dictionary_to_mysql_table
32class list_of_dictionaries(object):
33 """
34 *The dataset object is a list of python dictionaries. Using this class, the data can be rendered as various list and markup formats*
36 **Key Arguments:**
37 - ``log`` -- logger
38 - ``listOfDictionaries`` -- the list of dictionaries to render
39 - ``reDatetime`` -- a pre-compiled datetime regex. Default *False*fss
41 **Usage:**
43 To initialise the dataset object:
45 .. code-block:: python
47 dataList = [
48 {
49 "owner": "daisy",
50 "pet": "dog",
51 "address": "belfast, uk"
52 },
53 {
54 "owner": "john",
55 "pet": "snake",
56 "address": "the moon"
57 },
58 {
59 "owner": "susan",
60 "pet": "crocodile",
61 "address": "larne"
62 }
64 ]
66 from fundamentals.renderer import list_of_dictionaries
67 dataSet = list_of_dictionaries(
68 log=log,
69 listOfDictionaries=dataList
70 )
71 """
73 def __init__(
74 self,
75 log,
76 listOfDictionaries,
77 reDatetime=False
78 ):
79 self.log = log
80 self.log.debug("instansiating a new 'list_of_dictionaries' object")
81 self.listOfDictionaries = listOfDictionaries
82 self.reDatetime = reDatetime
84 return None
86 @property
87 def list(
88 self):
89 """*Returns the original list of dictionaries*
91 **Usage:**
93 dataSet.list
94 """
95 return self.listOfDictionaries
97 def csv(
98 self,
99 filepath=None
100 ):
101 """*Render the data in CSV format*
103 **Key Arguments:**
104 - ``filepath`` -- path to the file to write the csv content to. Default *None*
106 **Return:**
107 - ``renderedData`` -- the data rendered in csv format
109 **Usage:**
111 To render the data set as csv:
113 .. code-block:: python
115 print(dataSet.csv())
117 .. code-block:: text
119 owner,pet,address
120 daisy,dog,"belfast, uk"
121 john,snake,the moon
122 susan,crocodile,larne
124 and to save the csv rendering to file:
126 .. code-block:: python
128 dataSet.csv("/path/to/myfile.csv")
129 """
130 self.log.debug('starting the ``csv`` method')
132 renderedData = self._list_of_dictionaries_to_csv("machine")
134 if filepath and renderedData != "NO MATCH":
136 # RECURSIVELY CREATE MISSING DIRECTORIES
137 if not os.path.exists(os.path.dirname(filepath)):
138 os.makedirs(os.path.dirname(filepath))
140 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')
141 writeFile.write(renderedData)
142 writeFile.close()
144 self.log.debug('completed the ``csv`` method')
145 return renderedData
147 def table(
148 self,
149 filepath=None
150 ):
151 """*Render the data as a plain text table*
153 **Key Arguments:**
154 - ``filepath`` -- path to the file to write the table to. Default *None*
156 **Return:**
157 - ``renderedData`` -- the data rendered as a plain text table
159 **Usage:**
161 To render the data set as a plain text table:
163 .. code-block:: python
165 print(dataSet.table())
167 .. code-block:: text
169 +--------+------------+--------------+
170 | owner | pet | address |
171 +========+============+==============+
172 | daisy | dog | belfast, uk |
173 | john | snake | the moon |
174 | susan | crocodile | larne |
175 +--------+------------+--------------+
177 and to save the table rendering to file:
179 .. code-block:: python
181 dataSet.table("/path/to/myfile.ascii")
182 """
183 self.log.debug('starting the ``table`` method')
185 self.filepath = filepath
186 renderedData = self._list_of_dictionaries_to_csv("human")
188 if filepath and len(self.listOfDictionaries):
190 # RECURSIVELY CREATE MISSING DIRECTORIES
191 if not os.path.exists(os.path.dirname(filepath)):
192 os.makedirs(os.path.dirname(filepath))
194 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')
195 writeFile.write(renderedData)
196 writeFile.close()
198 self.log.debug('completed the ``table`` method')
199 return renderedData
201 def reST(
202 self,
203 filepath=None
204 ):
205 """*Render the data as a resturcturedText table*
207 **Key Arguments:**
208 - ``filepath`` -- path to the file to write the table to. Default *None*
210 **Return:**
211 - ``renderedData`` -- the data rendered as a resturcturedText table
213 **Usage:**
215 To render the data set as a resturcturedText table:
217 .. code-block:: python
219 print(dataSet.reST())
221 .. code-block:: text
223 +--------+------------+--------------+
224 | owner | pet | address |
225 +========+============+==============+
226 | daisy | dog | belfast, uk |
227 +--------+------------+--------------+
228 | john | snake | the moon |
229 +--------+------------+--------------+
230 | susan | crocodile | larne |
231 +--------+------------+--------------+
233 and to save the table rendering to file:
235 .. code-block:: python
237 dataSet.reST("/path/to/myfile.rst")
238 """
239 self.log.debug('starting the ``table`` method')
241 self.filepath = filepath
242 renderedData = self._list_of_dictionaries_to_csv("reST")
244 if filepath and len(self.listOfDictionaries):
246 # RECURSIVELY CREATE MISSING DIRECTORIES
247 if not os.path.exists(os.path.dirname(filepath)):
248 os.makedirs(os.path.dirname(filepath))
250 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')
251 writeFile.write(renderedData)
252 writeFile.close()
254 self.log.debug('completed the ``table`` method')
255 return renderedData
257 def markdown(
258 self,
259 filepath=None
260 ):
261 """*Render the data as a markdown table*
263 **Key Arguments:**
264 - ``filepath`` -- path to the file to write the markdown to. Default *None*
266 **Return:**
267 - ``renderedData`` -- the data rendered as a markdown table
269 **Usage:**
271 To render the data set as a markdown table:
273 .. code-block:: python
275 print(dataSet.markdown())
277 .. code-block:: markdown
279 | owner | pet | address |
280 |:-------|:-----------|:-------------|
281 | daisy | dog | belfast, uk |
282 | john | snake | the moon |
283 | susan | crocodile | larne |
285 and to save the markdown table rendering to file:
287 .. code-block:: python
289 dataSet.table("/path/to/myfile.md")
290 """
291 self.log.debug('starting the ``markdown`` method')
293 self.filepath = filepath
294 renderedData = self._list_of_dictionaries_to_csv("markdown")
296 if filepath and len(self.listOfDictionaries):
298 # RECURSIVELY CREATE MISSING DIRECTORIES
299 if not os.path.exists(os.path.dirname(filepath)):
300 os.makedirs(os.path.dirname(filepath))
302 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')
303 writeFile.write(renderedData)
304 writeFile.close()
306 self.log.debug('completed the ``markdown`` method')
307 return renderedData
309 def json(
310 self,
311 filepath=None
312 ):
313 """*Render the data in json format*
315 **Key Arguments:**
316 - ``filepath`` -- path to the file to write the json content to. Default *None*
318 **Return:**
319 - ``renderedData`` -- the data rendered as json
321 **Usage:**
323 To render the data set as json:
325 .. code-block:: python
327 print(dataSet.json())
329 .. code-block:: json
331 [
332 {
333 "address": "belfast, uk",
334 "owner": "daisy",
335 "pet": "dog"
336 },
337 {
338 "address": "the moon",
339 "owner": "john",
340 "pet": "snake"
341 },
342 {
343 "address": "larne",
344 "owner": "susan",
345 "pet": "crocodile"
346 }
347 ]
349 and to save the json rendering to file:
351 .. code-block:: python
353 dataSet.json("/path/to/myfile.json")
354 """
355 self.log.debug('starting the ``json`` method')
357 dataCopy = copy.deepcopy(self.listOfDictionaries)
358 for d in dataCopy:
359 for k, v in list(d.items()):
360 if isinstance(v, datetime):
361 d[k] = v.strftime("%Y%m%dt%H%M%S")
363 renderedData = json.dumps(
364 dataCopy,
365 separators=(',', ': '),
366 sort_keys=True,
367 indent=4
368 )
370 if filepath and len(self.listOfDictionaries):
372 # RECURSIVELY CREATE MISSING DIRECTORIES
373 if not os.path.exists(os.path.dirname(filepath)):
374 os.makedirs(os.path.dirname(filepath))
376 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')
377 writeFile.write(renderedData)
378 writeFile.close()
380 self.log.debug('completed the ``json`` method')
381 return renderedData
383 def yaml(
384 self,
385 filepath=None
386 ):
387 """*Render the data in yaml format*
389 **Key Arguments:**
390 - ``filepath`` -- path to the file to write the yaml content to. Default *None*
392 **Return:**
393 - ``renderedData`` -- the data rendered as yaml
395 **Usage:**
397 To render the data set as yaml:
399 .. code-block:: python
401 print(dataSet.yaml())
403 .. code-block:: yaml
405 - address: belfast, uk
406 owner: daisy
407 pet: dog
408 - address: the moon
409 owner: john
410 pet: snake
411 - address: larne
412 owner: susan
413 pet: crocodile
415 and to save the yaml rendering to file:
417 .. code-block:: python
419 dataSet.json("/path/to/myfile.yaml")
420 """
421 self.log.debug('starting the ``yaml`` method')
423 dataCopy = []
424 dataCopy[:] = [dict(l) for l in self.listOfDictionaries]
425 renderedData = yaml.dump(dataCopy, default_flow_style=False)
427 if filepath and len(self.listOfDictionaries):
429 # RECURSIVELY CREATE MISSING DIRECTORIES
430 if not os.path.exists(os.path.dirname(filepath)):
431 os.makedirs(os.path.dirname(filepath))
433 stream = open(filepath, 'w')
434 yaml.dump(dataCopy, stream, default_flow_style=False)
435 stream.close()
437 self.log.debug('completed the ``yaml`` method')
438 return renderedData
440 def mysql(
441 self,
442 tableName,
443 filepath=None,
444 createStatement=None
445 ):
446 """*Render the dataset as a series of mysql insert statements*
448 **Key Arguments:**
449 - ``tableName`` -- the name of the mysql db table to assign the insert statements to.
450 - ``filepath`` -- path to the file to write the mysql inserts content to. Default *None*
451 createStatement
453 **Return:**
454 - ``renderedData`` -- the data rendered mysql insert statements (string format)
456 **Usage:**
458 .. code-block:: python
460 print(dataSet.mysql("testing_table"))
462 this output the following:
464 .. code-block:: plain
466 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("belfast, uk" ,"2016-09-14T16:21:36" ,"daisy" ,"dog") ON DUPLICATE KEY UPDATE address="belfast, uk", dateCreated="2016-09-14T16:21:36", owner="daisy", pet="dog" ;
467 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("the moon" ,"2016-09-14T16:21:36" ,"john" ,"snake") ON DUPLICATE KEY UPDATE address="the moon", dateCreated="2016-09-14T16:21:36", owner="john", pet="snake" ;
468 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("larne" ,"2016-09-14T16:21:36" ,"susan" ,"crocodile") ON DUPLICATE KEY UPDATE address="larne", dateCreated="2016-09-14T16:21:36", owner="susan", pet="crocodile" ;
470 To save this rendering to file use:
472 .. code-block:: python
474 dataSet.mysql("testing_table", "/path/to/myfile.sql")
476 """
477 self.log.debug('starting the ``mysql`` method')
479 import re
480 if createStatement and "create table if not exists" not in createStatement.lower():
481 regex = re.compile(r'^\s*CREATE TABLE ', re.I | re.S)
482 createStatement = regex.sub(
483 "CREATE TABLE IF NOT EXISTS ", createStatement)
485 renderedData = self._list_of_dictionaries_to_mysql_inserts(
486 tableName=tableName,
487 createStatement=createStatement
488 )
490 if filepath and len(self.listOfDictionaries):
492 # RECURSIVELY CREATE MISSING DIRECTORIES
493 if not os.path.exists(os.path.dirname(filepath)):
494 os.makedirs(os.path.dirname(filepath))
496 writeFile = open(filepath, mode='w')
497 writeFile.write(renderedData)
498 writeFile.close()
500 self.log.debug('completed the ``mysql`` method')
501 return renderedData
503 def _list_of_dictionaries_to_csv(
504 self,
505 csvType="human"):
506 """Convert a python list of dictionaries to pretty csv output
508 **Key Arguments:**
509 - ``csvType`` -- human, machine or reST
511 **Return:**
512 - ``output`` -- the contents of a CSV file
513 """
514 self.log.debug(
515 'starting the ``_list_of_dictionaries_to_csv`` function')
517 if not len(self.listOfDictionaries):
518 return "NO MATCH"
520 dataCopy = copy.deepcopy(self.listOfDictionaries)
522 tableColumnNames = list(dataCopy[0].keys())
523 columnWidths = []
524 columnWidths[:] = [len(tableColumnNames[i])
525 for i in range(len(tableColumnNames))]
527 output = io.BytesIO()
528 # setup csv styles
529 if csvType == "machine":
530 delimiter = ","
531 elif csvType in ["human", "markdown"]:
532 delimiter = "|"
533 elif csvType in ["reST"]:
534 delimiter = "|"
535 if csvType in ["markdown"]:
536 writer = csv.writer(output, delimiter=delimiter,
537 quoting=csv.QUOTE_NONE, doublequote=False, quotechar='"', escapechar="\\", lineterminator="\n")
538 else:
539 writer = csv.writer(output, dialect='excel', delimiter=delimiter,
540 quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator="\n")
542 if csvType in ["markdown"]:
543 dividerWriter = csv.writer(
544 output, delimiter="|", quoting=csv.QUOTE_NONE, doublequote=False, quotechar='"', escapechar="\\", lineterminator="\n")
545 else:
546 dividerWriter = csv.writer(output, dialect='excel', delimiter="+",
547 quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator="\n")
548 # add column names to csv
549 header = []
550 divider = []
551 rstDivider = []
552 allRows = []
554 # clean up data
555 for row in dataCopy:
556 for c in tableColumnNames:
557 if isinstance(row[c], float) or isinstance(row[c], Decimal):
558 row[c] = "%0.9g" % row[c]
559 elif isinstance(row[c], datetime):
560 thisDate = str(row[c])[:10]
561 row[c] = "%(thisDate)s" % locals()
563 # set the column widths
564 for row in dataCopy:
565 for i, c in enumerate(tableColumnNames):
566 if len(str(row[c])) > columnWidths[i]:
567 columnWidths[i] = len(str(row[c]))
569 # table borders for human readable
570 if csvType in ["human", "markdown", "reST"]:
571 header.append("")
572 divider.append("")
573 rstDivider.append("")
575 for i, c in enumerate(tableColumnNames):
576 if csvType == "machine":
577 header.append(c)
578 elif csvType in ["human", "markdown", "reST"]:
579 header.append(
580 c.ljust(columnWidths[i] + 2).rjust(columnWidths[i] + 3))
581 divider.append('-' * (columnWidths[i] + 3))
582 rstDivider.append('=' * (columnWidths[i] + 3))
584 # table border for human readable
585 if csvType in ["human", "markdown", "reST"]:
586 header.append("")
587 divider.append("")
588 rstDivider.append("")
590 # fill in the data
591 for row in dataCopy:
592 thisRow = []
593 # table border for human readable
594 if csvType in ["human", "markdown", "reST"]:
595 thisRow.append("")
597 for i, c in enumerate(tableColumnNames):
598 if csvType in ["human", "markdown", "reST"]:
599 if row[c] == None:
600 row[c] = ""
601 row[c] = str(str(row[c]).ljust(columnWidths[i] + 2)
602 .rjust(columnWidths[i] + 3))
603 thisRow.append(row[c])
604 # table border for human readable
605 if csvType in ["human", "markdown", "reST"]:
606 thisRow.append("")
607 allRows.append(thisRow)
608 if csvType in ["reST"]:
609 allRows.append(divider)
611 if csvType == "machine":
612 writer.writerow(header)
613 if csvType in ["reST"]:
614 dividerWriter.writerow(divider)
615 writer.writerow(header)
616 dividerWriter.writerow(rstDivider)
617 if csvType in ["human"]:
618 dividerWriter.writerow(divider)
619 writer.writerow(header)
620 dividerWriter.writerow(divider)
621 elif csvType in ["markdown"]:
622 writer.writerow(header)
623 dividerWriter.writerow(divider)
625 # write out the data
626 writer.writerows(allRows)
627 # table border for human readable
628 if csvType in ["human"]:
629 dividerWriter.writerow(divider)
631 output = output.getvalue()
632 output = output.strip()
633 output = str(output)
635 if csvType in ["markdown"]:
636 output = output.replace("|--", "|:-")
637 if csvType in ["reST"]:
638 output = output.replace("|--", "+--").replace("--|", "--+")
640 self.log.debug(
641 'completed the ``_list_of_dictionaries_to_csv`` function')
643 return output
645 def _list_of_dictionaries_to_mysql_inserts(
646 self,
647 tableName,
648 createStatement=None):
649 """Convert a python list of dictionaries to pretty csv output
651 **Key Arguments:**
652 - ``tableName`` -- the name of the table to create the insert statements for
653 - ``createStatement`` -- add this create statement to the top of the file. Will only be executed if no table of that name exists in database. Default *None*
655 **Return:**
656 - ``output`` -- the mysql insert statements (as a string)
657 """
658 self.log.debug(
659 'completed the ````_list_of_dictionaries_to_mysql_inserts`` function')
661 if not len(self.listOfDictionaries):
662 return "NO MATCH"
664 dataCopy = copy.deepcopy(self.listOfDictionaries)
666 if createStatement:
667 output = createStatement + "\n"
668 else:
669 output = ""
671 inserts = []
673 inserts = []
674 inserts[:] = [convert_dictionary_to_mysql_table(log=self.log, dictionary=d, dbTableName=tableName, uniqueKeyList=[
675 ], dateModified=False, returnInsertOnly=True, replace=True, batchInserts=False, reDatetime=self.reDatetime) for d in dataCopy]
676 output += ";\n".join(inserts) + ";"
678 self.log.debug(
679 'completed the ``_list_of_dictionaries_to_mysql_inserts`` function')
680 return output