databased.databased

  1import logging
  2import os
  3import shutil
  4import sqlite3
  5from datetime import datetime
  6from functools import wraps
  7from pathlib import Path
  8from typing import Any
  9
 10from tabulate import tabulate
 11
 12
 13def _connect(func):
 14    """Decorator to open db connection if it isn't already open."""
 15
 16    @wraps(func)
 17    def inner(self, *args, **kwargs):
 18        if not self.connection_open:
 19            self.open()
 20        results = func(self, *args, **kwargs)
 21        return results
 22
 23    return inner
 24
 25
 26class DataBased:
 27    """Sqli wrapper so queries don't need to be written except table definitions.
 28
 29    Supports saving and reading dates as datetime objects.
 30
 31    Supports using a context manager."""
 32
 33    def __init__(
 34        self,
 35        dbpath: str | Path,
 36        logger_encoding: str = "utf-8",
 37        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 38    ):
 39        """
 40        :param dbpath: String or Path object to database file.
 41        If a relative path is given, it will be relative to the
 42        current working directory. The log file will be saved to the
 43        same directory.
 44
 45        :param logger_message_format: '{' style format string
 46        for the logger object."""
 47        self.dbpath = Path(dbpath)
 48        self.dbname = Path(dbpath).name
 49        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 50        self._logger_init(
 51            encoding=logger_encoding, message_format=logger_message_format
 52        )
 53        self.connection_open = False
 54        self.create_manager()
 55
 56    def __enter__(self):
 57        self.open()
 58        return self
 59
 60    def __exit__(self, exception_type, exception_value, exception_traceback):
 61        self.close()
 62
 63    def create_manager(self):
 64        """Create dbManager.py in the same directory
 65        as the database file if it doesn't exist."""
 66        manager_template = Path(__file__).parent / "dbmanager.py"
 67        manager_path = self.dbpath.parent / "dbmanager.py"
 68        if not manager_path.exists():
 69            shutil.copyfile(manager_template, manager_path)
 70
 71    def open(self):
 72        """Open connection to db."""
 73        self.connection = sqlite3.connect(
 74            self.dbpath,
 75            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 76            timeout=10,
 77        )
 78        self.connection.execute("pragma foreign_keys = 1")
 79        self.cursor = self.connection.cursor()
 80        self.connection_open = True
 81
 82    def close(self):
 83        """Save and close connection to db.
 84
 85        Call this as soon as you are done using the database if you have
 86        multiple threads or processes using the same database."""
 87        if self.connection_open:
 88            self.connection.commit()
 89            self.connection.close()
 90            self.connection_open = False
 91
 92    def _logger_init(
 93        self,
 94        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 95        encoding: str = "utf-8",
 96    ):
 97        """:param message_format: '{' style format string"""
 98        self.logger = logging.getLogger(self.dbname)
 99        if not self.logger.hasHandlers():
100            handler = logging.FileHandler(
101                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
102            )
103            handler.setFormatter(
104                logging.Formatter(
105                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
106                )
107            )
108            self.logger.addHandler(handler)
109            self.logger.setLevel(logging.INFO)
110
111    def _get_dict(
112        self, table: str, values: list, columns_to_return: list[str] = None
113    ) -> dict:
114        """Converts the values of a row into a dictionary with column names as keys.
115
116        :param table: The table that values were pulled from.
117
118        :param values: List of values expected to be the same quantity
119        and in the same order as the column names of table.
120
121        :param columns_to_return: An optional list of column names.
122        If given, only these columns will be included in the returned dictionary.
123        Otherwise all columns and values are returned."""
124        return {
125            column: value
126            for column, value in zip(self.get_column_names(table), values)
127            if not columns_to_return or column in columns_to_return
128        }
129
130    def _get_conditions(
131        self, match_criteria: list[tuple] | dict, exact_match: bool = True
132    ) -> str:
133        """Builds and returns the conditional portion of a query.
134
135        :param match_criteria: Can be a list of 2-tuples where each
136        tuple is (columnName, rowValue) or a dictionary where
137        keys are column names and values are row values.
138
139        :param exact_match: If False, the rowValue for a give column
140        will be matched as a substring.
141
142        Usage e.g.:
143
144        self.cursor.execute(f'select * from {table} where {conditions}')"""
145        if type(match_criteria) == dict:
146            match_criteria = [(k, v) for k, v in match_criteria.items()]
147        if exact_match:
148            conditions = " and ".join(
149                f'"{column_row[0]}" = "{column_row[1]}"'
150                for column_row in match_criteria
151            )
152        else:
153            conditions = " and ".join(
154                f'"{column_row[0]}" like "%{column_row[1]}%"'
155                for column_row in match_criteria
156            )
157        return f"({conditions})"
158
159    @_connect
160    def query(self, query_) -> list[Any]:
161        """Execute an arbitrary query and
162        return the results."""
163        self.cursor.execute(query_)
164        return self.cursor.fetchall()
165
166    @_connect
167    def create_tables(self, table_querys: list[str] = []):
168        """Create tables if they don't exist.
169
170        :param table_querys: Each query should be
171        in the form 'tableName(columnDefinitions)'"""
172        if len(table_querys) > 0:
173            table_names = self.get_table_names()
174            for table in table_querys:
175                if table.split("(")[0].strip() not in table_names:
176                    self.cursor.execute(f"create table {table}")
177                    self.logger.info(f'{table.split("(")[0]} table created.')
178
179    @_connect
180    def create_table(self, table: str, column_defs: list[str]):
181        """Create a table if it doesn't exist.
182
183        :param table: Name of the table to create.
184
185        :param column_defs: List of column definitions in
186        proper Sqlite3 sytax.
187        i.e. "columnName text unique" or "columnName int primary key" etc."""
188        if table not in self.get_table_names():
189            query = f"create table {table}({', '.join(column_defs)})"
190            self.cursor.execute(query)
191            self.logger.info(f"'{table}' table created.")
192
193    @_connect
194    def get_table_names(self) -> list[str]:
195        """Returns a list of table names from database."""
196        self.cursor.execute(
197            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
198        )
199        return [result[0] for result in self.cursor.fetchall()]
200
201    @_connect
202    def get_column_names(self, table: str) -> list[str]:
203        """Return a list of column names from a table."""
204        self.cursor.execute(f"select * from {table} where 1=0")
205        return [description[0] for description in self.cursor.description]
206
207    @_connect
208    def count(
209        self,
210        table: str,
211        match_criteria: list[tuple] | dict = None,
212        exact_match: bool = True,
213    ) -> int:
214        """Return number of items in table.
215
216        :param match_criteria: Can be a list of 2-tuples where each
217        tuple is (columnName, rowValue) or a dictionary where
218        keys are column names and values are row values.
219        If None, all rows from the table will be counted.
220
221        :param exact_match: If False, the row value for a give column
222        in match_criteria will be matched as a substring. Has no effect if
223        match_criteria is None.
224        """
225        query = f"select count(_rowid_) from {table}"
226        try:
227            if match_criteria:
228                self.cursor.execute(
229                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
230                )
231            else:
232                self.cursor.execute(f"{query}")
233            return self.cursor.fetchone()[0]
234        except:
235            return 0
236
237    @_connect
238    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
239        """Add row of values to table.
240
241        :param table: The table to insert into.
242
243        :param values: A tuple of values to be inserted into the table.
244
245        :param columns: If None, values param is expected to supply
246        a value for every column in the table. If columns is
247        provided, it should contain the same number of elements as values."""
248        parameterizer = ", ".join("?" for _ in values)
249        logger_values = ", ".join(str(value) for value in values)
250        try:
251            if columns:
252                columns = ", ".join(column for column in columns)
253                self.cursor.execute(
254                    f"insert into {table} ({columns}) values({parameterizer})", values
255                )
256            else:
257                self.cursor.execute(
258                    f"insert into {table} values({parameterizer})", values
259                )
260            self.logger.info(f'Added "{logger_values}" to {table} table.')
261        except Exception as e:
262            if "constraint" not in str(e).lower():
263                self.logger.exception(
264                    f'Error adding "{logger_values}" to {table} table.'
265                )
266            else:
267                self.logger.debug(str(e))
268
269    @_connect
270    def get_rows(
271        self,
272        table: str,
273        match_criteria: list[tuple] | dict = None,
274        exact_match: bool = True,
275        sort_by_column: str = None,
276        columns_to_return: list[str] = None,
277        values_only: bool = False,
278        order_by: str = None,
279        limit: str | int = None,
280    ) -> list[dict] | list[tuple]:
281        """Returns rows from table as a list of dictionaries
282        where the key-value pairs of the dictionaries are
283        column name: row value.
284
285        :param match_criteria: Can be a list of 2-tuples where each
286        tuple is (columnName, rowValue) or a dictionary where
287        keys are column names and values are row values.
288
289        :param exact_match: If False, the rowValue for a give column
290        will be matched as a substring.
291
292        :param sort_by_column: A column name to sort the results by.
293        This will sort results in Python after retrieving them from the db.
294        Use the 'order_by' param to use SQLite engine for ordering.
295
296        :param columns_to_return: Optional list of column names.
297        If provided, the elements returned by get_rows() will
298        only contain the provided columns. Otherwise every column
299        in the row is returned.
300
301        :param values_only: Return the results as a list of tuples
302        instead of a list of dictionaries that have column names as keys.
303        The results will still be sorted according to sort_by_column if
304        one is provided.
305
306        :param order_by: If given, a 'order by {order_by}' clause
307        will be added to the select query.
308
309        :param limit: If given, a 'limit {limit}' clause will be
310        added to the select query.
311        """
312        if type(columns_to_return) is str:
313            columns_to_return = [columns_to_return]
314        query = f"select * from {table}"
315        matches = []
316        if match_criteria:
317            query += f" where {self._get_conditions(match_criteria, exact_match)}"
318        if order_by:
319            query += f" order by {order_by}"
320        if limit:
321            query += f" limit {limit}"
322        query += ";"
323        self.cursor.execute(query)
324        matches = self.cursor.fetchall()
325        results = [self._get_dict(table, match, columns_to_return) for match in matches]
326        if sort_by_column:
327            results = sorted(results, key=lambda x: x[sort_by_column])
328        if values_only:
329            return [tuple(row.values()) for row in results]
330        else:
331            return results
332
333    @_connect
334    def find(
335        self, table: str, query_string: str, columns: list[str] = None
336    ) -> tuple[dict]:
337        """Search for rows that contain query_string as a substring
338        of any column.
339
340        :param table: The table to search.
341
342        :param query_string: The substring to search for in all columns.
343
344        :param columns: A list of columns to search for query_string.
345        If None, all columns in the table will be searched.
346        """
347        if type(columns) is str:
348            columns = [columns]
349        results = []
350        if not columns:
351            columns = self.get_column_names(table)
352        for column in columns:
353            results.extend(
354                [
355                    row
356                    for row in self.get_rows(
357                        table, [(column, query_string)], exact_match=False
358                    )
359                    if row not in results
360                ]
361            )
362        return results
363
364    @_connect
365    def delete(
366        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
367    ) -> int:
368        """Delete records from table.
369
370        Returns number of deleted records.
371
372        :param match_criteria: Can be a list of 2-tuples where each
373        tuple is (columnName, rowValue) or a dictionary where
374        keys are column names and values are row values.
375
376        :param exact_match: If False, the rowValue for a give column
377        will be matched as a substring.
378        """
379        num_matches = self.count(table, match_criteria, exact_match)
380        conditions = self._get_conditions(match_criteria, exact_match)
381        try:
382            self.cursor.execute(f"delete from {table} where {conditions}")
383            self.logger.info(
384                f'Deleted {num_matches} from "{table}" where {conditions}".'
385            )
386            return num_matches
387        except Exception as e:
388            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
389            return 0
390
391    @_connect
392    def update(
393        self,
394        table: str,
395        column_to_update: str,
396        new_value: Any,
397        match_criteria: list[tuple] | dict = None,
398    ) -> bool:
399        """Update row value for entry matched with match_criteria.
400
401        :param column_to_update: The column to be updated in the matched row.
402
403        :param new_value: The new value to insert.
404
405        :param match_criteria: Can be a list of 2-tuples where each
406        tuple is (columnName, rowValue) or a dictionary where
407        keys are column names and values are row values.
408        If None, every row will be updated.
409
410        Returns True if successful, False if not."""
411        query = f"update {table} set {column_to_update} = ?"
412        if match_criteria:
413            if self.count(table, match_criteria) == 0:
414                self.logger.info(
415                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
416                )
417                return False
418            conditions = self._get_conditions(match_criteria)
419            query += f" where {conditions}"
420        else:
421            conditions = None
422        try:
423            self.cursor.execute(
424                query,
425                (new_value,),
426            )
427            self.logger.info(
428                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
429            )
430            return True
431        except UnboundLocalError:
432            table_filter_string = "\n".join(
433                table_filter for table_filter in match_criteria
434            )
435            self.logger.error(
436                f"No records found matching filters: {table_filter_string}"
437            )
438            return False
439        except Exception as e:
440            self.logger.error(
441                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
442            )
443            return False
444
445    @_connect
446    def drop_table(self, table: str) -> bool:
447        """Drop a table from the database.
448
449        Returns True if successful, False if not."""
450        try:
451            self.cursor.execute(f"drop Table {table}")
452            self.logger.info(f'Dropped table "{table}"')
453        except Exception as e:
454            print(e)
455            self.logger.error(f'Failed to drop table "{table}"')
456
457    @_connect
458    def add_column(
459        self, table: str, column: str, _type: str, default_value: str = None
460    ):
461        """Add a new column to table.
462
463        :param column: Name of the column to add.
464
465        :param _type: The data type of the new column.
466
467        :param default_value: Optional default value for the column."""
468        try:
469            if default_value:
470                self.cursor.execute(
471                    f"alter table {table} add column {column} {_type} default {default_value}"
472                )
473            else:
474                self.cursor.execute(f"alter table {table} add column {column} {_type}")
475            self.logger.info(f'Added column "{column}" to "{table}" table.')
476        except Exception as e:
477            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
478
479    @staticmethod
480    def data_to_string(
481        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
482    ) -> str:
483        """Uses tabulate to produce pretty string output
484        from a list of dictionaries.
485
486        :param data: Assumes all dictionaries in list have the same set of keys.
487
488        :param sort_key: Optional dictionary key to sort data with.
489
490        :param wrap_to_terminal: If True, the table width will be wrapped
491        to fit within the current terminal window. Set to False
492        if the output is going into something like a txt file."""
493        return data_to_string(data, sort_key, wrap_to_terminal)
494
495
496def data_to_string(
497    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
498) -> str:
499    """Uses tabulate to produce pretty string output
500    from a list of dictionaries.
501
502    :param data: Assumes all dictionaries in list have the same set of keys.
503
504    :param sort_key: Optional dictionary key to sort data with.
505
506    :param wrap_to_terminal: If True, the table width will be wrapped
507    to fit within the current terminal window. Set to False
508    if the output is going into something like a txt file."""
509    if len(data) == 0:
510        return ""
511    if sort_key:
512        data = sorted(data, key=lambda d: d[sort_key])
513    for i, d in enumerate(data):
514        for k in d:
515            data[i][k] = str(data[i][k])
516    if wrap_to_terminal:
517        terminal_width = os.get_terminal_size().columns
518        max_col_widths = terminal_width
519        """ Reducing the column width by tabulating one row at a time
520        and then reducing further by tabulating the whole set proved to be 
521        faster than going straight to tabulating the whole set and reducing
522        the column width."""
523        too_wide = True
524        while too_wide and max_col_widths > 1:
525            for i, row in enumerate(data):
526                output = tabulate(
527                    [row],
528                    headers="keys",
529                    disable_numparse=True,
530                    tablefmt="grid",
531                    maxcolwidths=max_col_widths,
532                )
533                if output.index("\n") > terminal_width:
534                    max_col_widths -= 2
535                    too_wide = True
536                    break
537                too_wide = False
538    else:
539        max_col_widths = None
540    output = tabulate(
541        data,
542        headers="keys",
543        disable_numparse=True,
544        tablefmt="grid",
545        maxcolwidths=max_col_widths,
546    )
547    # trim max column width until the output string is less wide than the current terminal width.
548    if wrap_to_terminal:
549        while output.index("\n") > terminal_width and max_col_widths > 1:
550            max_col_widths -= 2
551            max_col_widths = max(1, max_col_widths)
552            output = tabulate(
553                data,
554                headers="keys",
555                disable_numparse=True,
556                tablefmt="grid",
557                maxcolwidths=max_col_widths,
558            )
559    return output
class DataBased:
 27class DataBased:
 28    """Sqli wrapper so queries don't need to be written except table definitions.
 29
 30    Supports saving and reading dates as datetime objects.
 31
 32    Supports using a context manager."""
 33
 34    def __init__(
 35        self,
 36        dbpath: str | Path,
 37        logger_encoding: str = "utf-8",
 38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 39    ):
 40        """
 41        :param dbpath: String or Path object to database file.
 42        If a relative path is given, it will be relative to the
 43        current working directory. The log file will be saved to the
 44        same directory.
 45
 46        :param logger_message_format: '{' style format string
 47        for the logger object."""
 48        self.dbpath = Path(dbpath)
 49        self.dbname = Path(dbpath).name
 50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 51        self._logger_init(
 52            encoding=logger_encoding, message_format=logger_message_format
 53        )
 54        self.connection_open = False
 55        self.create_manager()
 56
 57    def __enter__(self):
 58        self.open()
 59        return self
 60
 61    def __exit__(self, exception_type, exception_value, exception_traceback):
 62        self.close()
 63
 64    def create_manager(self):
 65        """Create dbManager.py in the same directory
 66        as the database file if it doesn't exist."""
 67        manager_template = Path(__file__).parent / "dbmanager.py"
 68        manager_path = self.dbpath.parent / "dbmanager.py"
 69        if not manager_path.exists():
 70            shutil.copyfile(manager_template, manager_path)
 71
 72    def open(self):
 73        """Open connection to db."""
 74        self.connection = sqlite3.connect(
 75            self.dbpath,
 76            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 77            timeout=10,
 78        )
 79        self.connection.execute("pragma foreign_keys = 1")
 80        self.cursor = self.connection.cursor()
 81        self.connection_open = True
 82
 83    def close(self):
 84        """Save and close connection to db.
 85
 86        Call this as soon as you are done using the database if you have
 87        multiple threads or processes using the same database."""
 88        if self.connection_open:
 89            self.connection.commit()
 90            self.connection.close()
 91            self.connection_open = False
 92
 93    def _logger_init(
 94        self,
 95        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 96        encoding: str = "utf-8",
 97    ):
 98        """:param message_format: '{' style format string"""
 99        self.logger = logging.getLogger(self.dbname)
100        if not self.logger.hasHandlers():
101            handler = logging.FileHandler(
102                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
103            )
104            handler.setFormatter(
105                logging.Formatter(
106                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
107                )
108            )
109            self.logger.addHandler(handler)
110            self.logger.setLevel(logging.INFO)
111
112    def _get_dict(
113        self, table: str, values: list, columns_to_return: list[str] = None
114    ) -> dict:
115        """Converts the values of a row into a dictionary with column names as keys.
116
117        :param table: The table that values were pulled from.
118
119        :param values: List of values expected to be the same quantity
120        and in the same order as the column names of table.
121
122        :param columns_to_return: An optional list of column names.
123        If given, only these columns will be included in the returned dictionary.
124        Otherwise all columns and values are returned."""
125        return {
126            column: value
127            for column, value in zip(self.get_column_names(table), values)
128            if not columns_to_return or column in columns_to_return
129        }
130
131    def _get_conditions(
132        self, match_criteria: list[tuple] | dict, exact_match: bool = True
133    ) -> str:
134        """Builds and returns the conditional portion of a query.
135
136        :param match_criteria: Can be a list of 2-tuples where each
137        tuple is (columnName, rowValue) or a dictionary where
138        keys are column names and values are row values.
139
140        :param exact_match: If False, the rowValue for a give column
141        will be matched as a substring.
142
143        Usage e.g.:
144
145        self.cursor.execute(f'select * from {table} where {conditions}')"""
146        if type(match_criteria) == dict:
147            match_criteria = [(k, v) for k, v in match_criteria.items()]
148        if exact_match:
149            conditions = " and ".join(
150                f'"{column_row[0]}" = "{column_row[1]}"'
151                for column_row in match_criteria
152            )
153        else:
154            conditions = " and ".join(
155                f'"{column_row[0]}" like "%{column_row[1]}%"'
156                for column_row in match_criteria
157            )
158        return f"({conditions})"
159
160    @_connect
161    def query(self, query_) -> list[Any]:
162        """Execute an arbitrary query and
163        return the results."""
164        self.cursor.execute(query_)
165        return self.cursor.fetchall()
166
167    @_connect
168    def create_tables(self, table_querys: list[str] = []):
169        """Create tables if they don't exist.
170
171        :param table_querys: Each query should be
172        in the form 'tableName(columnDefinitions)'"""
173        if len(table_querys) > 0:
174            table_names = self.get_table_names()
175            for table in table_querys:
176                if table.split("(")[0].strip() not in table_names:
177                    self.cursor.execute(f"create table {table}")
178                    self.logger.info(f'{table.split("(")[0]} table created.')
179
180    @_connect
181    def create_table(self, table: str, column_defs: list[str]):
182        """Create a table if it doesn't exist.
183
184        :param table: Name of the table to create.
185
186        :param column_defs: List of column definitions in
187        proper Sqlite3 sytax.
188        i.e. "columnName text unique" or "columnName int primary key" etc."""
189        if table not in self.get_table_names():
190            query = f"create table {table}({', '.join(column_defs)})"
191            self.cursor.execute(query)
192            self.logger.info(f"'{table}' table created.")
193
194    @_connect
195    def get_table_names(self) -> list[str]:
196        """Returns a list of table names from database."""
197        self.cursor.execute(
198            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
199        )
200        return [result[0] for result in self.cursor.fetchall()]
201
202    @_connect
203    def get_column_names(self, table: str) -> list[str]:
204        """Return a list of column names from a table."""
205        self.cursor.execute(f"select * from {table} where 1=0")
206        return [description[0] for description in self.cursor.description]
207
208    @_connect
209    def count(
210        self,
211        table: str,
212        match_criteria: list[tuple] | dict = None,
213        exact_match: bool = True,
214    ) -> int:
215        """Return number of items in table.
216
217        :param match_criteria: Can be a list of 2-tuples where each
218        tuple is (columnName, rowValue) or a dictionary where
219        keys are column names and values are row values.
220        If None, all rows from the table will be counted.
221
222        :param exact_match: If False, the row value for a give column
223        in match_criteria will be matched as a substring. Has no effect if
224        match_criteria is None.
225        """
226        query = f"select count(_rowid_) from {table}"
227        try:
228            if match_criteria:
229                self.cursor.execute(
230                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
231                )
232            else:
233                self.cursor.execute(f"{query}")
234            return self.cursor.fetchone()[0]
235        except:
236            return 0
237
238    @_connect
239    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
240        """Add row of values to table.
241
242        :param table: The table to insert into.
243
244        :param values: A tuple of values to be inserted into the table.
245
246        :param columns: If None, values param is expected to supply
247        a value for every column in the table. If columns is
248        provided, it should contain the same number of elements as values."""
249        parameterizer = ", ".join("?" for _ in values)
250        logger_values = ", ".join(str(value) for value in values)
251        try:
252            if columns:
253                columns = ", ".join(column for column in columns)
254                self.cursor.execute(
255                    f"insert into {table} ({columns}) values({parameterizer})", values
256                )
257            else:
258                self.cursor.execute(
259                    f"insert into {table} values({parameterizer})", values
260                )
261            self.logger.info(f'Added "{logger_values}" to {table} table.')
262        except Exception as e:
263            if "constraint" not in str(e).lower():
264                self.logger.exception(
265                    f'Error adding "{logger_values}" to {table} table.'
266                )
267            else:
268                self.logger.debug(str(e))
269
270    @_connect
271    def get_rows(
272        self,
273        table: str,
274        match_criteria: list[tuple] | dict = None,
275        exact_match: bool = True,
276        sort_by_column: str = None,
277        columns_to_return: list[str] = None,
278        values_only: bool = False,
279        order_by: str = None,
280        limit: str | int = None,
281    ) -> list[dict] | list[tuple]:
282        """Returns rows from table as a list of dictionaries
283        where the key-value pairs of the dictionaries are
284        column name: row value.
285
286        :param match_criteria: Can be a list of 2-tuples where each
287        tuple is (columnName, rowValue) or a dictionary where
288        keys are column names and values are row values.
289
290        :param exact_match: If False, the rowValue for a give column
291        will be matched as a substring.
292
293        :param sort_by_column: A column name to sort the results by.
294        This will sort results in Python after retrieving them from the db.
295        Use the 'order_by' param to use SQLite engine for ordering.
296
297        :param columns_to_return: Optional list of column names.
298        If provided, the elements returned by get_rows() will
299        only contain the provided columns. Otherwise every column
300        in the row is returned.
301
302        :param values_only: Return the results as a list of tuples
303        instead of a list of dictionaries that have column names as keys.
304        The results will still be sorted according to sort_by_column if
305        one is provided.
306
307        :param order_by: If given, a 'order by {order_by}' clause
308        will be added to the select query.
309
310        :param limit: If given, a 'limit {limit}' clause will be
311        added to the select query.
312        """
313        if type(columns_to_return) is str:
314            columns_to_return = [columns_to_return]
315        query = f"select * from {table}"
316        matches = []
317        if match_criteria:
318            query += f" where {self._get_conditions(match_criteria, exact_match)}"
319        if order_by:
320            query += f" order by {order_by}"
321        if limit:
322            query += f" limit {limit}"
323        query += ";"
324        self.cursor.execute(query)
325        matches = self.cursor.fetchall()
326        results = [self._get_dict(table, match, columns_to_return) for match in matches]
327        if sort_by_column:
328            results = sorted(results, key=lambda x: x[sort_by_column])
329        if values_only:
330            return [tuple(row.values()) for row in results]
331        else:
332            return results
333
334    @_connect
335    def find(
336        self, table: str, query_string: str, columns: list[str] = None
337    ) -> tuple[dict]:
338        """Search for rows that contain query_string as a substring
339        of any column.
340
341        :param table: The table to search.
342
343        :param query_string: The substring to search for in all columns.
344
345        :param columns: A list of columns to search for query_string.
346        If None, all columns in the table will be searched.
347        """
348        if type(columns) is str:
349            columns = [columns]
350        results = []
351        if not columns:
352            columns = self.get_column_names(table)
353        for column in columns:
354            results.extend(
355                [
356                    row
357                    for row in self.get_rows(
358                        table, [(column, query_string)], exact_match=False
359                    )
360                    if row not in results
361                ]
362            )
363        return results
364
365    @_connect
366    def delete(
367        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
368    ) -> int:
369        """Delete records from table.
370
371        Returns number of deleted records.
372
373        :param match_criteria: Can be a list of 2-tuples where each
374        tuple is (columnName, rowValue) or a dictionary where
375        keys are column names and values are row values.
376
377        :param exact_match: If False, the rowValue for a give column
378        will be matched as a substring.
379        """
380        num_matches = self.count(table, match_criteria, exact_match)
381        conditions = self._get_conditions(match_criteria, exact_match)
382        try:
383            self.cursor.execute(f"delete from {table} where {conditions}")
384            self.logger.info(
385                f'Deleted {num_matches} from "{table}" where {conditions}".'
386            )
387            return num_matches
388        except Exception as e:
389            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
390            return 0
391
392    @_connect
393    def update(
394        self,
395        table: str,
396        column_to_update: str,
397        new_value: Any,
398        match_criteria: list[tuple] | dict = None,
399    ) -> bool:
400        """Update row value for entry matched with match_criteria.
401
402        :param column_to_update: The column to be updated in the matched row.
403
404        :param new_value: The new value to insert.
405
406        :param match_criteria: Can be a list of 2-tuples where each
407        tuple is (columnName, rowValue) or a dictionary where
408        keys are column names and values are row values.
409        If None, every row will be updated.
410
411        Returns True if successful, False if not."""
412        query = f"update {table} set {column_to_update} = ?"
413        if match_criteria:
414            if self.count(table, match_criteria) == 0:
415                self.logger.info(
416                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
417                )
418                return False
419            conditions = self._get_conditions(match_criteria)
420            query += f" where {conditions}"
421        else:
422            conditions = None
423        try:
424            self.cursor.execute(
425                query,
426                (new_value,),
427            )
428            self.logger.info(
429                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
430            )
431            return True
432        except UnboundLocalError:
433            table_filter_string = "\n".join(
434                table_filter for table_filter in match_criteria
435            )
436            self.logger.error(
437                f"No records found matching filters: {table_filter_string}"
438            )
439            return False
440        except Exception as e:
441            self.logger.error(
442                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
443            )
444            return False
445
446    @_connect
447    def drop_table(self, table: str) -> bool:
448        """Drop a table from the database.
449
450        Returns True if successful, False if not."""
451        try:
452            self.cursor.execute(f"drop Table {table}")
453            self.logger.info(f'Dropped table "{table}"')
454        except Exception as e:
455            print(e)
456            self.logger.error(f'Failed to drop table "{table}"')
457
458    @_connect
459    def add_column(
460        self, table: str, column: str, _type: str, default_value: str = None
461    ):
462        """Add a new column to table.
463
464        :param column: Name of the column to add.
465
466        :param _type: The data type of the new column.
467
468        :param default_value: Optional default value for the column."""
469        try:
470            if default_value:
471                self.cursor.execute(
472                    f"alter table {table} add column {column} {_type} default {default_value}"
473                )
474            else:
475                self.cursor.execute(f"alter table {table} add column {column} {_type}")
476            self.logger.info(f'Added column "{column}" to "{table}" table.')
477        except Exception as e:
478            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
479
480    @staticmethod
481    def data_to_string(
482        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
483    ) -> str:
484        """Uses tabulate to produce pretty string output
485        from a list of dictionaries.
486
487        :param data: Assumes all dictionaries in list have the same set of keys.
488
489        :param sort_key: Optional dictionary key to sort data with.
490
491        :param wrap_to_terminal: If True, the table width will be wrapped
492        to fit within the current terminal window. Set to False
493        if the output is going into something like a txt file."""
494        return data_to_string(data, sort_key, wrap_to_terminal)

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathlib.Path, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}')
34    def __init__(
35        self,
36        dbpath: str | Path,
37        logger_encoding: str = "utf-8",
38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
39    ):
40        """
41        :param dbpath: String or Path object to database file.
42        If a relative path is given, it will be relative to the
43        current working directory. The log file will be saved to the
44        same directory.
45
46        :param logger_message_format: '{' style format string
47        for the logger object."""
48        self.dbpath = Path(dbpath)
49        self.dbname = Path(dbpath).name
50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
51        self._logger_init(
52            encoding=logger_encoding, message_format=logger_message_format
53        )
54        self.connection_open = False
55        self.create_manager()
Parameters
  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: '{' style format string for the logger object.

def create_manager(self):
64    def create_manager(self):
65        """Create dbManager.py in the same directory
66        as the database file if it doesn't exist."""
67        manager_template = Path(__file__).parent / "dbmanager.py"
68        manager_path = self.dbpath.parent / "dbmanager.py"
69        if not manager_path.exists():
70            shutil.copyfile(manager_template, manager_path)

Create dbManager.py in the same directory as the database file if it doesn't exist.

def open(self):
72    def open(self):
73        """Open connection to db."""
74        self.connection = sqlite3.connect(
75            self.dbpath,
76            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
77            timeout=10,
78        )
79        self.connection.execute("pragma foreign_keys = 1")
80        self.cursor = self.connection.cursor()
81        self.connection_open = True

Open connection to db.

def close(self):
83    def close(self):
84        """Save and close connection to db.
85
86        Call this as soon as you are done using the database if you have
87        multiple threads or processes using the same database."""
88        if self.connection_open:
89            self.connection.commit()
90            self.connection.close()
91            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def query(self, query_) -> list[typing.Any]:
160    @_connect
161    def query(self, query_) -> list[Any]:
162        """Execute an arbitrary query and
163        return the results."""
164        self.cursor.execute(query_)
165        return self.cursor.fetchall()

Execute an arbitrary query and return the results.

def create_tables(self, table_querys: list[str] = []):
167    @_connect
168    def create_tables(self, table_querys: list[str] = []):
169        """Create tables if they don't exist.
170
171        :param table_querys: Each query should be
172        in the form 'tableName(columnDefinitions)'"""
173        if len(table_querys) > 0:
174            table_names = self.get_table_names()
175            for table in table_querys:
176                if table.split("(")[0].strip() not in table_names:
177                    self.cursor.execute(f"create table {table}")
178                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_querys: Each query should be in the form 'tableName(columnDefinitions)'
def create_table(self, table: str, column_defs: list[str]):
180    @_connect
181    def create_table(self, table: str, column_defs: list[str]):
182        """Create a table if it doesn't exist.
183
184        :param table: Name of the table to create.
185
186        :param column_defs: List of column definitions in
187        proper Sqlite3 sytax.
188        i.e. "columnName text unique" or "columnName int primary key" etc."""
189        if table not in self.get_table_names():
190            query = f"create table {table}({', '.join(column_defs)})"
191            self.cursor.execute(query)
192            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

Parameters
  • table: Name of the table to create.

  • column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.

def get_table_names(self) -> list[str]:
194    @_connect
195    def get_table_names(self) -> list[str]:
196        """Returns a list of table names from database."""
197        self.cursor.execute(
198            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
199        )
200        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from database.

def get_column_names(self, table: str) -> list[str]:
202    @_connect
203    def get_column_names(self, table: str) -> list[str]:
204        """Return a list of column names from a table."""
205        self.cursor.execute(f"select * from {table} where 1=0")
206        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True) -> int:
208    @_connect
209    def count(
210        self,
211        table: str,
212        match_criteria: list[tuple] | dict = None,
213        exact_match: bool = True,
214    ) -> int:
215        """Return number of items in table.
216
217        :param match_criteria: Can be a list of 2-tuples where each
218        tuple is (columnName, rowValue) or a dictionary where
219        keys are column names and values are row values.
220        If None, all rows from the table will be counted.
221
222        :param exact_match: If False, the row value for a give column
223        in match_criteria will be matched as a substring. Has no effect if
224        match_criteria is None.
225        """
226        query = f"select count(_rowid_) from {table}"
227        try:
228            if match_criteria:
229                self.cursor.execute(
230                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
231                )
232            else:
233                self.cursor.execute(f"{query}")
234            return self.cursor.fetchone()[0]
235        except:
236            return 0

Return number of items in table.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

  • exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
238    @_connect
239    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
240        """Add row of values to table.
241
242        :param table: The table to insert into.
243
244        :param values: A tuple of values to be inserted into the table.
245
246        :param columns: If None, values param is expected to supply
247        a value for every column in the table. If columns is
248        provided, it should contain the same number of elements as values."""
249        parameterizer = ", ".join("?" for _ in values)
250        logger_values = ", ".join(str(value) for value in values)
251        try:
252            if columns:
253                columns = ", ".join(column for column in columns)
254                self.cursor.execute(
255                    f"insert into {table} ({columns}) values({parameterizer})", values
256                )
257            else:
258                self.cursor.execute(
259                    f"insert into {table} values({parameterizer})", values
260                )
261            self.logger.info(f'Added "{logger_values}" to {table} table.')
262        except Exception as e:
263            if "constraint" not in str(e).lower():
264                self.logger.exception(
265                    f'Error adding "{logger_values}" to {table} table.'
266                )
267            else:
268                self.logger.debug(str(e))

Add row of values to table.

Parameters
  • table: The table to insert into.

  • values: A tuple of values to be inserted into the table.

  • columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True, sort_by_column: str = None, columns_to_return: list[str] = None, values_only: bool = False, order_by: str = None, limit: str | int = None) -> list[dict] | list[tuple]:
270    @_connect
271    def get_rows(
272        self,
273        table: str,
274        match_criteria: list[tuple] | dict = None,
275        exact_match: bool = True,
276        sort_by_column: str = None,
277        columns_to_return: list[str] = None,
278        values_only: bool = False,
279        order_by: str = None,
280        limit: str | int = None,
281    ) -> list[dict] | list[tuple]:
282        """Returns rows from table as a list of dictionaries
283        where the key-value pairs of the dictionaries are
284        column name: row value.
285
286        :param match_criteria: Can be a list of 2-tuples where each
287        tuple is (columnName, rowValue) or a dictionary where
288        keys are column names and values are row values.
289
290        :param exact_match: If False, the rowValue for a give column
291        will be matched as a substring.
292
293        :param sort_by_column: A column name to sort the results by.
294        This will sort results in Python after retrieving them from the db.
295        Use the 'order_by' param to use SQLite engine for ordering.
296
297        :param columns_to_return: Optional list of column names.
298        If provided, the elements returned by get_rows() will
299        only contain the provided columns. Otherwise every column
300        in the row is returned.
301
302        :param values_only: Return the results as a list of tuples
303        instead of a list of dictionaries that have column names as keys.
304        The results will still be sorted according to sort_by_column if
305        one is provided.
306
307        :param order_by: If given, a 'order by {order_by}' clause
308        will be added to the select query.
309
310        :param limit: If given, a 'limit {limit}' clause will be
311        added to the select query.
312        """
313        if type(columns_to_return) is str:
314            columns_to_return = [columns_to_return]
315        query = f"select * from {table}"
316        matches = []
317        if match_criteria:
318            query += f" where {self._get_conditions(match_criteria, exact_match)}"
319        if order_by:
320            query += f" order by {order_by}"
321        if limit:
322            query += f" limit {limit}"
323        query += ";"
324        self.cursor.execute(query)
325        matches = self.cursor.fetchall()
326        results = [self._get_dict(table, match, columns_to_return) for match in matches]
327        if sort_by_column:
328            results = sorted(results, key=lambda x: x[sort_by_column])
329        if values_only:
330            return [tuple(row.values()) for row in results]
331        else:
332            return results

Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

  • sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.

  • columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.

  • values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.

  • order_by: If given, a 'order by {order_by}' clause will be added to the select query.

  • limit: If given, a 'limit {limit}' clause will be added to the select query.

def find( self, table: str, query_string: str, columns: list[str] = None) -> tuple[dict]:
334    @_connect
335    def find(
336        self, table: str, query_string: str, columns: list[str] = None
337    ) -> tuple[dict]:
338        """Search for rows that contain query_string as a substring
339        of any column.
340
341        :param table: The table to search.
342
343        :param query_string: The substring to search for in all columns.
344
345        :param columns: A list of columns to search for query_string.
346        If None, all columns in the table will be searched.
347        """
348        if type(columns) is str:
349            columns = [columns]
350        results = []
351        if not columns:
352            columns = self.get_column_names(table)
353        for column in columns:
354            results.extend(
355                [
356                    row
357                    for row in self.get_rows(
358                        table, [(column, query_string)], exact_match=False
359                    )
360                    if row not in results
361                ]
362            )
363        return results

Search for rows that contain query_string as a substring of any column.

Parameters
  • table: The table to search.

  • query_string: The substring to search for in all columns.

  • columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
365    @_connect
366    def delete(
367        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
368    ) -> int:
369        """Delete records from table.
370
371        Returns number of deleted records.
372
373        :param match_criteria: Can be a list of 2-tuples where each
374        tuple is (columnName, rowValue) or a dictionary where
375        keys are column names and values are row values.
376
377        :param exact_match: If False, the rowValue for a give column
378        will be matched as a substring.
379        """
380        num_matches = self.count(table, match_criteria, exact_match)
381        conditions = self._get_conditions(match_criteria, exact_match)
382        try:
383            self.cursor.execute(f"delete from {table} where {conditions}")
384            self.logger.info(
385                f'Deleted {num_matches} from "{table}" where {conditions}".'
386            )
387            return num_matches
388        except Exception as e:
389            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
390            return 0

Delete records from table.

Returns number of deleted records.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict = None) -> bool:
392    @_connect
393    def update(
394        self,
395        table: str,
396        column_to_update: str,
397        new_value: Any,
398        match_criteria: list[tuple] | dict = None,
399    ) -> bool:
400        """Update row value for entry matched with match_criteria.
401
402        :param column_to_update: The column to be updated in the matched row.
403
404        :param new_value: The new value to insert.
405
406        :param match_criteria: Can be a list of 2-tuples where each
407        tuple is (columnName, rowValue) or a dictionary where
408        keys are column names and values are row values.
409        If None, every row will be updated.
410
411        Returns True if successful, False if not."""
412        query = f"update {table} set {column_to_update} = ?"
413        if match_criteria:
414            if self.count(table, match_criteria) == 0:
415                self.logger.info(
416                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
417                )
418                return False
419            conditions = self._get_conditions(match_criteria)
420            query += f" where {conditions}"
421        else:
422            conditions = None
423        try:
424            self.cursor.execute(
425                query,
426                (new_value,),
427            )
428            self.logger.info(
429                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
430            )
431            return True
432        except UnboundLocalError:
433            table_filter_string = "\n".join(
434                table_filter for table_filter in match_criteria
435            )
436            self.logger.error(
437                f"No records found matching filters: {table_filter_string}"
438            )
439            return False
440        except Exception as e:
441            self.logger.error(
442                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
443            )
444            return False

Update row value for entry matched with match_criteria.

Parameters
  • column_to_update: The column to be updated in the matched row.

  • new_value: The new value to insert.

  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.

Returns True if successful, False if not.

def drop_table(self, table: str) -> bool:
446    @_connect
447    def drop_table(self, table: str) -> bool:
448        """Drop a table from the database.
449
450        Returns True if successful, False if not."""
451        try:
452            self.cursor.execute(f"drop Table {table}")
453            self.logger.info(f'Dropped table "{table}"')
454        except Exception as e:
455            print(e)
456            self.logger.error(f'Failed to drop table "{table}"')

Drop a table from the database.

Returns True if successful, False if not.

def add_column(self, table: str, column: str, _type: str, default_value: str = None):
458    @_connect
459    def add_column(
460        self, table: str, column: str, _type: str, default_value: str = None
461    ):
462        """Add a new column to table.
463
464        :param column: Name of the column to add.
465
466        :param _type: The data type of the new column.
467
468        :param default_value: Optional default value for the column."""
469        try:
470            if default_value:
471                self.cursor.execute(
472                    f"alter table {table} add column {column} {_type} default {default_value}"
473                )
474            else:
475                self.cursor.execute(f"alter table {table} add column {column} {_type}")
476            self.logger.info(f'Added column "{column}" to "{table}" table.')
477        except Exception as e:
478            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

Parameters
  • column: Name of the column to add.

  • _type: The data type of the new column.

  • default_value: Optional default value for the column.

@staticmethod
def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
480    @staticmethod
481    def data_to_string(
482        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
483    ) -> str:
484        """Uses tabulate to produce pretty string output
485        from a list of dictionaries.
486
487        :param data: Assumes all dictionaries in list have the same set of keys.
488
489        :param sort_key: Optional dictionary key to sort data with.
490
491        :param wrap_to_terminal: If True, the table width will be wrapped
492        to fit within the current terminal window. Set to False
493        if the output is going into something like a txt file."""
494        return data_to_string(data, sort_key, wrap_to_terminal)

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.

def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
497def data_to_string(
498    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
499) -> str:
500    """Uses tabulate to produce pretty string output
501    from a list of dictionaries.
502
503    :param data: Assumes all dictionaries in list have the same set of keys.
504
505    :param sort_key: Optional dictionary key to sort data with.
506
507    :param wrap_to_terminal: If True, the table width will be wrapped
508    to fit within the current terminal window. Set to False
509    if the output is going into something like a txt file."""
510    if len(data) == 0:
511        return ""
512    if sort_key:
513        data = sorted(data, key=lambda d: d[sort_key])
514    for i, d in enumerate(data):
515        for k in d:
516            data[i][k] = str(data[i][k])
517    if wrap_to_terminal:
518        terminal_width = os.get_terminal_size().columns
519        max_col_widths = terminal_width
520        """ Reducing the column width by tabulating one row at a time
521        and then reducing further by tabulating the whole set proved to be 
522        faster than going straight to tabulating the whole set and reducing
523        the column width."""
524        too_wide = True
525        while too_wide and max_col_widths > 1:
526            for i, row in enumerate(data):
527                output = tabulate(
528                    [row],
529                    headers="keys",
530                    disable_numparse=True,
531                    tablefmt="grid",
532                    maxcolwidths=max_col_widths,
533                )
534                if output.index("\n") > terminal_width:
535                    max_col_widths -= 2
536                    too_wide = True
537                    break
538                too_wide = False
539    else:
540        max_col_widths = None
541    output = tabulate(
542        data,
543        headers="keys",
544        disable_numparse=True,
545        tablefmt="grid",
546        maxcolwidths=max_col_widths,
547    )
548    # trim max column width until the output string is less wide than the current terminal width.
549    if wrap_to_terminal:
550        while output.index("\n") > terminal_width and max_col_widths > 1:
551            max_col_widths -= 2
552            max_col_widths = max(1, max_col_widths)
553            output = tabulate(
554                data,
555                headers="keys",
556                disable_numparse=True,
557                tablefmt="grid",
558                maxcolwidths=max_col_widths,
559            )
560    return output

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.