databased.databased

  1import logging
  2import os
  3import shutil
  4import sqlite3
  5from datetime import datetime
  6from functools import wraps
  7from pathlib import Path
  8from typing import Any
  9
 10import pandas
 11from tabulate import tabulate
 12
 13
 14def _connect(func):
 15    """Decorator to open db connection if it isn't already open."""
 16
 17    @wraps(func)
 18    def inner(self, *args, **kwargs):
 19        if not self.connection_open:
 20            self.open()
 21        results = func(self, *args, **kwargs)
 22        return results
 23
 24    return inner
 25
 26
 27class DataBased:
 28    """Sqli wrapper so queries don't need to be written except table definitions.
 29
 30    Supports saving and reading dates as datetime objects.
 31
 32    Supports using a context manager."""
 33
 34    def __init__(
 35        self,
 36        dbpath: str | Path,
 37        logger_encoding: str = "utf-8",
 38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 39    ):
 40        """
 41        :param dbpath: String or Path object to database file.
 42        If a relative path is given, it will be relative to the
 43        current working directory. The log file will be saved to the
 44        same directory.
 45
 46        :param logger_message_format: '{' style format string
 47        for the logger object."""
 48        self.dbpath = Path(dbpath)
 49        self.dbname = Path(dbpath).name
 50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 51        self._logger_init(
 52            encoding=logger_encoding, message_format=logger_message_format
 53        )
 54        self.connection_open = False
 55        self.create_manager()
 56
 57    def __enter__(self):
 58        self.open()
 59        return self
 60
 61    def __exit__(self, exception_type, exception_value, exception_traceback):
 62        self.close()
 63
 64    def create_manager(self):
 65        """Create dbManager.py in the same directory
 66        as the database file if it doesn't exist."""
 67        manager_template = Path(__file__).parent / "dbmanager.py"
 68        manager_path = self.dbpath.parent / "dbmanager.py"
 69        if not manager_path.exists():
 70            shutil.copyfile(manager_template, manager_path)
 71
 72    def open(self):
 73        """Open connection to db."""
 74        self.connection = sqlite3.connect(
 75            self.dbpath,
 76            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 77            timeout=10,
 78        )
 79        self.connection.execute("pragma foreign_keys = 1")
 80        self.cursor = self.connection.cursor()
 81        self.connection_open = True
 82
 83    def close(self):
 84        """Save and close connection to db.
 85
 86        Call this as soon as you are done using the database if you have
 87        multiple threads or processes using the same database."""
 88        if self.connection_open:
 89            self.connection.commit()
 90            self.connection.close()
 91            self.connection_open = False
 92
 93    def _logger_init(
 94        self,
 95        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 96        encoding: str = "utf-8",
 97    ):
 98        """:param message_format: '{' style format string"""
 99        self.logger = logging.getLogger(self.dbname)
100        if not self.logger.hasHandlers():
101            handler = logging.FileHandler(
102                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
103            )
104            handler.setFormatter(
105                logging.Formatter(
106                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
107                )
108            )
109            self.logger.addHandler(handler)
110            self.logger.setLevel(logging.INFO)
111
112    def _get_dict(
113        self, table: str, values: list, columns_to_return: list[str] = None
114    ) -> dict:
115        """Converts the values of a row into a dictionary with column names as keys.
116
117        :param table: The table that values were pulled from.
118
119        :param values: List of values expected to be the same quantity
120        and in the same order as the column names of table.
121
122        :param columns_to_return: An optional list of column names.
123        If given, only these columns will be included in the returned dictionary.
124        Otherwise all columns and values are returned."""
125        return {
126            column: value
127            for column, value in zip(self.get_column_names(table), values)
128            if not columns_to_return or column in columns_to_return
129        }
130
131    def _get_conditions(
132        self, match_criteria: list[tuple] | dict, exact_match: bool = True
133    ) -> str:
134        """Builds and returns the conditional portion of a query.
135
136        :param match_criteria: Can be a list of 2-tuples where each
137        tuple is (columnName, rowValue) or a dictionary where
138        keys are column names and values are row values.
139
140        :param exact_match: If False, the rowValue for a give column
141        will be matched as a substring.
142
143        Usage e.g.:
144
145        self.cursor.execute(f'select * from {table} where {conditions}')"""
146        if type(match_criteria) == dict:
147            match_criteria = [(k, v) for k, v in match_criteria.items()]
148        if exact_match:
149            conditions = " and ".join(
150                f'"{column_row[0]}" = "{column_row[1]}"'
151                for column_row in match_criteria
152            )
153        else:
154            conditions = " and ".join(
155                f'"{column_row[0]}" like "%{column_row[1]}%"'
156                for column_row in match_criteria
157            )
158        return f"({conditions})"
159
160    @_connect
161    def query(self, query_) -> list[Any]:
162        """Execute an arbitrary query and
163        return the results."""
164        self.cursor.execute(query_)
165        return self.cursor.fetchall()
166
167    @_connect
168    def create_tables(self, table_querys: list[str] = []):
169        """Create tables if they don't exist.
170
171        :param table_querys: Each query should be
172        in the form 'tableName(columnDefinitions)'"""
173        if len(table_querys) > 0:
174            table_names = self.get_table_names()
175            for table in table_querys:
176                if table.split("(")[0].strip() not in table_names:
177                    self.cursor.execute(f"create table {table}")
178                    self.logger.info(f'{table.split("(")[0]} table created.')
179
180    @_connect
181    def create_table(self, table: str, column_defs: list[str]):
182        """Create a table if it doesn't exist.
183
184        :param table: Name of the table to create.
185
186        :param column_defs: List of column definitions in
187        proper Sqlite3 sytax.
188        i.e. "columnName text unique" or "columnName int primary key" etc."""
189        if table not in self.get_table_names():
190            query = f"create table {table}({', '.join(column_defs)})"
191            self.cursor.execute(query)
192            self.logger.info(f"'{table}' table created.")
193
194    @_connect
195    def get_table_names(self) -> list[str]:
196        """Returns a list of table names from database."""
197        self.cursor.execute(
198            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
199        )
200        return [result[0] for result in self.cursor.fetchall()]
201
202    @_connect
203    def get_column_names(self, table: str) -> list[str]:
204        """Return a list of column names from a table."""
205        self.cursor.execute(f"select * from {table} where 1=0")
206        return [description[0] for description in self.cursor.description]
207
208    @_connect
209    def count(
210        self,
211        table: str,
212        match_criteria: list[tuple] | dict = None,
213        exact_match: bool = True,
214    ) -> int:
215        """Return number of items in table.
216
217        :param match_criteria: Can be a list of 2-tuples where each
218        tuple is (columnName, rowValue) or a dictionary where
219        keys are column names and values are row values.
220        If None, all rows from the table will be counted.
221
222        :param exact_match: If False, the row value for a give column
223        in match_criteria will be matched as a substring. Has no effect if
224        match_criteria is None.
225        """
226        query = f"select count(_rowid_) from {table}"
227        try:
228            if match_criteria:
229                self.cursor.execute(
230                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
231                )
232            else:
233                self.cursor.execute(f"{query}")
234            return self.cursor.fetchone()[0]
235        except:
236            return 0
237
238    @_connect
239    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
240        """Add row of values to table.
241
242        :param table: The table to insert into.
243
244        :param values: A tuple of values to be inserted into the table.
245
246        :param columns: If None, values param is expected to supply
247        a value for every column in the table. If columns is
248        provided, it should contain the same number of elements as values."""
249        parameterizer = ", ".join("?" for _ in values)
250        logger_values = ", ".join(str(value) for value in values)
251        try:
252            if columns:
253                columns = ", ".join(column for column in columns)
254                self.cursor.execute(
255                    f"insert into {table} ({columns}) values({parameterizer})", values
256                )
257            else:
258                self.cursor.execute(
259                    f"insert into {table} values({parameterizer})", values
260                )
261            self.logger.info(f'Added "{logger_values}" to {table} table.')
262        except Exception as e:
263            if "constraint" not in str(e).lower():
264                self.logger.exception(
265                    f'Error adding "{logger_values}" to {table} table.'
266                )
267            else:
268                self.logger.debug(str(e))
269
270    @_connect
271    def get_rows(
272        self,
273        table: str,
274        match_criteria: list[tuple] | dict = None,
275        exact_match: bool = True,
276        sort_by_column: str = None,
277        columns_to_return: list[str] = None,
278        return_as_dataframe: bool = False,
279        values_only: bool = False,
280        order_by: str = None,
281        limit: str | int = None,
282    ) -> list[dict] | list[tuple] | pandas.DataFrame:
283        """Returns rows from table as a list of dictionaries
284        where the key-value pairs of the dictionaries are
285        column name: row value.
286
287        :param match_criteria: Can be a list of 2-tuples where each
288        tuple is (columnName, rowValue) or a dictionary where
289        keys are column names and values are row values.
290
291        :param exact_match: If False, the rowValue for a give column
292        will be matched as a substring.
293
294        :param sort_by_column: A column name to sort the results by.
295        This will sort results in Python after retrieving them from the db.
296        Use the 'order_by' param to use SQLite engine for ordering.
297
298        :param columns_to_return: Optional list of column names.
299        If provided, the elements returned by get_rows() will
300        only contain the provided columns. Otherwise every column
301        in the row is returned.
302
303        :param return_as_dataframe: If True,
304        the results will be returned as a pandas.DataFrame object.
305
306        :param values_only: Return the results as a list of tuples
307        instead of a list of dictionaries that have column names as keys.
308        The results will still be sorted according to sort_by_column if
309        one is provided.
310
311        :param order_by: If given, a 'order by {order_by}' clause
312        will be added to the select query.
313
314        :param limit: If given, a 'limit {limit}' clause will be
315        added to the select query.
316        """
317
318        if type(columns_to_return) is str:
319            columns_to_return = [columns_to_return]
320        query = f"select * from {table}"
321        matches = []
322        if match_criteria:
323            query += f" where {self._get_conditions(match_criteria, exact_match)}"
324        if order_by:
325            query += f" order by {order_by}"
326        if limit:
327            query += f" limit {limit}"
328        query += ";"
329        self.cursor.execute(query)
330        matches = self.cursor.fetchall()
331        results = [self._get_dict(table, match, columns_to_return) for match in matches]
332        if sort_by_column:
333            results = sorted(results, key=lambda x: x[sort_by_column])
334        if return_as_dataframe:
335            return pandas.DataFrame(results)
336        if values_only:
337            return [tuple(row.values()) for row in results]
338        else:
339            return results
340
341    @_connect
342    def find(
343        self, table: str, query_string: str, columns: list[str] = None
344    ) -> tuple[dict]:
345        """Search for rows that contain query_string as a substring
346        of any column.
347
348        :param table: The table to search.
349
350        :param query_string: The substring to search for in all columns.
351
352        :param columns: A list of columns to search for query_string.
353        If None, all columns in the table will be searched.
354        """
355        if type(columns) is str:
356            columns = [columns]
357        results = []
358        if not columns:
359            columns = self.get_column_names(table)
360        for column in columns:
361            results.extend(
362                [
363                    row
364                    for row in self.get_rows(
365                        table, [(column, query_string)], exact_match=False
366                    )
367                    if row not in results
368                ]
369            )
370        return results
371
372    @_connect
373    def delete(
374        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
375    ) -> int:
376        """Delete records from table.
377
378        Returns number of deleted records.
379
380        :param match_criteria: Can be a list of 2-tuples where each
381        tuple is (columnName, rowValue) or a dictionary where
382        keys are column names and values are row values.
383
384        :param exact_match: If False, the rowValue for a give column
385        will be matched as a substring.
386        """
387        num_matches = self.count(table, match_criteria, exact_match)
388        conditions = self._get_conditions(match_criteria, exact_match)
389        try:
390            self.cursor.execute(f"delete from {table} where {conditions}")
391            self.logger.info(
392                f'Deleted {num_matches} from "{table}" where {conditions}".'
393            )
394            return num_matches
395        except Exception as e:
396            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
397            return 0
398
399    @_connect
400    def update(
401        self,
402        table: str,
403        column_to_update: str,
404        new_value: Any,
405        match_criteria: list[tuple] | dict = None,
406    ) -> bool:
407        """Update row value for entry matched with match_criteria.
408
409        :param column_to_update: The column to be updated in the matched row.
410
411        :param new_value: The new value to insert.
412
413        :param match_criteria: Can be a list of 2-tuples where each
414        tuple is (columnName, rowValue) or a dictionary where
415        keys are column names and values are row values.
416        If None, every row will be updated.
417
418        Returns True if successful, False if not."""
419        query = f"update {table} set {column_to_update} = ?"
420        if match_criteria:
421            if self.count(table, match_criteria) == 0:
422                self.logger.info(
423                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
424                )
425                return False
426            conditions = self._get_conditions(match_criteria)
427            query += f" where {conditions}"
428        else:
429            conditions = None
430        try:
431            self.cursor.execute(
432                query,
433                (new_value,),
434            )
435            self.logger.info(
436                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
437            )
438            return True
439        except UnboundLocalError:
440            table_filter_string = "\n".join(
441                table_filter for table_filter in match_criteria
442            )
443            self.logger.error(
444                f"No records found matching filters: {table_filter_string}"
445            )
446            return False
447        except Exception as e:
448            self.logger.error(
449                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
450            )
451            return False
452
453    @_connect
454    def drop_table(self, table: str) -> bool:
455        """Drop a table from the database.
456
457        Returns True if successful, False if not."""
458        try:
459            self.cursor.execute(f"drop Table {table}")
460            self.logger.info(f'Dropped table "{table}"')
461        except Exception as e:
462            print(e)
463            self.logger.error(f'Failed to drop table "{table}"')
464
465    @_connect
466    def add_column(
467        self, table: str, column: str, _type: str, default_value: str = None
468    ):
469        """Add a new column to table.
470
471        :param column: Name of the column to add.
472
473        :param _type: The data type of the new column.
474
475        :param default_value: Optional default value for the column."""
476        try:
477            if default_value:
478                self.cursor.execute(
479                    f"alter table {table} add column {column} {_type} default {default_value}"
480                )
481            else:
482                self.cursor.execute(f"alter table {table} add column {column} {_type}")
483            self.logger.info(f'Added column "{column}" to "{table}" table.')
484        except Exception as e:
485            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
486
487    @staticmethod
488    def data_to_string(
489        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
490    ) -> str:
491        """Uses tabulate to produce pretty string output
492        from a list of dictionaries.
493
494        :param data: Assumes all dictionaries in list have the same set of keys.
495
496        :param sort_key: Optional dictionary key to sort data with.
497
498        :param wrap_to_terminal: If True, the table width will be wrapped
499        to fit within the current terminal window. Set to False
500        if the output is going into something like a txt file."""
501        return data_to_string(data, sort_key, wrap_to_terminal)
502
503
504def data_to_string(
505    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
506) -> str:
507    """Uses tabulate to produce pretty string output
508    from a list of dictionaries.
509
510    :param data: Assumes all dictionaries in list have the same set of keys.
511
512    :param sort_key: Optional dictionary key to sort data with.
513
514    :param wrap_to_terminal: If True, the table width will be wrapped
515    to fit within the current terminal window. Set to False
516    if the output is going into something like a txt file."""
517    if len(data) == 0:
518        return ""
519    if sort_key:
520        data = sorted(data, key=lambda d: d[sort_key])
521    for i, d in enumerate(data):
522        for k in d:
523            data[i][k] = str(data[i][k])
524    if wrap_to_terminal:
525        terminal_width = os.get_terminal_size().columns
526        max_col_widths = terminal_width
527        """ Reducing the column width by tabulating one row at a time
528        and then reducing further by tabulating the whole set proved to be 
529        faster than going straight to tabulating the whole set and reducing
530        the column width."""
531        too_wide = True
532        while too_wide and max_col_widths > 1:
533            for i, row in enumerate(data):
534                output = tabulate(
535                    [row],
536                    headers="keys",
537                    disable_numparse=True,
538                    tablefmt="grid",
539                    maxcolwidths=max_col_widths,
540                )
541                if output.index("\n") > terminal_width:
542                    max_col_widths -= 2
543                    too_wide = True
544                    break
545                too_wide = False
546    else:
547        max_col_widths = None
548    output = tabulate(
549        data,
550        headers="keys",
551        disable_numparse=True,
552        tablefmt="grid",
553        maxcolwidths=max_col_widths,
554    )
555    # trim max column width until the output string is less wide than the current terminal width.
556    if wrap_to_terminal:
557        while output.index("\n") > terminal_width and max_col_widths > 1:
558            max_col_widths -= 2
559            max_col_widths = max(1, max_col_widths)
560            output = tabulate(
561                data,
562                headers="keys",
563                disable_numparse=True,
564                tablefmt="grid",
565                maxcolwidths=max_col_widths,
566            )
567    return output
class DataBased:
 28class DataBased:
 29    """Sqli wrapper so queries don't need to be written except table definitions.
 30
 31    Supports saving and reading dates as datetime objects.
 32
 33    Supports using a context manager."""
 34
 35    def __init__(
 36        self,
 37        dbpath: str | Path,
 38        logger_encoding: str = "utf-8",
 39        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 40    ):
 41        """
 42        :param dbpath: String or Path object to database file.
 43        If a relative path is given, it will be relative to the
 44        current working directory. The log file will be saved to the
 45        same directory.
 46
 47        :param logger_message_format: '{' style format string
 48        for the logger object."""
 49        self.dbpath = Path(dbpath)
 50        self.dbname = Path(dbpath).name
 51        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 52        self._logger_init(
 53            encoding=logger_encoding, message_format=logger_message_format
 54        )
 55        self.connection_open = False
 56        self.create_manager()
 57
 58    def __enter__(self):
 59        self.open()
 60        return self
 61
 62    def __exit__(self, exception_type, exception_value, exception_traceback):
 63        self.close()
 64
 65    def create_manager(self):
 66        """Create dbManager.py in the same directory
 67        as the database file if it doesn't exist."""
 68        manager_template = Path(__file__).parent / "dbmanager.py"
 69        manager_path = self.dbpath.parent / "dbmanager.py"
 70        if not manager_path.exists():
 71            shutil.copyfile(manager_template, manager_path)
 72
 73    def open(self):
 74        """Open connection to db."""
 75        self.connection = sqlite3.connect(
 76            self.dbpath,
 77            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 78            timeout=10,
 79        )
 80        self.connection.execute("pragma foreign_keys = 1")
 81        self.cursor = self.connection.cursor()
 82        self.connection_open = True
 83
 84    def close(self):
 85        """Save and close connection to db.
 86
 87        Call this as soon as you are done using the database if you have
 88        multiple threads or processes using the same database."""
 89        if self.connection_open:
 90            self.connection.commit()
 91            self.connection.close()
 92            self.connection_open = False
 93
 94    def _logger_init(
 95        self,
 96        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 97        encoding: str = "utf-8",
 98    ):
 99        """:param message_format: '{' style format string"""
100        self.logger = logging.getLogger(self.dbname)
101        if not self.logger.hasHandlers():
102            handler = logging.FileHandler(
103                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
104            )
105            handler.setFormatter(
106                logging.Formatter(
107                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
108                )
109            )
110            self.logger.addHandler(handler)
111            self.logger.setLevel(logging.INFO)
112
113    def _get_dict(
114        self, table: str, values: list, columns_to_return: list[str] = None
115    ) -> dict:
116        """Converts the values of a row into a dictionary with column names as keys.
117
118        :param table: The table that values were pulled from.
119
120        :param values: List of values expected to be the same quantity
121        and in the same order as the column names of table.
122
123        :param columns_to_return: An optional list of column names.
124        If given, only these columns will be included in the returned dictionary.
125        Otherwise all columns and values are returned."""
126        return {
127            column: value
128            for column, value in zip(self.get_column_names(table), values)
129            if not columns_to_return or column in columns_to_return
130        }
131
132    def _get_conditions(
133        self, match_criteria: list[tuple] | dict, exact_match: bool = True
134    ) -> str:
135        """Builds and returns the conditional portion of a query.
136
137        :param match_criteria: Can be a list of 2-tuples where each
138        tuple is (columnName, rowValue) or a dictionary where
139        keys are column names and values are row values.
140
141        :param exact_match: If False, the rowValue for a give column
142        will be matched as a substring.
143
144        Usage e.g.:
145
146        self.cursor.execute(f'select * from {table} where {conditions}')"""
147        if type(match_criteria) == dict:
148            match_criteria = [(k, v) for k, v in match_criteria.items()]
149        if exact_match:
150            conditions = " and ".join(
151                f'"{column_row[0]}" = "{column_row[1]}"'
152                for column_row in match_criteria
153            )
154        else:
155            conditions = " and ".join(
156                f'"{column_row[0]}" like "%{column_row[1]}%"'
157                for column_row in match_criteria
158            )
159        return f"({conditions})"
160
161    @_connect
162    def query(self, query_) -> list[Any]:
163        """Execute an arbitrary query and
164        return the results."""
165        self.cursor.execute(query_)
166        return self.cursor.fetchall()
167
168    @_connect
169    def create_tables(self, table_querys: list[str] = []):
170        """Create tables if they don't exist.
171
172        :param table_querys: Each query should be
173        in the form 'tableName(columnDefinitions)'"""
174        if len(table_querys) > 0:
175            table_names = self.get_table_names()
176            for table in table_querys:
177                if table.split("(")[0].strip() not in table_names:
178                    self.cursor.execute(f"create table {table}")
179                    self.logger.info(f'{table.split("(")[0]} table created.')
180
181    @_connect
182    def create_table(self, table: str, column_defs: list[str]):
183        """Create a table if it doesn't exist.
184
185        :param table: Name of the table to create.
186
187        :param column_defs: List of column definitions in
188        proper Sqlite3 sytax.
189        i.e. "columnName text unique" or "columnName int primary key" etc."""
190        if table not in self.get_table_names():
191            query = f"create table {table}({', '.join(column_defs)})"
192            self.cursor.execute(query)
193            self.logger.info(f"'{table}' table created.")
194
195    @_connect
196    def get_table_names(self) -> list[str]:
197        """Returns a list of table names from database."""
198        self.cursor.execute(
199            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
200        )
201        return [result[0] for result in self.cursor.fetchall()]
202
203    @_connect
204    def get_column_names(self, table: str) -> list[str]:
205        """Return a list of column names from a table."""
206        self.cursor.execute(f"select * from {table} where 1=0")
207        return [description[0] for description in self.cursor.description]
208
209    @_connect
210    def count(
211        self,
212        table: str,
213        match_criteria: list[tuple] | dict = None,
214        exact_match: bool = True,
215    ) -> int:
216        """Return number of items in table.
217
218        :param match_criteria: Can be a list of 2-tuples where each
219        tuple is (columnName, rowValue) or a dictionary where
220        keys are column names and values are row values.
221        If None, all rows from the table will be counted.
222
223        :param exact_match: If False, the row value for a give column
224        in match_criteria will be matched as a substring. Has no effect if
225        match_criteria is None.
226        """
227        query = f"select count(_rowid_) from {table}"
228        try:
229            if match_criteria:
230                self.cursor.execute(
231                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
232                )
233            else:
234                self.cursor.execute(f"{query}")
235            return self.cursor.fetchone()[0]
236        except:
237            return 0
238
239    @_connect
240    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
241        """Add row of values to table.
242
243        :param table: The table to insert into.
244
245        :param values: A tuple of values to be inserted into the table.
246
247        :param columns: If None, values param is expected to supply
248        a value for every column in the table. If columns is
249        provided, it should contain the same number of elements as values."""
250        parameterizer = ", ".join("?" for _ in values)
251        logger_values = ", ".join(str(value) for value in values)
252        try:
253            if columns:
254                columns = ", ".join(column for column in columns)
255                self.cursor.execute(
256                    f"insert into {table} ({columns}) values({parameterizer})", values
257                )
258            else:
259                self.cursor.execute(
260                    f"insert into {table} values({parameterizer})", values
261                )
262            self.logger.info(f'Added "{logger_values}" to {table} table.')
263        except Exception as e:
264            if "constraint" not in str(e).lower():
265                self.logger.exception(
266                    f'Error adding "{logger_values}" to {table} table.'
267                )
268            else:
269                self.logger.debug(str(e))
270
271    @_connect
272    def get_rows(
273        self,
274        table: str,
275        match_criteria: list[tuple] | dict = None,
276        exact_match: bool = True,
277        sort_by_column: str = None,
278        columns_to_return: list[str] = None,
279        return_as_dataframe: bool = False,
280        values_only: bool = False,
281        order_by: str = None,
282        limit: str | int = None,
283    ) -> list[dict] | list[tuple] | pandas.DataFrame:
284        """Returns rows from table as a list of dictionaries
285        where the key-value pairs of the dictionaries are
286        column name: row value.
287
288        :param match_criteria: Can be a list of 2-tuples where each
289        tuple is (columnName, rowValue) or a dictionary where
290        keys are column names and values are row values.
291
292        :param exact_match: If False, the rowValue for a give column
293        will be matched as a substring.
294
295        :param sort_by_column: A column name to sort the results by.
296        This will sort results in Python after retrieving them from the db.
297        Use the 'order_by' param to use SQLite engine for ordering.
298
299        :param columns_to_return: Optional list of column names.
300        If provided, the elements returned by get_rows() will
301        only contain the provided columns. Otherwise every column
302        in the row is returned.
303
304        :param return_as_dataframe: If True,
305        the results will be returned as a pandas.DataFrame object.
306
307        :param values_only: Return the results as a list of tuples
308        instead of a list of dictionaries that have column names as keys.
309        The results will still be sorted according to sort_by_column if
310        one is provided.
311
312        :param order_by: If given, a 'order by {order_by}' clause
313        will be added to the select query.
314
315        :param limit: If given, a 'limit {limit}' clause will be
316        added to the select query.
317        """
318
319        if type(columns_to_return) is str:
320            columns_to_return = [columns_to_return]
321        query = f"select * from {table}"
322        matches = []
323        if match_criteria:
324            query += f" where {self._get_conditions(match_criteria, exact_match)}"
325        if order_by:
326            query += f" order by {order_by}"
327        if limit:
328            query += f" limit {limit}"
329        query += ";"
330        self.cursor.execute(query)
331        matches = self.cursor.fetchall()
332        results = [self._get_dict(table, match, columns_to_return) for match in matches]
333        if sort_by_column:
334            results = sorted(results, key=lambda x: x[sort_by_column])
335        if return_as_dataframe:
336            return pandas.DataFrame(results)
337        if values_only:
338            return [tuple(row.values()) for row in results]
339        else:
340            return results
341
342    @_connect
343    def find(
344        self, table: str, query_string: str, columns: list[str] = None
345    ) -> tuple[dict]:
346        """Search for rows that contain query_string as a substring
347        of any column.
348
349        :param table: The table to search.
350
351        :param query_string: The substring to search for in all columns.
352
353        :param columns: A list of columns to search for query_string.
354        If None, all columns in the table will be searched.
355        """
356        if type(columns) is str:
357            columns = [columns]
358        results = []
359        if not columns:
360            columns = self.get_column_names(table)
361        for column in columns:
362            results.extend(
363                [
364                    row
365                    for row in self.get_rows(
366                        table, [(column, query_string)], exact_match=False
367                    )
368                    if row not in results
369                ]
370            )
371        return results
372
373    @_connect
374    def delete(
375        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
376    ) -> int:
377        """Delete records from table.
378
379        Returns number of deleted records.
380
381        :param match_criteria: Can be a list of 2-tuples where each
382        tuple is (columnName, rowValue) or a dictionary where
383        keys are column names and values are row values.
384
385        :param exact_match: If False, the rowValue for a give column
386        will be matched as a substring.
387        """
388        num_matches = self.count(table, match_criteria, exact_match)
389        conditions = self._get_conditions(match_criteria, exact_match)
390        try:
391            self.cursor.execute(f"delete from {table} where {conditions}")
392            self.logger.info(
393                f'Deleted {num_matches} from "{table}" where {conditions}".'
394            )
395            return num_matches
396        except Exception as e:
397            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
398            return 0
399
400    @_connect
401    def update(
402        self,
403        table: str,
404        column_to_update: str,
405        new_value: Any,
406        match_criteria: list[tuple] | dict = None,
407    ) -> bool:
408        """Update row value for entry matched with match_criteria.
409
410        :param column_to_update: The column to be updated in the matched row.
411
412        :param new_value: The new value to insert.
413
414        :param match_criteria: Can be a list of 2-tuples where each
415        tuple is (columnName, rowValue) or a dictionary where
416        keys are column names and values are row values.
417        If None, every row will be updated.
418
419        Returns True if successful, False if not."""
420        query = f"update {table} set {column_to_update} = ?"
421        if match_criteria:
422            if self.count(table, match_criteria) == 0:
423                self.logger.info(
424                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
425                )
426                return False
427            conditions = self._get_conditions(match_criteria)
428            query += f" where {conditions}"
429        else:
430            conditions = None
431        try:
432            self.cursor.execute(
433                query,
434                (new_value,),
435            )
436            self.logger.info(
437                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
438            )
439            return True
440        except UnboundLocalError:
441            table_filter_string = "\n".join(
442                table_filter for table_filter in match_criteria
443            )
444            self.logger.error(
445                f"No records found matching filters: {table_filter_string}"
446            )
447            return False
448        except Exception as e:
449            self.logger.error(
450                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
451            )
452            return False
453
454    @_connect
455    def drop_table(self, table: str) -> bool:
456        """Drop a table from the database.
457
458        Returns True if successful, False if not."""
459        try:
460            self.cursor.execute(f"drop Table {table}")
461            self.logger.info(f'Dropped table "{table}"')
462        except Exception as e:
463            print(e)
464            self.logger.error(f'Failed to drop table "{table}"')
465
466    @_connect
467    def add_column(
468        self, table: str, column: str, _type: str, default_value: str = None
469    ):
470        """Add a new column to table.
471
472        :param column: Name of the column to add.
473
474        :param _type: The data type of the new column.
475
476        :param default_value: Optional default value for the column."""
477        try:
478            if default_value:
479                self.cursor.execute(
480                    f"alter table {table} add column {column} {_type} default {default_value}"
481                )
482            else:
483                self.cursor.execute(f"alter table {table} add column {column} {_type}")
484            self.logger.info(f'Added column "{column}" to "{table}" table.')
485        except Exception as e:
486            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
487
488    @staticmethod
489    def data_to_string(
490        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
491    ) -> str:
492        """Uses tabulate to produce pretty string output
493        from a list of dictionaries.
494
495        :param data: Assumes all dictionaries in list have the same set of keys.
496
497        :param sort_key: Optional dictionary key to sort data with.
498
499        :param wrap_to_terminal: If True, the table width will be wrapped
500        to fit within the current terminal window. Set to False
501        if the output is going into something like a txt file."""
502        return data_to_string(data, sort_key, wrap_to_terminal)

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathlib.Path, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}')
35    def __init__(
36        self,
37        dbpath: str | Path,
38        logger_encoding: str = "utf-8",
39        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
40    ):
41        """
42        :param dbpath: String or Path object to database file.
43        If a relative path is given, it will be relative to the
44        current working directory. The log file will be saved to the
45        same directory.
46
47        :param logger_message_format: '{' style format string
48        for the logger object."""
49        self.dbpath = Path(dbpath)
50        self.dbname = Path(dbpath).name
51        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
52        self._logger_init(
53            encoding=logger_encoding, message_format=logger_message_format
54        )
55        self.connection_open = False
56        self.create_manager()
Parameters
  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: '{' style format string for the logger object.

def create_manager(self):
65    def create_manager(self):
66        """Create dbManager.py in the same directory
67        as the database file if it doesn't exist."""
68        manager_template = Path(__file__).parent / "dbmanager.py"
69        manager_path = self.dbpath.parent / "dbmanager.py"
70        if not manager_path.exists():
71            shutil.copyfile(manager_template, manager_path)

Create dbManager.py in the same directory as the database file if it doesn't exist.

def open(self):
73    def open(self):
74        """Open connection to db."""
75        self.connection = sqlite3.connect(
76            self.dbpath,
77            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
78            timeout=10,
79        )
80        self.connection.execute("pragma foreign_keys = 1")
81        self.cursor = self.connection.cursor()
82        self.connection_open = True

Open connection to db.

def close(self):
84    def close(self):
85        """Save and close connection to db.
86
87        Call this as soon as you are done using the database if you have
88        multiple threads or processes using the same database."""
89        if self.connection_open:
90            self.connection.commit()
91            self.connection.close()
92            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def query(self, query_) -> list[typing.Any]:
161    @_connect
162    def query(self, query_) -> list[Any]:
163        """Execute an arbitrary query and
164        return the results."""
165        self.cursor.execute(query_)
166        return self.cursor.fetchall()

Execute an arbitrary query and return the results.

def create_tables(self, table_querys: list[str] = []):
168    @_connect
169    def create_tables(self, table_querys: list[str] = []):
170        """Create tables if they don't exist.
171
172        :param table_querys: Each query should be
173        in the form 'tableName(columnDefinitions)'"""
174        if len(table_querys) > 0:
175            table_names = self.get_table_names()
176            for table in table_querys:
177                if table.split("(")[0].strip() not in table_names:
178                    self.cursor.execute(f"create table {table}")
179                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_querys: Each query should be in the form 'tableName(columnDefinitions)'
def create_table(self, table: str, column_defs: list[str]):
181    @_connect
182    def create_table(self, table: str, column_defs: list[str]):
183        """Create a table if it doesn't exist.
184
185        :param table: Name of the table to create.
186
187        :param column_defs: List of column definitions in
188        proper Sqlite3 sytax.
189        i.e. "columnName text unique" or "columnName int primary key" etc."""
190        if table not in self.get_table_names():
191            query = f"create table {table}({', '.join(column_defs)})"
192            self.cursor.execute(query)
193            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

Parameters
  • table: Name of the table to create.

  • column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.

def get_table_names(self) -> list[str]:
195    @_connect
196    def get_table_names(self) -> list[str]:
197        """Returns a list of table names from database."""
198        self.cursor.execute(
199            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
200        )
201        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from database.

def get_column_names(self, table: str) -> list[str]:
203    @_connect
204    def get_column_names(self, table: str) -> list[str]:
205        """Return a list of column names from a table."""
206        self.cursor.execute(f"select * from {table} where 1=0")
207        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True) -> int:
209    @_connect
210    def count(
211        self,
212        table: str,
213        match_criteria: list[tuple] | dict = None,
214        exact_match: bool = True,
215    ) -> int:
216        """Return number of items in table.
217
218        :param match_criteria: Can be a list of 2-tuples where each
219        tuple is (columnName, rowValue) or a dictionary where
220        keys are column names and values are row values.
221        If None, all rows from the table will be counted.
222
223        :param exact_match: If False, the row value for a give column
224        in match_criteria will be matched as a substring. Has no effect if
225        match_criteria is None.
226        """
227        query = f"select count(_rowid_) from {table}"
228        try:
229            if match_criteria:
230                self.cursor.execute(
231                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
232                )
233            else:
234                self.cursor.execute(f"{query}")
235            return self.cursor.fetchone()[0]
236        except:
237            return 0

Return number of items in table.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

  • exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
239    @_connect
240    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
241        """Add row of values to table.
242
243        :param table: The table to insert into.
244
245        :param values: A tuple of values to be inserted into the table.
246
247        :param columns: If None, values param is expected to supply
248        a value for every column in the table. If columns is
249        provided, it should contain the same number of elements as values."""
250        parameterizer = ", ".join("?" for _ in values)
251        logger_values = ", ".join(str(value) for value in values)
252        try:
253            if columns:
254                columns = ", ".join(column for column in columns)
255                self.cursor.execute(
256                    f"insert into {table} ({columns}) values({parameterizer})", values
257                )
258            else:
259                self.cursor.execute(
260                    f"insert into {table} values({parameterizer})", values
261                )
262            self.logger.info(f'Added "{logger_values}" to {table} table.')
263        except Exception as e:
264            if "constraint" not in str(e).lower():
265                self.logger.exception(
266                    f'Error adding "{logger_values}" to {table} table.'
267                )
268            else:
269                self.logger.debug(str(e))

Add row of values to table.

Parameters
  • table: The table to insert into.

  • values: A tuple of values to be inserted into the table.

  • columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True, sort_by_column: str = None, columns_to_return: list[str] = None, return_as_dataframe: bool = False, values_only: bool = False, order_by: str = None, limit: str | int = None) -> list[dict] | list[tuple] | pandas.core.frame.DataFrame:
271    @_connect
272    def get_rows(
273        self,
274        table: str,
275        match_criteria: list[tuple] | dict = None,
276        exact_match: bool = True,
277        sort_by_column: str = None,
278        columns_to_return: list[str] = None,
279        return_as_dataframe: bool = False,
280        values_only: bool = False,
281        order_by: str = None,
282        limit: str | int = None,
283    ) -> list[dict] | list[tuple] | pandas.DataFrame:
284        """Returns rows from table as a list of dictionaries
285        where the key-value pairs of the dictionaries are
286        column name: row value.
287
288        :param match_criteria: Can be a list of 2-tuples where each
289        tuple is (columnName, rowValue) or a dictionary where
290        keys are column names and values are row values.
291
292        :param exact_match: If False, the rowValue for a give column
293        will be matched as a substring.
294
295        :param sort_by_column: A column name to sort the results by.
296        This will sort results in Python after retrieving them from the db.
297        Use the 'order_by' param to use SQLite engine for ordering.
298
299        :param columns_to_return: Optional list of column names.
300        If provided, the elements returned by get_rows() will
301        only contain the provided columns. Otherwise every column
302        in the row is returned.
303
304        :param return_as_dataframe: If True,
305        the results will be returned as a pandas.DataFrame object.
306
307        :param values_only: Return the results as a list of tuples
308        instead of a list of dictionaries that have column names as keys.
309        The results will still be sorted according to sort_by_column if
310        one is provided.
311
312        :param order_by: If given, a 'order by {order_by}' clause
313        will be added to the select query.
314
315        :param limit: If given, a 'limit {limit}' clause will be
316        added to the select query.
317        """
318
319        if type(columns_to_return) is str:
320            columns_to_return = [columns_to_return]
321        query = f"select * from {table}"
322        matches = []
323        if match_criteria:
324            query += f" where {self._get_conditions(match_criteria, exact_match)}"
325        if order_by:
326            query += f" order by {order_by}"
327        if limit:
328            query += f" limit {limit}"
329        query += ";"
330        self.cursor.execute(query)
331        matches = self.cursor.fetchall()
332        results = [self._get_dict(table, match, columns_to_return) for match in matches]
333        if sort_by_column:
334            results = sorted(results, key=lambda x: x[sort_by_column])
335        if return_as_dataframe:
336            return pandas.DataFrame(results)
337        if values_only:
338            return [tuple(row.values()) for row in results]
339        else:
340            return results

Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

  • sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.

  • columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.

  • return_as_dataframe: If True, the results will be returned as a pandas.DataFrame object.

  • values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.

  • order_by: If given, a 'order by {order_by}' clause will be added to the select query.

  • limit: If given, a 'limit {limit}' clause will be added to the select query.

def find( self, table: str, query_string: str, columns: list[str] = None) -> tuple[dict]:
342    @_connect
343    def find(
344        self, table: str, query_string: str, columns: list[str] = None
345    ) -> tuple[dict]:
346        """Search for rows that contain query_string as a substring
347        of any column.
348
349        :param table: The table to search.
350
351        :param query_string: The substring to search for in all columns.
352
353        :param columns: A list of columns to search for query_string.
354        If None, all columns in the table will be searched.
355        """
356        if type(columns) is str:
357            columns = [columns]
358        results = []
359        if not columns:
360            columns = self.get_column_names(table)
361        for column in columns:
362            results.extend(
363                [
364                    row
365                    for row in self.get_rows(
366                        table, [(column, query_string)], exact_match=False
367                    )
368                    if row not in results
369                ]
370            )
371        return results

Search for rows that contain query_string as a substring of any column.

Parameters
  • table: The table to search.

  • query_string: The substring to search for in all columns.

  • columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
373    @_connect
374    def delete(
375        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
376    ) -> int:
377        """Delete records from table.
378
379        Returns number of deleted records.
380
381        :param match_criteria: Can be a list of 2-tuples where each
382        tuple is (columnName, rowValue) or a dictionary where
383        keys are column names and values are row values.
384
385        :param exact_match: If False, the rowValue for a give column
386        will be matched as a substring.
387        """
388        num_matches = self.count(table, match_criteria, exact_match)
389        conditions = self._get_conditions(match_criteria, exact_match)
390        try:
391            self.cursor.execute(f"delete from {table} where {conditions}")
392            self.logger.info(
393                f'Deleted {num_matches} from "{table}" where {conditions}".'
394            )
395            return num_matches
396        except Exception as e:
397            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
398            return 0

Delete records from table.

Returns number of deleted records.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict = None) -> bool:
400    @_connect
401    def update(
402        self,
403        table: str,
404        column_to_update: str,
405        new_value: Any,
406        match_criteria: list[tuple] | dict = None,
407    ) -> bool:
408        """Update row value for entry matched with match_criteria.
409
410        :param column_to_update: The column to be updated in the matched row.
411
412        :param new_value: The new value to insert.
413
414        :param match_criteria: Can be a list of 2-tuples where each
415        tuple is (columnName, rowValue) or a dictionary where
416        keys are column names and values are row values.
417        If None, every row will be updated.
418
419        Returns True if successful, False if not."""
420        query = f"update {table} set {column_to_update} = ?"
421        if match_criteria:
422            if self.count(table, match_criteria) == 0:
423                self.logger.info(
424                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
425                )
426                return False
427            conditions = self._get_conditions(match_criteria)
428            query += f" where {conditions}"
429        else:
430            conditions = None
431        try:
432            self.cursor.execute(
433                query,
434                (new_value,),
435            )
436            self.logger.info(
437                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
438            )
439            return True
440        except UnboundLocalError:
441            table_filter_string = "\n".join(
442                table_filter for table_filter in match_criteria
443            )
444            self.logger.error(
445                f"No records found matching filters: {table_filter_string}"
446            )
447            return False
448        except Exception as e:
449            self.logger.error(
450                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
451            )
452            return False

Update row value for entry matched with match_criteria.

Parameters
  • column_to_update: The column to be updated in the matched row.

  • new_value: The new value to insert.

  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.

Returns True if successful, False if not.

def drop_table(self, table: str) -> bool:
454    @_connect
455    def drop_table(self, table: str) -> bool:
456        """Drop a table from the database.
457
458        Returns True if successful, False if not."""
459        try:
460            self.cursor.execute(f"drop Table {table}")
461            self.logger.info(f'Dropped table "{table}"')
462        except Exception as e:
463            print(e)
464            self.logger.error(f'Failed to drop table "{table}"')

Drop a table from the database.

Returns True if successful, False if not.

def add_column(self, table: str, column: str, _type: str, default_value: str = None):
466    @_connect
467    def add_column(
468        self, table: str, column: str, _type: str, default_value: str = None
469    ):
470        """Add a new column to table.
471
472        :param column: Name of the column to add.
473
474        :param _type: The data type of the new column.
475
476        :param default_value: Optional default value for the column."""
477        try:
478            if default_value:
479                self.cursor.execute(
480                    f"alter table {table} add column {column} {_type} default {default_value}"
481                )
482            else:
483                self.cursor.execute(f"alter table {table} add column {column} {_type}")
484            self.logger.info(f'Added column "{column}" to "{table}" table.')
485        except Exception as e:
486            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

Parameters
  • column: Name of the column to add.

  • _type: The data type of the new column.

  • default_value: Optional default value for the column.

@staticmethod
def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
488    @staticmethod
489    def data_to_string(
490        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
491    ) -> str:
492        """Uses tabulate to produce pretty string output
493        from a list of dictionaries.
494
495        :param data: Assumes all dictionaries in list have the same set of keys.
496
497        :param sort_key: Optional dictionary key to sort data with.
498
499        :param wrap_to_terminal: If True, the table width will be wrapped
500        to fit within the current terminal window. Set to False
501        if the output is going into something like a txt file."""
502        return data_to_string(data, sort_key, wrap_to_terminal)

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.

def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
505def data_to_string(
506    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
507) -> str:
508    """Uses tabulate to produce pretty string output
509    from a list of dictionaries.
510
511    :param data: Assumes all dictionaries in list have the same set of keys.
512
513    :param sort_key: Optional dictionary key to sort data with.
514
515    :param wrap_to_terminal: If True, the table width will be wrapped
516    to fit within the current terminal window. Set to False
517    if the output is going into something like a txt file."""
518    if len(data) == 0:
519        return ""
520    if sort_key:
521        data = sorted(data, key=lambda d: d[sort_key])
522    for i, d in enumerate(data):
523        for k in d:
524            data[i][k] = str(data[i][k])
525    if wrap_to_terminal:
526        terminal_width = os.get_terminal_size().columns
527        max_col_widths = terminal_width
528        """ Reducing the column width by tabulating one row at a time
529        and then reducing further by tabulating the whole set proved to be 
530        faster than going straight to tabulating the whole set and reducing
531        the column width."""
532        too_wide = True
533        while too_wide and max_col_widths > 1:
534            for i, row in enumerate(data):
535                output = tabulate(
536                    [row],
537                    headers="keys",
538                    disable_numparse=True,
539                    tablefmt="grid",
540                    maxcolwidths=max_col_widths,
541                )
542                if output.index("\n") > terminal_width:
543                    max_col_widths -= 2
544                    too_wide = True
545                    break
546                too_wide = False
547    else:
548        max_col_widths = None
549    output = tabulate(
550        data,
551        headers="keys",
552        disable_numparse=True,
553        tablefmt="grid",
554        maxcolwidths=max_col_widths,
555    )
556    # trim max column width until the output string is less wide than the current terminal width.
557    if wrap_to_terminal:
558        while output.index("\n") > terminal_width and max_col_widths > 1:
559            max_col_widths -= 2
560            max_col_widths = max(1, max_col_widths)
561            output = tabulate(
562                data,
563                headers="keys",
564                disable_numparse=True,
565                tablefmt="grid",
566                maxcolwidths=max_col_widths,
567            )
568    return output

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.