databased.databased

  1import logging
  2import os
  3import sqlite3
  4from datetime import datetime
  5from functools import wraps
  6from pathlib import Path
  7from typing import Any
  8
  9import pandas
 10from tabulate import tabulate
 11
 12
 13def _connect(func):
 14    """Decorator to open db connection if it isn't already open."""
 15
 16    @wraps(func)
 17    def inner(self, *args, **kwargs):
 18        if not self.connection_open:
 19            self.open()
 20        results = func(self, *args, **kwargs)
 21        return results
 22
 23    return inner
 24
 25
 26class DataBased:
 27    """Sqli wrapper so queries don't need to be written except table definitions.
 28
 29    Supports saving and reading dates as datetime objects.
 30
 31    Supports using a context manager."""
 32
 33    def __init__(
 34        self,
 35        dbpath: str | Path,
 36        logger_encoding: str = "utf-8",
 37        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 38    ):
 39        """
 40        :param dbpath: String or Path object to database file.
 41        If a relative path is given, it will be relative to the
 42        current working directory. The log file will be saved to the
 43        same directory.
 44
 45        :param logger_message_format: '{' style format string
 46        for the logger object."""
 47        self.dbpath = Path(dbpath)
 48        self.dbname = Path(dbpath).name
 49        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 50        self._logger_init(
 51            encoding=logger_encoding, message_format=logger_message_format
 52        )
 53        self.connection_open = False
 54        self.create_manager()
 55
 56    def __enter__(self):
 57        self.open()
 58        return self
 59
 60    def __exit__(self, exception_type, exception_value, exception_traceback):
 61        self.close()
 62
 63    def create_manager(self):
 64        """Create dbmanager.py in the same directory
 65        as the database file if it doesn't exist."""
 66        manager_path = self.dbpath.parent / "dbmanager.py"
 67        if not manager_path.exists():
 68            manager_template = (Path(__file__).parent / "dbmanager.py").read_text()
 69            manager_path.write_text(manager_template.replace("$dbname", self.dbname))
 70
 71    def open(self):
 72        """Open connection to db."""
 73        self.connection = sqlite3.connect(
 74            self.dbpath,
 75            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 76            timeout=10,
 77        )
 78        self.connection.execute("pragma foreign_keys = 1")
 79        self.cursor = self.connection.cursor()
 80        self.connection_open = True
 81
 82    def close(self):
 83        """Save and close connection to db.
 84
 85        Call this as soon as you are done using the database if you have
 86        multiple threads or processes using the same database."""
 87        if self.connection_open:
 88            self.connection.commit()
 89            self.connection.close()
 90            self.connection_open = False
 91
 92    def _logger_init(
 93        self,
 94        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 95        encoding: str = "utf-8",
 96    ):
 97        """:param message_format: '{' style format string"""
 98        self.logger = logging.getLogger(self.dbname)
 99        if not self.logger.hasHandlers():
100            handler = logging.FileHandler(
101                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
102            )
103            handler.setFormatter(
104                logging.Formatter(
105                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
106                )
107            )
108            self.logger.addHandler(handler)
109            self.logger.setLevel(logging.INFO)
110
111    def _get_dict(
112        self, table: str, values: list, columns_to_return: list[str] = None
113    ) -> dict:
114        """Converts the values of a row into a dictionary with column names as keys.
115
116        :param table: The table that values were pulled from.
117
118        :param values: List of values expected to be the same quantity
119        and in the same order as the column names of table.
120
121        :param columns_to_return: An optional list of column names.
122        If given, only these columns will be included in the returned dictionary.
123        Otherwise all columns and values are returned."""
124        return {
125            column: value
126            for column, value in zip(self.get_column_names(table), values)
127            if not columns_to_return or column in columns_to_return
128        }
129
130    def _get_conditions(
131        self, match_criteria: list[tuple] | dict, exact_match: bool = True
132    ) -> str:
133        """Builds and returns the conditional portion of a query.
134
135        :param match_criteria: Can be a list of 2-tuples where each
136        tuple is (columnName, rowValue) or a dictionary where
137        keys are column names and values are row values.
138
139        :param exact_match: If False, the rowValue for a give column
140        will be matched as a substring.
141
142        Usage e.g.:
143
144        self.cursor.execute(f'select * from {table} where {conditions}')"""
145        if type(match_criteria) == dict:
146            match_criteria = [(k, v) for k, v in match_criteria.items()]
147        if exact_match:
148            conditions = " and ".join(
149                f'"{column_row[0]}" = "{column_row[1]}"'
150                for column_row in match_criteria
151            )
152        else:
153            conditions = " and ".join(
154                f'"{column_row[0]}" like "%{column_row[1]}%"'
155                for column_row in match_criteria
156            )
157        return f"({conditions})"
158
159    @_connect
160    def query(self, query_) -> list[Any]:
161        """Execute an arbitrary query and
162        return the results."""
163        self.cursor.execute(query_)
164        return self.cursor.fetchall()
165
166    @_connect
167    def create_tables(self, table_querys: list[str] = []):
168        """Create tables if they don't exist.
169
170        :param table_querys: Each query should be
171        in the form 'tableName(columnDefinitions)'"""
172        if len(table_querys) > 0:
173            table_names = self.get_table_names()
174            for table in table_querys:
175                if table.split("(")[0].strip() not in table_names:
176                    self.cursor.execute(f"create table {table}")
177                    self.logger.info(f'{table.split("(")[0]} table created.')
178
179    @_connect
180    def create_table(self, table: str, column_defs: list[str]):
181        """Create a table if it doesn't exist.
182
183        :param table: Name of the table to create.
184
185        :param column_defs: List of column definitions in
186        proper Sqlite3 sytax.
187        i.e. "columnName text unique" or "columnName int primary key" etc."""
188        if table not in self.get_table_names():
189            query = f"create table {table}({', '.join(column_defs)})"
190            self.cursor.execute(query)
191            self.logger.info(f"'{table}' table created.")
192
193    @_connect
194    def get_table_names(self) -> list[str]:
195        """Returns a list of table names from database."""
196        self.cursor.execute(
197            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
198        )
199        return [result[0] for result in self.cursor.fetchall()]
200
201    @_connect
202    def get_column_names(self, table: str) -> list[str]:
203        """Return a list of column names from a table."""
204        self.cursor.execute(f"select * from {table} where 1=0")
205        return [description[0] for description in self.cursor.description]
206
207    @_connect
208    def count(
209        self,
210        table: str,
211        match_criteria: list[tuple] | dict = None,
212        exact_match: bool = True,
213    ) -> int:
214        """Return number of items in table.
215
216        :param match_criteria: Can be a list of 2-tuples where each
217        tuple is (columnName, rowValue) or a dictionary where
218        keys are column names and values are row values.
219        If None, all rows from the table will be counted.
220
221        :param exact_match: If False, the row value for a give column
222        in match_criteria will be matched as a substring. Has no effect if
223        match_criteria is None.
224        """
225        query = f"select count(_rowid_) from {table}"
226        try:
227            if match_criteria:
228                self.cursor.execute(
229                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
230                )
231            else:
232                self.cursor.execute(f"{query}")
233            return self.cursor.fetchone()[0]
234        except:
235            return 0
236
237    @_connect
238    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
239        """Add row of values to table.
240
241        :param table: The table to insert into.
242
243        :param values: A tuple of values to be inserted into the table.
244
245        :param columns: If None, values param is expected to supply
246        a value for every column in the table. If columns is
247        provided, it should contain the same number of elements as values."""
248        parameterizer = ", ".join("?" for _ in values)
249        logger_values = ", ".join(str(value) for value in values)
250        try:
251            if columns:
252                columns = ", ".join(column for column in columns)
253                self.cursor.execute(
254                    f"insert into {table} ({columns}) values({parameterizer})", values
255                )
256            else:
257                self.cursor.execute(
258                    f"insert into {table} values({parameterizer})", values
259                )
260            self.logger.info(f'Added "{logger_values}" to {table} table.')
261        except Exception as e:
262            if "constraint" not in str(e).lower():
263                self.logger.exception(
264                    f'Error adding "{logger_values}" to {table} table.'
265                )
266            else:
267                self.logger.debug(str(e))
268
269    @_connect
270    def get_rows(
271        self,
272        table: str,
273        match_criteria: list[tuple] | dict = None,
274        exact_match: bool = True,
275        sort_by_column: str = None,
276        columns_to_return: list[str] = None,
277        return_as_dataframe: bool = False,
278        values_only: bool = False,
279        order_by: str = None,
280        limit: str | int = None,
281    ) -> list[dict] | list[tuple] | pandas.DataFrame:
282        """Returns rows from table as a list of dictionaries
283        where the key-value pairs of the dictionaries are
284        column name: row value.
285
286        :param match_criteria: Can be a list of 2-tuples where each
287        tuple is (columnName, rowValue) or a dictionary where
288        keys are column names and values are row values.
289
290        :param exact_match: If False, the rowValue for a give column
291        will be matched as a substring.
292
293        :param sort_by_column: A column name to sort the results by.
294        This will sort results in Python after retrieving them from the db.
295        Use the 'order_by' param to use SQLite engine for ordering.
296
297        :param columns_to_return: Optional list of column names.
298        If provided, the elements returned by get_rows() will
299        only contain the provided columns. Otherwise every column
300        in the row is returned.
301
302        :param return_as_dataframe: If True,
303        the results will be returned as a pandas.DataFrame object.
304
305        :param values_only: Return the results as a list of tuples
306        instead of a list of dictionaries that have column names as keys.
307        The results will still be sorted according to sort_by_column if
308        one is provided.
309
310        :param order_by: If given, a 'order by {order_by}' clause
311        will be added to the select query.
312
313        :param limit: If given, a 'limit {limit}' clause will be
314        added to the select query.
315        """
316
317        if type(columns_to_return) is str:
318            columns_to_return = [columns_to_return]
319        query = f"select * from {table}"
320        matches = []
321        if match_criteria:
322            query += f" where {self._get_conditions(match_criteria, exact_match)}"
323        if order_by:
324            query += f" order by {order_by}"
325        if limit:
326            query += f" limit {limit}"
327        query += ";"
328        self.cursor.execute(query)
329        matches = self.cursor.fetchall()
330        results = [self._get_dict(table, match, columns_to_return) for match in matches]
331        if sort_by_column:
332            results = sorted(results, key=lambda x: x[sort_by_column])
333        if return_as_dataframe:
334            return pandas.DataFrame(results)
335        if values_only:
336            return [tuple(row.values()) for row in results]
337        else:
338            return results
339
340    @_connect
341    def find(
342        self, table: str, query_string: str, columns: list[str] = None
343    ) -> tuple[dict]:
344        """Search for rows that contain query_string as a substring
345        of any column.
346
347        :param table: The table to search.
348
349        :param query_string: The substring to search for in all columns.
350
351        :param columns: A list of columns to search for query_string.
352        If None, all columns in the table will be searched.
353        """
354        if type(columns) is str:
355            columns = [columns]
356        results = []
357        if not columns:
358            columns = self.get_column_names(table)
359        for column in columns:
360            results.extend(
361                [
362                    row
363                    for row in self.get_rows(
364                        table, [(column, query_string)], exact_match=False
365                    )
366                    if row not in results
367                ]
368            )
369        return results
370
371    @_connect
372    def delete(
373        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
374    ) -> int:
375        """Delete records from table.
376
377        Returns number of deleted records.
378
379        :param match_criteria: Can be a list of 2-tuples where each
380        tuple is (columnName, rowValue) or a dictionary where
381        keys are column names and values are row values.
382
383        :param exact_match: If False, the rowValue for a give column
384        will be matched as a substring.
385        """
386        num_matches = self.count(table, match_criteria, exact_match)
387        conditions = self._get_conditions(match_criteria, exact_match)
388        try:
389            self.cursor.execute(f"delete from {table} where {conditions}")
390            self.logger.info(
391                f'Deleted {num_matches} from "{table}" where {conditions}".'
392            )
393            return num_matches
394        except Exception as e:
395            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
396            return 0
397
398    @_connect
399    def update(
400        self,
401        table: str,
402        column_to_update: str,
403        new_value: Any,
404        match_criteria: list[tuple] | dict = None,
405    ) -> bool:
406        """Update row value for entry matched with match_criteria.
407
408        :param column_to_update: The column to be updated in the matched row.
409
410        :param new_value: The new value to insert.
411
412        :param match_criteria: Can be a list of 2-tuples where each
413        tuple is (columnName, rowValue) or a dictionary where
414        keys are column names and values are row values.
415        If None, every row will be updated.
416
417        Returns True if successful, False if not."""
418        query = f"update {table} set {column_to_update} = ?"
419        if match_criteria:
420            if self.count(table, match_criteria) == 0:
421                self.logger.info(
422                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
423                )
424                return False
425            conditions = self._get_conditions(match_criteria)
426            query += f" where {conditions}"
427        else:
428            conditions = None
429        try:
430            self.cursor.execute(
431                query,
432                (new_value,),
433            )
434            self.logger.info(
435                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
436            )
437            return True
438        except UnboundLocalError:
439            table_filter_string = "\n".join(
440                table_filter for table_filter in match_criteria
441            )
442            self.logger.error(
443                f"No records found matching filters: {table_filter_string}"
444            )
445            return False
446        except Exception as e:
447            self.logger.error(
448                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
449            )
450            return False
451
452    @_connect
453    def drop_table(self, table: str) -> bool:
454        """Drop a table from the database.
455
456        Returns True if successful, False if not."""
457        try:
458            self.cursor.execute(f"drop Table {table}")
459            self.logger.info(f'Dropped table "{table}"')
460        except Exception as e:
461            print(e)
462            self.logger.error(f'Failed to drop table "{table}"')
463
464    @_connect
465    def add_column(
466        self, table: str, column: str, _type: str, default_value: str = None
467    ):
468        """Add a new column to table.
469
470        :param column: Name of the column to add.
471
472        :param _type: The data type of the new column.
473
474        :param default_value: Optional default value for the column."""
475        try:
476            if default_value:
477                self.cursor.execute(
478                    f"alter table {table} add column {column} {_type} default {default_value}"
479                )
480            else:
481                self.cursor.execute(f"alter table {table} add column {column} {_type}")
482            self.logger.info(f'Added column "{column}" to "{table}" table.')
483        except Exception as e:
484            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
485
486    @staticmethod
487    def data_to_string(
488        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
489    ) -> str:
490        """Uses tabulate to produce pretty string output
491        from a list of dictionaries.
492
493        :param data: Assumes all dictionaries in list have the same set of keys.
494
495        :param sort_key: Optional dictionary key to sort data with.
496
497        :param wrap_to_terminal: If True, the table width will be wrapped
498        to fit within the current terminal window. Set to False
499        if the output is going into something like a txt file."""
500        return data_to_string(data, sort_key, wrap_to_terminal)
501
502
503def data_to_string(
504    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
505) -> str:
506    """Use tabulate to produce grid output from a list of dictionaries.
507
508    :param data: Assumes all dictionaries in list have the same set of keys.
509
510    :param sort_key: Optional dictionary key to sort data with.
511
512    :param wrap_to_terminal: If True, the column widths will be reduced so the grid fits
513    within the current terminal window without wrapping. If the column widths have reduced to 1
514    and the grid is still too wide, str(data) will be returned."""
515    if len(data) == 0:
516        return ""
517    if sort_key:
518        data = sorted(data, key=lambda d: d[sort_key])
519    for i, d in enumerate(data):
520        for k in d:
521            data[i][k] = str(data[i][k])
522
523    too_wide = True
524    terminal_width = os.get_terminal_size().columns
525    max_col_widths = terminal_width
526    # Make an output with effectively unrestricted column widths
527    # to see if shrinking is necessary
528    output = tabulate(
529        data,
530        headers="keys",
531        disable_numparse=True,
532        tablefmt="grid",
533        maxcolwidths=max_col_widths,
534    )
535    current_width = output.index("\n")
536    if current_width < terminal_width:
537        too_wide = False
538    if wrap_to_terminal and too_wide:
539        previous_col_widths = max_col_widths
540        while too_wide and max_col_widths > 1:
541            if current_width > terminal_width:
542                previous_col_widths = max_col_widths
543                max_col_widths = int(max_col_widths * 0.5)
544            elif current_width < terminal_width:
545                max_col_widths = int(
546                    max_col_widths + ((previous_col_widths - max_col_widths) * 0.5)
547                )
548            output = tabulate(
549                data,
550                headers="keys",
551                disable_numparse=True,
552                tablefmt="grid",
553                maxcolwidths=max_col_widths,
554            )
555            current_width = output.index("\n")
556            if terminal_width - 10 < current_width < terminal_width:
557                too_wide = False
558        if too_wide:
559            return str(data)
560        return output
class DataBased:
 27class DataBased:
 28    """Sqli wrapper so queries don't need to be written except table definitions.
 29
 30    Supports saving and reading dates as datetime objects.
 31
 32    Supports using a context manager."""
 33
 34    def __init__(
 35        self,
 36        dbpath: str | Path,
 37        logger_encoding: str = "utf-8",
 38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 39    ):
 40        """
 41        :param dbpath: String or Path object to database file.
 42        If a relative path is given, it will be relative to the
 43        current working directory. The log file will be saved to the
 44        same directory.
 45
 46        :param logger_message_format: '{' style format string
 47        for the logger object."""
 48        self.dbpath = Path(dbpath)
 49        self.dbname = Path(dbpath).name
 50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
 51        self._logger_init(
 52            encoding=logger_encoding, message_format=logger_message_format
 53        )
 54        self.connection_open = False
 55        self.create_manager()
 56
 57    def __enter__(self):
 58        self.open()
 59        return self
 60
 61    def __exit__(self, exception_type, exception_value, exception_traceback):
 62        self.close()
 63
 64    def create_manager(self):
 65        """Create dbmanager.py in the same directory
 66        as the database file if it doesn't exist."""
 67        manager_path = self.dbpath.parent / "dbmanager.py"
 68        if not manager_path.exists():
 69            manager_template = (Path(__file__).parent / "dbmanager.py").read_text()
 70            manager_path.write_text(manager_template.replace("$dbname", self.dbname))
 71
 72    def open(self):
 73        """Open connection to db."""
 74        self.connection = sqlite3.connect(
 75            self.dbpath,
 76            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 77            timeout=10,
 78        )
 79        self.connection.execute("pragma foreign_keys = 1")
 80        self.cursor = self.connection.cursor()
 81        self.connection_open = True
 82
 83    def close(self):
 84        """Save and close connection to db.
 85
 86        Call this as soon as you are done using the database if you have
 87        multiple threads or processes using the same database."""
 88        if self.connection_open:
 89            self.connection.commit()
 90            self.connection.close()
 91            self.connection_open = False
 92
 93    def _logger_init(
 94        self,
 95        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 96        encoding: str = "utf-8",
 97    ):
 98        """:param message_format: '{' style format string"""
 99        self.logger = logging.getLogger(self.dbname)
100        if not self.logger.hasHandlers():
101            handler = logging.FileHandler(
102                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
103            )
104            handler.setFormatter(
105                logging.Formatter(
106                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
107                )
108            )
109            self.logger.addHandler(handler)
110            self.logger.setLevel(logging.INFO)
111
112    def _get_dict(
113        self, table: str, values: list, columns_to_return: list[str] = None
114    ) -> dict:
115        """Converts the values of a row into a dictionary with column names as keys.
116
117        :param table: The table that values were pulled from.
118
119        :param values: List of values expected to be the same quantity
120        and in the same order as the column names of table.
121
122        :param columns_to_return: An optional list of column names.
123        If given, only these columns will be included in the returned dictionary.
124        Otherwise all columns and values are returned."""
125        return {
126            column: value
127            for column, value in zip(self.get_column_names(table), values)
128            if not columns_to_return or column in columns_to_return
129        }
130
131    def _get_conditions(
132        self, match_criteria: list[tuple] | dict, exact_match: bool = True
133    ) -> str:
134        """Builds and returns the conditional portion of a query.
135
136        :param match_criteria: Can be a list of 2-tuples where each
137        tuple is (columnName, rowValue) or a dictionary where
138        keys are column names and values are row values.
139
140        :param exact_match: If False, the rowValue for a give column
141        will be matched as a substring.
142
143        Usage e.g.:
144
145        self.cursor.execute(f'select * from {table} where {conditions}')"""
146        if type(match_criteria) == dict:
147            match_criteria = [(k, v) for k, v in match_criteria.items()]
148        if exact_match:
149            conditions = " and ".join(
150                f'"{column_row[0]}" = "{column_row[1]}"'
151                for column_row in match_criteria
152            )
153        else:
154            conditions = " and ".join(
155                f'"{column_row[0]}" like "%{column_row[1]}%"'
156                for column_row in match_criteria
157            )
158        return f"({conditions})"
159
160    @_connect
161    def query(self, query_) -> list[Any]:
162        """Execute an arbitrary query and
163        return the results."""
164        self.cursor.execute(query_)
165        return self.cursor.fetchall()
166
167    @_connect
168    def create_tables(self, table_querys: list[str] = []):
169        """Create tables if they don't exist.
170
171        :param table_querys: Each query should be
172        in the form 'tableName(columnDefinitions)'"""
173        if len(table_querys) > 0:
174            table_names = self.get_table_names()
175            for table in table_querys:
176                if table.split("(")[0].strip() not in table_names:
177                    self.cursor.execute(f"create table {table}")
178                    self.logger.info(f'{table.split("(")[0]} table created.')
179
180    @_connect
181    def create_table(self, table: str, column_defs: list[str]):
182        """Create a table if it doesn't exist.
183
184        :param table: Name of the table to create.
185
186        :param column_defs: List of column definitions in
187        proper Sqlite3 sytax.
188        i.e. "columnName text unique" or "columnName int primary key" etc."""
189        if table not in self.get_table_names():
190            query = f"create table {table}({', '.join(column_defs)})"
191            self.cursor.execute(query)
192            self.logger.info(f"'{table}' table created.")
193
194    @_connect
195    def get_table_names(self) -> list[str]:
196        """Returns a list of table names from database."""
197        self.cursor.execute(
198            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
199        )
200        return [result[0] for result in self.cursor.fetchall()]
201
202    @_connect
203    def get_column_names(self, table: str) -> list[str]:
204        """Return a list of column names from a table."""
205        self.cursor.execute(f"select * from {table} where 1=0")
206        return [description[0] for description in self.cursor.description]
207
208    @_connect
209    def count(
210        self,
211        table: str,
212        match_criteria: list[tuple] | dict = None,
213        exact_match: bool = True,
214    ) -> int:
215        """Return number of items in table.
216
217        :param match_criteria: Can be a list of 2-tuples where each
218        tuple is (columnName, rowValue) or a dictionary where
219        keys are column names and values are row values.
220        If None, all rows from the table will be counted.
221
222        :param exact_match: If False, the row value for a give column
223        in match_criteria will be matched as a substring. Has no effect if
224        match_criteria is None.
225        """
226        query = f"select count(_rowid_) from {table}"
227        try:
228            if match_criteria:
229                self.cursor.execute(
230                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
231                )
232            else:
233                self.cursor.execute(f"{query}")
234            return self.cursor.fetchone()[0]
235        except:
236            return 0
237
238    @_connect
239    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
240        """Add row of values to table.
241
242        :param table: The table to insert into.
243
244        :param values: A tuple of values to be inserted into the table.
245
246        :param columns: If None, values param is expected to supply
247        a value for every column in the table. If columns is
248        provided, it should contain the same number of elements as values."""
249        parameterizer = ", ".join("?" for _ in values)
250        logger_values = ", ".join(str(value) for value in values)
251        try:
252            if columns:
253                columns = ", ".join(column for column in columns)
254                self.cursor.execute(
255                    f"insert into {table} ({columns}) values({parameterizer})", values
256                )
257            else:
258                self.cursor.execute(
259                    f"insert into {table} values({parameterizer})", values
260                )
261            self.logger.info(f'Added "{logger_values}" to {table} table.')
262        except Exception as e:
263            if "constraint" not in str(e).lower():
264                self.logger.exception(
265                    f'Error adding "{logger_values}" to {table} table.'
266                )
267            else:
268                self.logger.debug(str(e))
269
270    @_connect
271    def get_rows(
272        self,
273        table: str,
274        match_criteria: list[tuple] | dict = None,
275        exact_match: bool = True,
276        sort_by_column: str = None,
277        columns_to_return: list[str] = None,
278        return_as_dataframe: bool = False,
279        values_only: bool = False,
280        order_by: str = None,
281        limit: str | int = None,
282    ) -> list[dict] | list[tuple] | pandas.DataFrame:
283        """Returns rows from table as a list of dictionaries
284        where the key-value pairs of the dictionaries are
285        column name: row value.
286
287        :param match_criteria: Can be a list of 2-tuples where each
288        tuple is (columnName, rowValue) or a dictionary where
289        keys are column names and values are row values.
290
291        :param exact_match: If False, the rowValue for a give column
292        will be matched as a substring.
293
294        :param sort_by_column: A column name to sort the results by.
295        This will sort results in Python after retrieving them from the db.
296        Use the 'order_by' param to use SQLite engine for ordering.
297
298        :param columns_to_return: Optional list of column names.
299        If provided, the elements returned by get_rows() will
300        only contain the provided columns. Otherwise every column
301        in the row is returned.
302
303        :param return_as_dataframe: If True,
304        the results will be returned as a pandas.DataFrame object.
305
306        :param values_only: Return the results as a list of tuples
307        instead of a list of dictionaries that have column names as keys.
308        The results will still be sorted according to sort_by_column if
309        one is provided.
310
311        :param order_by: If given, a 'order by {order_by}' clause
312        will be added to the select query.
313
314        :param limit: If given, a 'limit {limit}' clause will be
315        added to the select query.
316        """
317
318        if type(columns_to_return) is str:
319            columns_to_return = [columns_to_return]
320        query = f"select * from {table}"
321        matches = []
322        if match_criteria:
323            query += f" where {self._get_conditions(match_criteria, exact_match)}"
324        if order_by:
325            query += f" order by {order_by}"
326        if limit:
327            query += f" limit {limit}"
328        query += ";"
329        self.cursor.execute(query)
330        matches = self.cursor.fetchall()
331        results = [self._get_dict(table, match, columns_to_return) for match in matches]
332        if sort_by_column:
333            results = sorted(results, key=lambda x: x[sort_by_column])
334        if return_as_dataframe:
335            return pandas.DataFrame(results)
336        if values_only:
337            return [tuple(row.values()) for row in results]
338        else:
339            return results
340
341    @_connect
342    def find(
343        self, table: str, query_string: str, columns: list[str] = None
344    ) -> tuple[dict]:
345        """Search for rows that contain query_string as a substring
346        of any column.
347
348        :param table: The table to search.
349
350        :param query_string: The substring to search for in all columns.
351
352        :param columns: A list of columns to search for query_string.
353        If None, all columns in the table will be searched.
354        """
355        if type(columns) is str:
356            columns = [columns]
357        results = []
358        if not columns:
359            columns = self.get_column_names(table)
360        for column in columns:
361            results.extend(
362                [
363                    row
364                    for row in self.get_rows(
365                        table, [(column, query_string)], exact_match=False
366                    )
367                    if row not in results
368                ]
369            )
370        return results
371
372    @_connect
373    def delete(
374        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
375    ) -> int:
376        """Delete records from table.
377
378        Returns number of deleted records.
379
380        :param match_criteria: Can be a list of 2-tuples where each
381        tuple is (columnName, rowValue) or a dictionary where
382        keys are column names and values are row values.
383
384        :param exact_match: If False, the rowValue for a give column
385        will be matched as a substring.
386        """
387        num_matches = self.count(table, match_criteria, exact_match)
388        conditions = self._get_conditions(match_criteria, exact_match)
389        try:
390            self.cursor.execute(f"delete from {table} where {conditions}")
391            self.logger.info(
392                f'Deleted {num_matches} from "{table}" where {conditions}".'
393            )
394            return num_matches
395        except Exception as e:
396            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
397            return 0
398
399    @_connect
400    def update(
401        self,
402        table: str,
403        column_to_update: str,
404        new_value: Any,
405        match_criteria: list[tuple] | dict = None,
406    ) -> bool:
407        """Update row value for entry matched with match_criteria.
408
409        :param column_to_update: The column to be updated in the matched row.
410
411        :param new_value: The new value to insert.
412
413        :param match_criteria: Can be a list of 2-tuples where each
414        tuple is (columnName, rowValue) or a dictionary where
415        keys are column names and values are row values.
416        If None, every row will be updated.
417
418        Returns True if successful, False if not."""
419        query = f"update {table} set {column_to_update} = ?"
420        if match_criteria:
421            if self.count(table, match_criteria) == 0:
422                self.logger.info(
423                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
424                )
425                return False
426            conditions = self._get_conditions(match_criteria)
427            query += f" where {conditions}"
428        else:
429            conditions = None
430        try:
431            self.cursor.execute(
432                query,
433                (new_value,),
434            )
435            self.logger.info(
436                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
437            )
438            return True
439        except UnboundLocalError:
440            table_filter_string = "\n".join(
441                table_filter for table_filter in match_criteria
442            )
443            self.logger.error(
444                f"No records found matching filters: {table_filter_string}"
445            )
446            return False
447        except Exception as e:
448            self.logger.error(
449                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
450            )
451            return False
452
453    @_connect
454    def drop_table(self, table: str) -> bool:
455        """Drop a table from the database.
456
457        Returns True if successful, False if not."""
458        try:
459            self.cursor.execute(f"drop Table {table}")
460            self.logger.info(f'Dropped table "{table}"')
461        except Exception as e:
462            print(e)
463            self.logger.error(f'Failed to drop table "{table}"')
464
465    @_connect
466    def add_column(
467        self, table: str, column: str, _type: str, default_value: str = None
468    ):
469        """Add a new column to table.
470
471        :param column: Name of the column to add.
472
473        :param _type: The data type of the new column.
474
475        :param default_value: Optional default value for the column."""
476        try:
477            if default_value:
478                self.cursor.execute(
479                    f"alter table {table} add column {column} {_type} default {default_value}"
480                )
481            else:
482                self.cursor.execute(f"alter table {table} add column {column} {_type}")
483            self.logger.info(f'Added column "{column}" to "{table}" table.')
484        except Exception as e:
485            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
486
487    @staticmethod
488    def data_to_string(
489        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
490    ) -> str:
491        """Uses tabulate to produce pretty string output
492        from a list of dictionaries.
493
494        :param data: Assumes all dictionaries in list have the same set of keys.
495
496        :param sort_key: Optional dictionary key to sort data with.
497
498        :param wrap_to_terminal: If True, the table width will be wrapped
499        to fit within the current terminal window. Set to False
500        if the output is going into something like a txt file."""
501        return data_to_string(data, sort_key, wrap_to_terminal)

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathlib.Path, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}')
34    def __init__(
35        self,
36        dbpath: str | Path,
37        logger_encoding: str = "utf-8",
38        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
39    ):
40        """
41        :param dbpath: String or Path object to database file.
42        If a relative path is given, it will be relative to the
43        current working directory. The log file will be saved to the
44        same directory.
45
46        :param logger_message_format: '{' style format string
47        for the logger object."""
48        self.dbpath = Path(dbpath)
49        self.dbname = Path(dbpath).name
50        self.dbpath.parent.mkdir(parents=True, exist_ok=True)
51        self._logger_init(
52            encoding=logger_encoding, message_format=logger_message_format
53        )
54        self.connection_open = False
55        self.create_manager()
Parameters
  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: '{' style format string for the logger object.

def create_manager(self):
64    def create_manager(self):
65        """Create dbmanager.py in the same directory
66        as the database file if it doesn't exist."""
67        manager_path = self.dbpath.parent / "dbmanager.py"
68        if not manager_path.exists():
69            manager_template = (Path(__file__).parent / "dbmanager.py").read_text()
70            manager_path.write_text(manager_template.replace("$dbname", self.dbname))

Create dbmanager.py in the same directory as the database file if it doesn't exist.

def open(self):
72    def open(self):
73        """Open connection to db."""
74        self.connection = sqlite3.connect(
75            self.dbpath,
76            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
77            timeout=10,
78        )
79        self.connection.execute("pragma foreign_keys = 1")
80        self.cursor = self.connection.cursor()
81        self.connection_open = True

Open connection to db.

def close(self):
83    def close(self):
84        """Save and close connection to db.
85
86        Call this as soon as you are done using the database if you have
87        multiple threads or processes using the same database."""
88        if self.connection_open:
89            self.connection.commit()
90            self.connection.close()
91            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def query(self, query_) -> list[typing.Any]:
160    @_connect
161    def query(self, query_) -> list[Any]:
162        """Execute an arbitrary query and
163        return the results."""
164        self.cursor.execute(query_)
165        return self.cursor.fetchall()

Execute an arbitrary query and return the results.

def create_tables(self, table_querys: list[str] = []):
167    @_connect
168    def create_tables(self, table_querys: list[str] = []):
169        """Create tables if they don't exist.
170
171        :param table_querys: Each query should be
172        in the form 'tableName(columnDefinitions)'"""
173        if len(table_querys) > 0:
174            table_names = self.get_table_names()
175            for table in table_querys:
176                if table.split("(")[0].strip() not in table_names:
177                    self.cursor.execute(f"create table {table}")
178                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_querys: Each query should be in the form 'tableName(columnDefinitions)'
def create_table(self, table: str, column_defs: list[str]):
180    @_connect
181    def create_table(self, table: str, column_defs: list[str]):
182        """Create a table if it doesn't exist.
183
184        :param table: Name of the table to create.
185
186        :param column_defs: List of column definitions in
187        proper Sqlite3 sytax.
188        i.e. "columnName text unique" or "columnName int primary key" etc."""
189        if table not in self.get_table_names():
190            query = f"create table {table}({', '.join(column_defs)})"
191            self.cursor.execute(query)
192            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

Parameters
  • table: Name of the table to create.

  • column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.

def get_table_names(self) -> list[str]:
194    @_connect
195    def get_table_names(self) -> list[str]:
196        """Returns a list of table names from database."""
197        self.cursor.execute(
198            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
199        )
200        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from database.

def get_column_names(self, table: str) -> list[str]:
202    @_connect
203    def get_column_names(self, table: str) -> list[str]:
204        """Return a list of column names from a table."""
205        self.cursor.execute(f"select * from {table} where 1=0")
206        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True) -> int:
208    @_connect
209    def count(
210        self,
211        table: str,
212        match_criteria: list[tuple] | dict = None,
213        exact_match: bool = True,
214    ) -> int:
215        """Return number of items in table.
216
217        :param match_criteria: Can be a list of 2-tuples where each
218        tuple is (columnName, rowValue) or a dictionary where
219        keys are column names and values are row values.
220        If None, all rows from the table will be counted.
221
222        :param exact_match: If False, the row value for a give column
223        in match_criteria will be matched as a substring. Has no effect if
224        match_criteria is None.
225        """
226        query = f"select count(_rowid_) from {table}"
227        try:
228            if match_criteria:
229                self.cursor.execute(
230                    f"{query} where {self._get_conditions(match_criteria, exact_match)}"
231                )
232            else:
233                self.cursor.execute(f"{query}")
234            return self.cursor.fetchone()[0]
235        except:
236            return 0

Return number of items in table.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

  • exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
238    @_connect
239    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
240        """Add row of values to table.
241
242        :param table: The table to insert into.
243
244        :param values: A tuple of values to be inserted into the table.
245
246        :param columns: If None, values param is expected to supply
247        a value for every column in the table. If columns is
248        provided, it should contain the same number of elements as values."""
249        parameterizer = ", ".join("?" for _ in values)
250        logger_values = ", ".join(str(value) for value in values)
251        try:
252            if columns:
253                columns = ", ".join(column for column in columns)
254                self.cursor.execute(
255                    f"insert into {table} ({columns}) values({parameterizer})", values
256                )
257            else:
258                self.cursor.execute(
259                    f"insert into {table} values({parameterizer})", values
260                )
261            self.logger.info(f'Added "{logger_values}" to {table} table.')
262        except Exception as e:
263            if "constraint" not in str(e).lower():
264                self.logger.exception(
265                    f'Error adding "{logger_values}" to {table} table.'
266                )
267            else:
268                self.logger.debug(str(e))

Add row of values to table.

Parameters
  • table: The table to insert into.

  • values: A tuple of values to be inserted into the table.

  • columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True, sort_by_column: str = None, columns_to_return: list[str] = None, return_as_dataframe: bool = False, values_only: bool = False, order_by: str = None, limit: str | int = None) -> list[dict] | list[tuple] | pandas.core.frame.DataFrame:
270    @_connect
271    def get_rows(
272        self,
273        table: str,
274        match_criteria: list[tuple] | dict = None,
275        exact_match: bool = True,
276        sort_by_column: str = None,
277        columns_to_return: list[str] = None,
278        return_as_dataframe: bool = False,
279        values_only: bool = False,
280        order_by: str = None,
281        limit: str | int = None,
282    ) -> list[dict] | list[tuple] | pandas.DataFrame:
283        """Returns rows from table as a list of dictionaries
284        where the key-value pairs of the dictionaries are
285        column name: row value.
286
287        :param match_criteria: Can be a list of 2-tuples where each
288        tuple is (columnName, rowValue) or a dictionary where
289        keys are column names and values are row values.
290
291        :param exact_match: If False, the rowValue for a give column
292        will be matched as a substring.
293
294        :param sort_by_column: A column name to sort the results by.
295        This will sort results in Python after retrieving them from the db.
296        Use the 'order_by' param to use SQLite engine for ordering.
297
298        :param columns_to_return: Optional list of column names.
299        If provided, the elements returned by get_rows() will
300        only contain the provided columns. Otherwise every column
301        in the row is returned.
302
303        :param return_as_dataframe: If True,
304        the results will be returned as a pandas.DataFrame object.
305
306        :param values_only: Return the results as a list of tuples
307        instead of a list of dictionaries that have column names as keys.
308        The results will still be sorted according to sort_by_column if
309        one is provided.
310
311        :param order_by: If given, a 'order by {order_by}' clause
312        will be added to the select query.
313
314        :param limit: If given, a 'limit {limit}' clause will be
315        added to the select query.
316        """
317
318        if type(columns_to_return) is str:
319            columns_to_return = [columns_to_return]
320        query = f"select * from {table}"
321        matches = []
322        if match_criteria:
323            query += f" where {self._get_conditions(match_criteria, exact_match)}"
324        if order_by:
325            query += f" order by {order_by}"
326        if limit:
327            query += f" limit {limit}"
328        query += ";"
329        self.cursor.execute(query)
330        matches = self.cursor.fetchall()
331        results = [self._get_dict(table, match, columns_to_return) for match in matches]
332        if sort_by_column:
333            results = sorted(results, key=lambda x: x[sort_by_column])
334        if return_as_dataframe:
335            return pandas.DataFrame(results)
336        if values_only:
337            return [tuple(row.values()) for row in results]
338        else:
339            return results

Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

  • sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.

  • columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.

  • return_as_dataframe: If True, the results will be returned as a pandas.DataFrame object.

  • values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.

  • order_by: If given, a 'order by {order_by}' clause will be added to the select query.

  • limit: If given, a 'limit {limit}' clause will be added to the select query.

def find( self, table: str, query_string: str, columns: list[str] = None) -> tuple[dict]:
341    @_connect
342    def find(
343        self, table: str, query_string: str, columns: list[str] = None
344    ) -> tuple[dict]:
345        """Search for rows that contain query_string as a substring
346        of any column.
347
348        :param table: The table to search.
349
350        :param query_string: The substring to search for in all columns.
351
352        :param columns: A list of columns to search for query_string.
353        If None, all columns in the table will be searched.
354        """
355        if type(columns) is str:
356            columns = [columns]
357        results = []
358        if not columns:
359            columns = self.get_column_names(table)
360        for column in columns:
361            results.extend(
362                [
363                    row
364                    for row in self.get_rows(
365                        table, [(column, query_string)], exact_match=False
366                    )
367                    if row not in results
368                ]
369            )
370        return results

Search for rows that contain query_string as a substring of any column.

Parameters
  • table: The table to search.

  • query_string: The substring to search for in all columns.

  • columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
372    @_connect
373    def delete(
374        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
375    ) -> int:
376        """Delete records from table.
377
378        Returns number of deleted records.
379
380        :param match_criteria: Can be a list of 2-tuples where each
381        tuple is (columnName, rowValue) or a dictionary where
382        keys are column names and values are row values.
383
384        :param exact_match: If False, the rowValue for a give column
385        will be matched as a substring.
386        """
387        num_matches = self.count(table, match_criteria, exact_match)
388        conditions = self._get_conditions(match_criteria, exact_match)
389        try:
390            self.cursor.execute(f"delete from {table} where {conditions}")
391            self.logger.info(
392                f'Deleted {num_matches} from "{table}" where {conditions}".'
393            )
394            return num_matches
395        except Exception as e:
396            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
397            return 0

Delete records from table.

Returns number of deleted records.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict = None) -> bool:
399    @_connect
400    def update(
401        self,
402        table: str,
403        column_to_update: str,
404        new_value: Any,
405        match_criteria: list[tuple] | dict = None,
406    ) -> bool:
407        """Update row value for entry matched with match_criteria.
408
409        :param column_to_update: The column to be updated in the matched row.
410
411        :param new_value: The new value to insert.
412
413        :param match_criteria: Can be a list of 2-tuples where each
414        tuple is (columnName, rowValue) or a dictionary where
415        keys are column names and values are row values.
416        If None, every row will be updated.
417
418        Returns True if successful, False if not."""
419        query = f"update {table} set {column_to_update} = ?"
420        if match_criteria:
421            if self.count(table, match_criteria) == 0:
422                self.logger.info(
423                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
424                )
425                return False
426            conditions = self._get_conditions(match_criteria)
427            query += f" where {conditions}"
428        else:
429            conditions = None
430        try:
431            self.cursor.execute(
432                query,
433                (new_value,),
434            )
435            self.logger.info(
436                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
437            )
438            return True
439        except UnboundLocalError:
440            table_filter_string = "\n".join(
441                table_filter for table_filter in match_criteria
442            )
443            self.logger.error(
444                f"No records found matching filters: {table_filter_string}"
445            )
446            return False
447        except Exception as e:
448            self.logger.error(
449                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
450            )
451            return False

Update row value for entry matched with match_criteria.

Parameters
  • column_to_update: The column to be updated in the matched row.

  • new_value: The new value to insert.

  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.

Returns True if successful, False if not.

def drop_table(self, table: str) -> bool:
453    @_connect
454    def drop_table(self, table: str) -> bool:
455        """Drop a table from the database.
456
457        Returns True if successful, False if not."""
458        try:
459            self.cursor.execute(f"drop Table {table}")
460            self.logger.info(f'Dropped table "{table}"')
461        except Exception as e:
462            print(e)
463            self.logger.error(f'Failed to drop table "{table}"')

Drop a table from the database.

Returns True if successful, False if not.

def add_column(self, table: str, column: str, _type: str, default_value: str = None):
465    @_connect
466    def add_column(
467        self, table: str, column: str, _type: str, default_value: str = None
468    ):
469        """Add a new column to table.
470
471        :param column: Name of the column to add.
472
473        :param _type: The data type of the new column.
474
475        :param default_value: Optional default value for the column."""
476        try:
477            if default_value:
478                self.cursor.execute(
479                    f"alter table {table} add column {column} {_type} default {default_value}"
480                )
481            else:
482                self.cursor.execute(f"alter table {table} add column {column} {_type}")
483            self.logger.info(f'Added column "{column}" to "{table}" table.')
484        except Exception as e:
485            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

Parameters
  • column: Name of the column to add.

  • _type: The data type of the new column.

  • default_value: Optional default value for the column.

@staticmethod
def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
487    @staticmethod
488    def data_to_string(
489        data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
490    ) -> str:
491        """Uses tabulate to produce pretty string output
492        from a list of dictionaries.
493
494        :param data: Assumes all dictionaries in list have the same set of keys.
495
496        :param sort_key: Optional dictionary key to sort data with.
497
498        :param wrap_to_terminal: If True, the table width will be wrapped
499        to fit within the current terminal window. Set to False
500        if the output is going into something like a txt file."""
501        return data_to_string(data, sort_key, wrap_to_terminal)

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.

def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
504def data_to_string(
505    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
506) -> str:
507    """Use tabulate to produce grid output from a list of dictionaries.
508
509    :param data: Assumes all dictionaries in list have the same set of keys.
510
511    :param sort_key: Optional dictionary key to sort data with.
512
513    :param wrap_to_terminal: If True, the column widths will be reduced so the grid fits
514    within the current terminal window without wrapping. If the column widths have reduced to 1
515    and the grid is still too wide, str(data) will be returned."""
516    if len(data) == 0:
517        return ""
518    if sort_key:
519        data = sorted(data, key=lambda d: d[sort_key])
520    for i, d in enumerate(data):
521        for k in d:
522            data[i][k] = str(data[i][k])
523
524    too_wide = True
525    terminal_width = os.get_terminal_size().columns
526    max_col_widths = terminal_width
527    # Make an output with effectively unrestricted column widths
528    # to see if shrinking is necessary
529    output = tabulate(
530        data,
531        headers="keys",
532        disable_numparse=True,
533        tablefmt="grid",
534        maxcolwidths=max_col_widths,
535    )
536    current_width = output.index("\n")
537    if current_width < terminal_width:
538        too_wide = False
539    if wrap_to_terminal and too_wide:
540        previous_col_widths = max_col_widths
541        while too_wide and max_col_widths > 1:
542            if current_width > terminal_width:
543                previous_col_widths = max_col_widths
544                max_col_widths = int(max_col_widths * 0.5)
545            elif current_width < terminal_width:
546                max_col_widths = int(
547                    max_col_widths + ((previous_col_widths - max_col_widths) * 0.5)
548                )
549            output = tabulate(
550                data,
551                headers="keys",
552                disable_numparse=True,
553                tablefmt="grid",
554                maxcolwidths=max_col_widths,
555            )
556            current_width = output.index("\n")
557            if terminal_width - 10 < current_width < terminal_width:
558                too_wide = False
559        if too_wide:
560            return str(data)
561        return output

Use tabulate to produce grid output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the column widths will be reduced so the grid fits within the current terminal window without wrapping. If the column widths have reduced to 1 and the grid is still too wide, str(data) will be returned.