databased.databased

  1import logging
  2import os
  3import sqlite3
  4from datetime import datetime
  5from functools import wraps
  6from pathlib import Path
  7from typing import Any
  8
  9from tabulate import tabulate
 10
 11
 12class DataBased:
 13    """Sqli wrapper so queries don't need to be written except table definitions.
 14
 15    Supports saving and reading dates as datetime objects.
 16
 17    Supports using a context manager."""
 18
 19    def __init__(
 20        self,
 21        dbpath: str | Path,
 22        logger_encoding: str = "utf-8",
 23        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 24    ):
 25        """
 26        :param dbpath: String or Path object to database file.
 27        If a relative path is given, it will be relative to the
 28        current working directory. The log file will be saved to the
 29        same directory.
 30
 31        :param logger_message_format: '{' style format string
 32        for the logger object."""
 33        self.dbpath = Path(dbpath)
 34        self.dbname = Path(dbpath).name
 35        self._logger_init(
 36            encoding=logger_encoding, message_format=logger_message_format
 37        )
 38        self.connection_open = False
 39        self.create_manager()
 40
 41    def __enter__(self):
 42        self.open()
 43        return self
 44
 45    def __exit__(self, exception_type, exception_value, exception_traceback):
 46        self.close()
 47
 48    def create_manager(self):
 49        """Create dbManager.py in the same directory
 50        as the database file if it doesn't exist."""
 51        manager_template = Path(__file__).parent / "dbmanager.py"
 52        manager_path = self.dbpath.parent / "dbmanager.py"
 53        if not manager_path.exists():
 54            manager_path.write_text(manager_template.read_text())
 55
 56    def open(self):
 57        """Open connection to db."""
 58        self.connection = sqlite3.connect(
 59            self.dbpath,
 60            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 61            timeout=10,
 62        )
 63        self.connection.execute("pragma foreign_keys = 1")
 64        self.cursor = self.connection.cursor()
 65        self.connection_open = True
 66
 67    def close(self):
 68        """Save and close connection to db.
 69
 70        Call this as soon as you are done using the database if you have
 71        multiple threads or processes using the same database."""
 72        if self.connection_open:
 73            self.connection.commit()
 74            self.connection.close()
 75            self.connection_open = False
 76
 77    def _connect(func):
 78        """Decorator to open db connection if it isn't already open."""
 79
 80        @wraps(func)
 81        def inner(*args, **kwargs):
 82            self = args[0]
 83            if not self.connection_open:
 84                self.open()
 85            results = func(*args, **kwargs)
 86            return results
 87
 88        return inner
 89
 90    def _logger_init(
 91        self,
 92        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 93        encoding: str = "utf-8",
 94    ):
 95        """:param message_format: '{' style format string"""
 96        self.logger = logging.getLogger(self.dbname)
 97        if not self.logger.hasHandlers():
 98            handler = logging.FileHandler(
 99                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
100            )
101            handler.setFormatter(
102                logging.Formatter(
103                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
104                )
105            )
106            self.logger.addHandler(handler)
107            self.logger.setLevel(logging.INFO)
108
109    def _get_dict(
110        self, table: str, values: list, columns_to_return: list[str] = None
111    ) -> dict:
112        """Converts the values of a row into a dictionary with column names as keys.
113
114        :param table: The table that values were pulled from.
115
116        :param values: List of values expected to be the same quantity
117        and in the same order as the column names of table.
118
119        :param columns_to_return: An optional list of column names.
120        If given, only these columns will be included in the returned dictionary.
121        Otherwise all columns and values are returned."""
122        return {
123            column: value
124            for column, value in zip(self.get_column_names(table), values)
125            if not columns_to_return or column in columns_to_return
126        }
127
128    def _get_conditions(
129        self, match_criteria: list[tuple] | dict, exact_match: bool = True
130    ) -> str:
131        """Builds and returns the conditional portion of a query.
132
133        :param match_criteria: Can be a list of 2-tuples where each
134        tuple is (columnName, rowValue) or a dictionary where
135        keys are column names and values are row values.
136
137        :param exact_match: If False, the rowValue for a give column
138        will be matched as a substring.
139
140        Usage e.g.:
141
142        self.cursor.execute(f'select * from {table} where {conditions}')"""
143        if type(match_criteria) == dict:
144            match_criteria = [(k, v) for k, v in match_criteria.items()]
145        if exact_match:
146            conditions = " and ".join(
147                f'"{column_row[0]}" = "{column_row[1]}"'
148                for column_row in match_criteria
149            )
150        else:
151            conditions = " and ".join(
152                f'"{column_row[0]}" like "%{column_row[1]}%"'
153                for column_row in match_criteria
154            )
155        return f"({conditions})"
156
157    @_connect
158    def create_tables(self, table_statements: list[str] = []):
159        """Create tables if they don't exist.
160
161        :param table_statements: Each statement should be
162        in the form 'tableName(columnDefinitions)'"""
163        if len(table_statements) > 0:
164            table_names = self.get_table_names()
165            for table in table_statements:
166                if table.split("(")[0].strip() not in table_names:
167                    self.cursor.execute(f"create table {table}")
168                    self.logger.info(f'{table.split("(")[0]} table created.')
169
170    @_connect
171    def create_table(self, table: str, column_defs: list[str]):
172        """Create a table if it doesn't exist.
173
174        :param table: Name of the table to create.
175
176        :param column_defs: List of column definitions in
177        proper Sqlite3 sytax.
178        i.e. "columnName text unique" or "columnName int primary key" etc."""
179        if table not in self.get_table_names():
180            statement = f"create table {table}({', '.join(column_defs)})"
181            self.cursor.execute(statement)
182            self.logger.info(f"'{table}' table created.")
183
184    @_connect
185    def get_table_names(self) -> list[str]:
186        """Returns a list of table names from database."""
187        self.cursor.execute(
188            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
189        )
190        return [result[0] for result in self.cursor.fetchall()]
191
192    @_connect
193    def get_column_names(self, table: str) -> list[str]:
194        """Return a list of column names from a table."""
195        self.cursor.execute(f"select * from {table} where 1=0")
196        return [description[0] for description in self.cursor.description]
197
198    @_connect
199    def count(
200        self,
201        table: str,
202        match_criteria: list[tuple] | dict = None,
203        exact_match: bool = True,
204    ) -> int:
205        """Return number of items in table.
206
207        :param match_criteria: Can be a list of 2-tuples where each
208        tuple is (columnName, rowValue) or a dictionary where
209        keys are column names and values are row values.
210        If None, all rows from the table will be counted.
211
212        :param exact_match: If False, the row value for a give column
213        in match_criteria will be matched as a substring. Has no effect if
214        match_criteria is None.
215        """
216        statement = f"select count(_rowid_) from {table}"
217        try:
218            if match_criteria:
219                self.cursor.execute(
220                    f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
221                )
222            else:
223                self.cursor.execute(f"{statement}")
224            return self.cursor.fetchone()[0]
225        except:
226            return 0
227
228    @_connect
229    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
230        """Add row of values to table.
231
232        :param table: The table to insert into.
233
234        :param values: A tuple of values to be inserted into the table.
235
236        :param columns: If None, values param is expected to supply
237        a value for every column in the table. If columns is
238        provided, it should contain the same number of elements as values."""
239        parameterizer = ", ".join("?" for _ in values)
240        logger_values = ", ".join(str(value) for value in values)
241        try:
242            if columns:
243                columns = ", ".join(column for column in columns)
244                self.cursor.execute(
245                    f"insert into {table} ({columns}) values({parameterizer})", values
246                )
247            else:
248                self.cursor.execute(
249                    f"insert into {table} values({parameterizer})", values
250                )
251            self.logger.info(f'Added "{logger_values}" to {table} table.')
252        except Exception as e:
253            if "constraint" not in str(e).lower():
254                self.logger.exception(
255                    f'Error adding "{logger_values}" to {table} table.'
256                )
257            else:
258                self.logger.debug(str(e))
259
260    @_connect
261    def get_rows(
262        self,
263        table: str,
264        match_criteria: list[tuple] | dict = None,
265        exact_match: bool = True,
266        sort_by_column: str = None,
267        columns_to_return: list[str] = None,
268        values_only: bool = False,
269    ) -> list[dict] | list[tuple]:
270        """Returns rows from table as a list of dictionaries
271        where the key-value pairs of the dictionaries are
272        column name: row value.
273
274        :param match_criteria: Can be a list of 2-tuples where each
275        tuple is (columnName, rowValue) or a dictionary where
276        keys are column names and values are row values.
277
278        :param exact_match: If False, the rowValue for a give column
279        will be matched as a substring.
280
281        :param sort_by_column: A column name to sort the results by.
282
283        :param columns_to_return: Optional list of column names.
284        If provided, the elements returned by get_rows() will
285        only contain the provided columns. Otherwise every column
286        in the row is returned.
287
288        :param values_only: Return the results as a list of tuples
289        instead of a list of dictionaries that have column names as keys.
290        The results will still be sorted according to sort_by_column if
291        one is provided.
292        """
293        if type(columns_to_return) is str:
294            columns_to_return = [columns_to_return]
295        statement = f"select * from {table}"
296        matches = []
297        if not match_criteria:
298            self.cursor.execute(statement)
299        else:
300            self.cursor.execute(
301                f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
302            )
303        matches = self.cursor.fetchall()
304        results = [self._get_dict(table, match, columns_to_return) for match in matches]
305        if sort_by_column:
306            results = sorted(results, key=lambda x: x[sort_by_column])
307        if values_only:
308            return [tuple(row.values()) for row in results]
309        else:
310            return results
311
312    @_connect
313    def find(
314        self, table: str, query_string: str, columns: list[str] = None
315    ) -> tuple[dict]:
316        """Search for rows that contain query_string as a substring
317        of any column.
318
319        :param table: The table to search.
320
321        :param query_string: The substring to search for in all columns.
322
323        :param columns: A list of columns to search for query_string.
324        If None, all columns in the table will be searched.
325        """
326        if type(columns) is str:
327            columns = [columns]
328        results = []
329        if not columns:
330            columns = self.get_column_names(table)
331        for column in columns:
332            results.extend(
333                [
334                    row
335                    for row in self.get_rows(
336                        table, [(column, query_string)], exact_match=False
337                    )
338                    if row not in results
339                ]
340            )
341        return results
342
343    @_connect
344    def delete(
345        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
346    ) -> int:
347        """Delete records from table.
348
349        Returns number of deleted records.
350
351        :param match_criteria: Can be a list of 2-tuples where each
352        tuple is (columnName, rowValue) or a dictionary where
353        keys are column names and values are row values.
354
355        :param exact_match: If False, the rowValue for a give column
356        will be matched as a substring.
357        """
358        num_matches = self.count(table, match_criteria, exact_match)
359        conditions = self._get_conditions(match_criteria, exact_match)
360        try:
361            self.cursor.execute(f"delete from {table} where {conditions}")
362            self.logger.info(
363                f'Deleted {num_matches} from "{table}" where {conditions}".'
364            )
365            return num_matches
366        except Exception as e:
367            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
368            return 0
369
370    @_connect
371    def update(
372        self,
373        table: str,
374        column_to_update: str,
375        new_value: Any,
376        match_criteria: list[tuple] | dict = None,
377    ) -> bool:
378        """Update row value for entry matched with match_criteria.
379
380        :param column_to_update: The column to be updated in the matched row.
381
382        :param new_value: The new value to insert.
383
384        :param match_criteria: Can be a list of 2-tuples where each
385        tuple is (columnName, rowValue) or a dictionary where
386        keys are column names and values are row values.
387        If None, every row will be updated.
388
389        Returns True if successful, False if not."""
390        statement = f"update {table} set {column_to_update} = ?"
391        if match_criteria:
392            if self.count(table, match_criteria) == 0:
393                self.logger.info(
394                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
395                )
396                return False
397            conditions = self._get_conditions(match_criteria)
398            statement += f" where {conditions}"
399        else:
400            conditions = None
401        try:
402            self.cursor.execute(
403                statement,
404                (new_value,),
405            )
406            self.logger.info(
407                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
408            )
409            return True
410        except UnboundLocalError:
411            table_filter_string = "\n".join(
412                table_filter for table_filter in match_criteria
413            )
414            self.logger.error(
415                f"No records found matching filters: {table_filter_string}"
416            )
417            return False
418        except Exception as e:
419            self.logger.error(
420                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
421            )
422            return False
423
424    @_connect
425    def drop_table(self, table: str) -> bool:
426        """Drop a table from the database.
427
428        Returns True if successful, False if not."""
429        try:
430            self.cursor.execute(f"drop Table {table}")
431            self.logger.info(f'Dropped table "{table}"')
432        except Exception as e:
433            print(e)
434            self.logger.error(f'Failed to drop table "{table}"')
435
436    @_connect
437    def add_column(
438        self, table: str, column: str, _type: str, default_value: str = None
439    ):
440        """Add a new column to table.
441
442        :param column: Name of the column to add.
443
444        :param _type: The data type of the new column.
445
446        :param default_value: Optional default value for the column."""
447        try:
448            if default_value:
449                self.cursor.execute(
450                    f"alter table {table} add column {column} {_type} default {default_value}"
451                )
452            else:
453                self.cursor.execute(f"alter table {table} add column {column} {_type}")
454            self.logger.info(f'Added column "{column}" to "{table}" table.')
455        except Exception as e:
456            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
457
458
459def data_to_string(
460    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
461) -> str:
462    """Uses tabulate to produce pretty string output
463    from a list of dictionaries.
464
465    :param data: Assumes all dictionaries in list have the same set of keys.
466
467    :param sort_key: Optional dictionary key to sort data with.
468
469    :param wrap_to_terminal: If True, the table width will be wrapped
470    to fit within the current terminal window. Set to False
471    if the output is going into something like a txt file."""
472    if len(data) == 0:
473        return ""
474    if sort_key:
475        data = sorted(data, key=lambda d: d[sort_key])
476    for i, d in enumerate(data):
477        for k in d:
478            data[i][k] = str(data[i][k])
479    if wrap_to_terminal:
480        terminal_width = os.get_terminal_size().columns
481        max_col_widths = terminal_width
482        """ Reducing the column width by tabulating one row at a time
483        and then reducing further by tabulating the whole set proved to be 
484        faster than going straight to tabulating the whole set and reducing
485        the column width."""
486        too_wide = True
487        while too_wide and max_col_widths > 1:
488            for i, row in enumerate(data):
489                output = tabulate(
490                    [row],
491                    headers="keys",
492                    disable_numparse=True,
493                    tablefmt="grid",
494                    maxcolwidths=max_col_widths,
495                )
496                if output.index("\n") > terminal_width:
497                    max_col_widths -= 2
498                    too_wide = True
499                    break
500                too_wide = False
501    else:
502        max_col_widths = None
503    output = tabulate(
504        data,
505        headers="keys",
506        disable_numparse=True,
507        tablefmt="grid",
508        maxcolwidths=max_col_widths,
509    )
510    # trim max column width until the output string is less wide than the current terminal width.
511    if wrap_to_terminal:
512        while output.index("\n") > terminal_width and max_col_widths > 1:
513            max_col_widths -= 2
514            max_col_widths = max(1, max_col_widths)
515            output = tabulate(
516                data,
517                headers="keys",
518                disable_numparse=True,
519                tablefmt="grid",
520                maxcolwidths=max_col_widths,
521            )
522    return output
class DataBased:
 13class DataBased:
 14    """Sqli wrapper so queries don't need to be written except table definitions.
 15
 16    Supports saving and reading dates as datetime objects.
 17
 18    Supports using a context manager."""
 19
 20    def __init__(
 21        self,
 22        dbpath: str | Path,
 23        logger_encoding: str = "utf-8",
 24        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 25    ):
 26        """
 27        :param dbpath: String or Path object to database file.
 28        If a relative path is given, it will be relative to the
 29        current working directory. The log file will be saved to the
 30        same directory.
 31
 32        :param logger_message_format: '{' style format string
 33        for the logger object."""
 34        self.dbpath = Path(dbpath)
 35        self.dbname = Path(dbpath).name
 36        self._logger_init(
 37            encoding=logger_encoding, message_format=logger_message_format
 38        )
 39        self.connection_open = False
 40        self.create_manager()
 41
 42    def __enter__(self):
 43        self.open()
 44        return self
 45
 46    def __exit__(self, exception_type, exception_value, exception_traceback):
 47        self.close()
 48
 49    def create_manager(self):
 50        """Create dbManager.py in the same directory
 51        as the database file if it doesn't exist."""
 52        manager_template = Path(__file__).parent / "dbmanager.py"
 53        manager_path = self.dbpath.parent / "dbmanager.py"
 54        if not manager_path.exists():
 55            manager_path.write_text(manager_template.read_text())
 56
 57    def open(self):
 58        """Open connection to db."""
 59        self.connection = sqlite3.connect(
 60            self.dbpath,
 61            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 62            timeout=10,
 63        )
 64        self.connection.execute("pragma foreign_keys = 1")
 65        self.cursor = self.connection.cursor()
 66        self.connection_open = True
 67
 68    def close(self):
 69        """Save and close connection to db.
 70
 71        Call this as soon as you are done using the database if you have
 72        multiple threads or processes using the same database."""
 73        if self.connection_open:
 74            self.connection.commit()
 75            self.connection.close()
 76            self.connection_open = False
 77
 78    def _connect(func):
 79        """Decorator to open db connection if it isn't already open."""
 80
 81        @wraps(func)
 82        def inner(*args, **kwargs):
 83            self = args[0]
 84            if not self.connection_open:
 85                self.open()
 86            results = func(*args, **kwargs)
 87            return results
 88
 89        return inner
 90
 91    def _logger_init(
 92        self,
 93        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 94        encoding: str = "utf-8",
 95    ):
 96        """:param message_format: '{' style format string"""
 97        self.logger = logging.getLogger(self.dbname)
 98        if not self.logger.hasHandlers():
 99            handler = logging.FileHandler(
100                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
101            )
102            handler.setFormatter(
103                logging.Formatter(
104                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
105                )
106            )
107            self.logger.addHandler(handler)
108            self.logger.setLevel(logging.INFO)
109
110    def _get_dict(
111        self, table: str, values: list, columns_to_return: list[str] = None
112    ) -> dict:
113        """Converts the values of a row into a dictionary with column names as keys.
114
115        :param table: The table that values were pulled from.
116
117        :param values: List of values expected to be the same quantity
118        and in the same order as the column names of table.
119
120        :param columns_to_return: An optional list of column names.
121        If given, only these columns will be included in the returned dictionary.
122        Otherwise all columns and values are returned."""
123        return {
124            column: value
125            for column, value in zip(self.get_column_names(table), values)
126            if not columns_to_return or column in columns_to_return
127        }
128
129    def _get_conditions(
130        self, match_criteria: list[tuple] | dict, exact_match: bool = True
131    ) -> str:
132        """Builds and returns the conditional portion of a query.
133
134        :param match_criteria: Can be a list of 2-tuples where each
135        tuple is (columnName, rowValue) or a dictionary where
136        keys are column names and values are row values.
137
138        :param exact_match: If False, the rowValue for a give column
139        will be matched as a substring.
140
141        Usage e.g.:
142
143        self.cursor.execute(f'select * from {table} where {conditions}')"""
144        if type(match_criteria) == dict:
145            match_criteria = [(k, v) for k, v in match_criteria.items()]
146        if exact_match:
147            conditions = " and ".join(
148                f'"{column_row[0]}" = "{column_row[1]}"'
149                for column_row in match_criteria
150            )
151        else:
152            conditions = " and ".join(
153                f'"{column_row[0]}" like "%{column_row[1]}%"'
154                for column_row in match_criteria
155            )
156        return f"({conditions})"
157
158    @_connect
159    def create_tables(self, table_statements: list[str] = []):
160        """Create tables if they don't exist.
161
162        :param table_statements: Each statement should be
163        in the form 'tableName(columnDefinitions)'"""
164        if len(table_statements) > 0:
165            table_names = self.get_table_names()
166            for table in table_statements:
167                if table.split("(")[0].strip() not in table_names:
168                    self.cursor.execute(f"create table {table}")
169                    self.logger.info(f'{table.split("(")[0]} table created.')
170
171    @_connect
172    def create_table(self, table: str, column_defs: list[str]):
173        """Create a table if it doesn't exist.
174
175        :param table: Name of the table to create.
176
177        :param column_defs: List of column definitions in
178        proper Sqlite3 sytax.
179        i.e. "columnName text unique" or "columnName int primary key" etc."""
180        if table not in self.get_table_names():
181            statement = f"create table {table}({', '.join(column_defs)})"
182            self.cursor.execute(statement)
183            self.logger.info(f"'{table}' table created.")
184
185    @_connect
186    def get_table_names(self) -> list[str]:
187        """Returns a list of table names from database."""
188        self.cursor.execute(
189            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
190        )
191        return [result[0] for result in self.cursor.fetchall()]
192
193    @_connect
194    def get_column_names(self, table: str) -> list[str]:
195        """Return a list of column names from a table."""
196        self.cursor.execute(f"select * from {table} where 1=0")
197        return [description[0] for description in self.cursor.description]
198
199    @_connect
200    def count(
201        self,
202        table: str,
203        match_criteria: list[tuple] | dict = None,
204        exact_match: bool = True,
205    ) -> int:
206        """Return number of items in table.
207
208        :param match_criteria: Can be a list of 2-tuples where each
209        tuple is (columnName, rowValue) or a dictionary where
210        keys are column names and values are row values.
211        If None, all rows from the table will be counted.
212
213        :param exact_match: If False, the row value for a give column
214        in match_criteria will be matched as a substring. Has no effect if
215        match_criteria is None.
216        """
217        statement = f"select count(_rowid_) from {table}"
218        try:
219            if match_criteria:
220                self.cursor.execute(
221                    f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
222                )
223            else:
224                self.cursor.execute(f"{statement}")
225            return self.cursor.fetchone()[0]
226        except:
227            return 0
228
229    @_connect
230    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
231        """Add row of values to table.
232
233        :param table: The table to insert into.
234
235        :param values: A tuple of values to be inserted into the table.
236
237        :param columns: If None, values param is expected to supply
238        a value for every column in the table. If columns is
239        provided, it should contain the same number of elements as values."""
240        parameterizer = ", ".join("?" for _ in values)
241        logger_values = ", ".join(str(value) for value in values)
242        try:
243            if columns:
244                columns = ", ".join(column for column in columns)
245                self.cursor.execute(
246                    f"insert into {table} ({columns}) values({parameterizer})", values
247                )
248            else:
249                self.cursor.execute(
250                    f"insert into {table} values({parameterizer})", values
251                )
252            self.logger.info(f'Added "{logger_values}" to {table} table.')
253        except Exception as e:
254            if "constraint" not in str(e).lower():
255                self.logger.exception(
256                    f'Error adding "{logger_values}" to {table} table.'
257                )
258            else:
259                self.logger.debug(str(e))
260
261    @_connect
262    def get_rows(
263        self,
264        table: str,
265        match_criteria: list[tuple] | dict = None,
266        exact_match: bool = True,
267        sort_by_column: str = None,
268        columns_to_return: list[str] = None,
269        values_only: bool = False,
270    ) -> list[dict] | list[tuple]:
271        """Returns rows from table as a list of dictionaries
272        where the key-value pairs of the dictionaries are
273        column name: row value.
274
275        :param match_criteria: Can be a list of 2-tuples where each
276        tuple is (columnName, rowValue) or a dictionary where
277        keys are column names and values are row values.
278
279        :param exact_match: If False, the rowValue for a give column
280        will be matched as a substring.
281
282        :param sort_by_column: A column name to sort the results by.
283
284        :param columns_to_return: Optional list of column names.
285        If provided, the elements returned by get_rows() will
286        only contain the provided columns. Otherwise every column
287        in the row is returned.
288
289        :param values_only: Return the results as a list of tuples
290        instead of a list of dictionaries that have column names as keys.
291        The results will still be sorted according to sort_by_column if
292        one is provided.
293        """
294        if type(columns_to_return) is str:
295            columns_to_return = [columns_to_return]
296        statement = f"select * from {table}"
297        matches = []
298        if not match_criteria:
299            self.cursor.execute(statement)
300        else:
301            self.cursor.execute(
302                f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
303            )
304        matches = self.cursor.fetchall()
305        results = [self._get_dict(table, match, columns_to_return) for match in matches]
306        if sort_by_column:
307            results = sorted(results, key=lambda x: x[sort_by_column])
308        if values_only:
309            return [tuple(row.values()) for row in results]
310        else:
311            return results
312
313    @_connect
314    def find(
315        self, table: str, query_string: str, columns: list[str] = None
316    ) -> tuple[dict]:
317        """Search for rows that contain query_string as a substring
318        of any column.
319
320        :param table: The table to search.
321
322        :param query_string: The substring to search for in all columns.
323
324        :param columns: A list of columns to search for query_string.
325        If None, all columns in the table will be searched.
326        """
327        if type(columns) is str:
328            columns = [columns]
329        results = []
330        if not columns:
331            columns = self.get_column_names(table)
332        for column in columns:
333            results.extend(
334                [
335                    row
336                    for row in self.get_rows(
337                        table, [(column, query_string)], exact_match=False
338                    )
339                    if row not in results
340                ]
341            )
342        return results
343
344    @_connect
345    def delete(
346        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
347    ) -> int:
348        """Delete records from table.
349
350        Returns number of deleted records.
351
352        :param match_criteria: Can be a list of 2-tuples where each
353        tuple is (columnName, rowValue) or a dictionary where
354        keys are column names and values are row values.
355
356        :param exact_match: If False, the rowValue for a give column
357        will be matched as a substring.
358        """
359        num_matches = self.count(table, match_criteria, exact_match)
360        conditions = self._get_conditions(match_criteria, exact_match)
361        try:
362            self.cursor.execute(f"delete from {table} where {conditions}")
363            self.logger.info(
364                f'Deleted {num_matches} from "{table}" where {conditions}".'
365            )
366            return num_matches
367        except Exception as e:
368            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
369            return 0
370
371    @_connect
372    def update(
373        self,
374        table: str,
375        column_to_update: str,
376        new_value: Any,
377        match_criteria: list[tuple] | dict = None,
378    ) -> bool:
379        """Update row value for entry matched with match_criteria.
380
381        :param column_to_update: The column to be updated in the matched row.
382
383        :param new_value: The new value to insert.
384
385        :param match_criteria: Can be a list of 2-tuples where each
386        tuple is (columnName, rowValue) or a dictionary where
387        keys are column names and values are row values.
388        If None, every row will be updated.
389
390        Returns True if successful, False if not."""
391        statement = f"update {table} set {column_to_update} = ?"
392        if match_criteria:
393            if self.count(table, match_criteria) == 0:
394                self.logger.info(
395                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
396                )
397                return False
398            conditions = self._get_conditions(match_criteria)
399            statement += f" where {conditions}"
400        else:
401            conditions = None
402        try:
403            self.cursor.execute(
404                statement,
405                (new_value,),
406            )
407            self.logger.info(
408                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
409            )
410            return True
411        except UnboundLocalError:
412            table_filter_string = "\n".join(
413                table_filter for table_filter in match_criteria
414            )
415            self.logger.error(
416                f"No records found matching filters: {table_filter_string}"
417            )
418            return False
419        except Exception as e:
420            self.logger.error(
421                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
422            )
423            return False
424
425    @_connect
426    def drop_table(self, table: str) -> bool:
427        """Drop a table from the database.
428
429        Returns True if successful, False if not."""
430        try:
431            self.cursor.execute(f"drop Table {table}")
432            self.logger.info(f'Dropped table "{table}"')
433        except Exception as e:
434            print(e)
435            self.logger.error(f'Failed to drop table "{table}"')
436
437    @_connect
438    def add_column(
439        self, table: str, column: str, _type: str, default_value: str = None
440    ):
441        """Add a new column to table.
442
443        :param column: Name of the column to add.
444
445        :param _type: The data type of the new column.
446
447        :param default_value: Optional default value for the column."""
448        try:
449            if default_value:
450                self.cursor.execute(
451                    f"alter table {table} add column {column} {_type} default {default_value}"
452                )
453            else:
454                self.cursor.execute(f"alter table {table} add column {column} {_type}")
455            self.logger.info(f'Added column "{column}" to "{table}" table.')
456        except Exception as e:
457            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathlib.Path, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}')
20    def __init__(
21        self,
22        dbpath: str | Path,
23        logger_encoding: str = "utf-8",
24        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
25    ):
26        """
27        :param dbpath: String or Path object to database file.
28        If a relative path is given, it will be relative to the
29        current working directory. The log file will be saved to the
30        same directory.
31
32        :param logger_message_format: '{' style format string
33        for the logger object."""
34        self.dbpath = Path(dbpath)
35        self.dbname = Path(dbpath).name
36        self._logger_init(
37            encoding=logger_encoding, message_format=logger_message_format
38        )
39        self.connection_open = False
40        self.create_manager()
Parameters
  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: '{' style format string for the logger object.

def create_manager(self):
49    def create_manager(self):
50        """Create dbManager.py in the same directory
51        as the database file if it doesn't exist."""
52        manager_template = Path(__file__).parent / "dbmanager.py"
53        manager_path = self.dbpath.parent / "dbmanager.py"
54        if not manager_path.exists():
55            manager_path.write_text(manager_template.read_text())

Create dbManager.py in the same directory as the database file if it doesn't exist.

def open(self):
57    def open(self):
58        """Open connection to db."""
59        self.connection = sqlite3.connect(
60            self.dbpath,
61            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
62            timeout=10,
63        )
64        self.connection.execute("pragma foreign_keys = 1")
65        self.cursor = self.connection.cursor()
66        self.connection_open = True

Open connection to db.

def close(self):
68    def close(self):
69        """Save and close connection to db.
70
71        Call this as soon as you are done using the database if you have
72        multiple threads or processes using the same database."""
73        if self.connection_open:
74            self.connection.commit()
75            self.connection.close()
76            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def create_tables(self, table_statements: list[str] = []):
158    @_connect
159    def create_tables(self, table_statements: list[str] = []):
160        """Create tables if they don't exist.
161
162        :param table_statements: Each statement should be
163        in the form 'tableName(columnDefinitions)'"""
164        if len(table_statements) > 0:
165            table_names = self.get_table_names()
166            for table in table_statements:
167                if table.split("(")[0].strip() not in table_names:
168                    self.cursor.execute(f"create table {table}")
169                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_statements: Each statement should be in the form 'tableName(columnDefinitions)'
def create_table(self, table: str, column_defs: list[str]):
171    @_connect
172    def create_table(self, table: str, column_defs: list[str]):
173        """Create a table if it doesn't exist.
174
175        :param table: Name of the table to create.
176
177        :param column_defs: List of column definitions in
178        proper Sqlite3 sytax.
179        i.e. "columnName text unique" or "columnName int primary key" etc."""
180        if table not in self.get_table_names():
181            statement = f"create table {table}({', '.join(column_defs)})"
182            self.cursor.execute(statement)
183            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

Parameters
  • table: Name of the table to create.

  • column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.

def get_table_names(self) -> list[str]:
185    @_connect
186    def get_table_names(self) -> list[str]:
187        """Returns a list of table names from database."""
188        self.cursor.execute(
189            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
190        )
191        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from database.

def get_column_names(self, table: str) -> list[str]:
193    @_connect
194    def get_column_names(self, table: str) -> list[str]:
195        """Return a list of column names from a table."""
196        self.cursor.execute(f"select * from {table} where 1=0")
197        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True) -> int:
199    @_connect
200    def count(
201        self,
202        table: str,
203        match_criteria: list[tuple] | dict = None,
204        exact_match: bool = True,
205    ) -> int:
206        """Return number of items in table.
207
208        :param match_criteria: Can be a list of 2-tuples where each
209        tuple is (columnName, rowValue) or a dictionary where
210        keys are column names and values are row values.
211        If None, all rows from the table will be counted.
212
213        :param exact_match: If False, the row value for a give column
214        in match_criteria will be matched as a substring. Has no effect if
215        match_criteria is None.
216        """
217        statement = f"select count(_rowid_) from {table}"
218        try:
219            if match_criteria:
220                self.cursor.execute(
221                    f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
222                )
223            else:
224                self.cursor.execute(f"{statement}")
225            return self.cursor.fetchone()[0]
226        except:
227            return 0

Return number of items in table.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

  • exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
229    @_connect
230    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
231        """Add row of values to table.
232
233        :param table: The table to insert into.
234
235        :param values: A tuple of values to be inserted into the table.
236
237        :param columns: If None, values param is expected to supply
238        a value for every column in the table. If columns is
239        provided, it should contain the same number of elements as values."""
240        parameterizer = ", ".join("?" for _ in values)
241        logger_values = ", ".join(str(value) for value in values)
242        try:
243            if columns:
244                columns = ", ".join(column for column in columns)
245                self.cursor.execute(
246                    f"insert into {table} ({columns}) values({parameterizer})", values
247                )
248            else:
249                self.cursor.execute(
250                    f"insert into {table} values({parameterizer})", values
251                )
252            self.logger.info(f'Added "{logger_values}" to {table} table.')
253        except Exception as e:
254            if "constraint" not in str(e).lower():
255                self.logger.exception(
256                    f'Error adding "{logger_values}" to {table} table.'
257                )
258            else:
259                self.logger.debug(str(e))

Add row of values to table.

Parameters
  • table: The table to insert into.

  • values: A tuple of values to be inserted into the table.

  • columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True, sort_by_column: str = None, columns_to_return: list[str] = None, values_only: bool = False) -> list[dict] | list[tuple]:
261    @_connect
262    def get_rows(
263        self,
264        table: str,
265        match_criteria: list[tuple] | dict = None,
266        exact_match: bool = True,
267        sort_by_column: str = None,
268        columns_to_return: list[str] = None,
269        values_only: bool = False,
270    ) -> list[dict] | list[tuple]:
271        """Returns rows from table as a list of dictionaries
272        where the key-value pairs of the dictionaries are
273        column name: row value.
274
275        :param match_criteria: Can be a list of 2-tuples where each
276        tuple is (columnName, rowValue) or a dictionary where
277        keys are column names and values are row values.
278
279        :param exact_match: If False, the rowValue for a give column
280        will be matched as a substring.
281
282        :param sort_by_column: A column name to sort the results by.
283
284        :param columns_to_return: Optional list of column names.
285        If provided, the elements returned by get_rows() will
286        only contain the provided columns. Otherwise every column
287        in the row is returned.
288
289        :param values_only: Return the results as a list of tuples
290        instead of a list of dictionaries that have column names as keys.
291        The results will still be sorted according to sort_by_column if
292        one is provided.
293        """
294        if type(columns_to_return) is str:
295            columns_to_return = [columns_to_return]
296        statement = f"select * from {table}"
297        matches = []
298        if not match_criteria:
299            self.cursor.execute(statement)
300        else:
301            self.cursor.execute(
302                f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
303            )
304        matches = self.cursor.fetchall()
305        results = [self._get_dict(table, match, columns_to_return) for match in matches]
306        if sort_by_column:
307            results = sorted(results, key=lambda x: x[sort_by_column])
308        if values_only:
309            return [tuple(row.values()) for row in results]
310        else:
311            return results

Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

  • sort_by_column: A column name to sort the results by.

  • columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.

  • values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.

def find( self, table: str, query_string: str, columns: list[str] = None) -> tuple[dict]:
313    @_connect
314    def find(
315        self, table: str, query_string: str, columns: list[str] = None
316    ) -> tuple[dict]:
317        """Search for rows that contain query_string as a substring
318        of any column.
319
320        :param table: The table to search.
321
322        :param query_string: The substring to search for in all columns.
323
324        :param columns: A list of columns to search for query_string.
325        If None, all columns in the table will be searched.
326        """
327        if type(columns) is str:
328            columns = [columns]
329        results = []
330        if not columns:
331            columns = self.get_column_names(table)
332        for column in columns:
333            results.extend(
334                [
335                    row
336                    for row in self.get_rows(
337                        table, [(column, query_string)], exact_match=False
338                    )
339                    if row not in results
340                ]
341            )
342        return results

Search for rows that contain query_string as a substring of any column.

Parameters
  • table: The table to search.

  • query_string: The substring to search for in all columns.

  • columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
344    @_connect
345    def delete(
346        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
347    ) -> int:
348        """Delete records from table.
349
350        Returns number of deleted records.
351
352        :param match_criteria: Can be a list of 2-tuples where each
353        tuple is (columnName, rowValue) or a dictionary where
354        keys are column names and values are row values.
355
356        :param exact_match: If False, the rowValue for a give column
357        will be matched as a substring.
358        """
359        num_matches = self.count(table, match_criteria, exact_match)
360        conditions = self._get_conditions(match_criteria, exact_match)
361        try:
362            self.cursor.execute(f"delete from {table} where {conditions}")
363            self.logger.info(
364                f'Deleted {num_matches} from "{table}" where {conditions}".'
365            )
366            return num_matches
367        except Exception as e:
368            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
369            return 0

Delete records from table.

Returns number of deleted records.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict = None) -> bool:
371    @_connect
372    def update(
373        self,
374        table: str,
375        column_to_update: str,
376        new_value: Any,
377        match_criteria: list[tuple] | dict = None,
378    ) -> bool:
379        """Update row value for entry matched with match_criteria.
380
381        :param column_to_update: The column to be updated in the matched row.
382
383        :param new_value: The new value to insert.
384
385        :param match_criteria: Can be a list of 2-tuples where each
386        tuple is (columnName, rowValue) or a dictionary where
387        keys are column names and values are row values.
388        If None, every row will be updated.
389
390        Returns True if successful, False if not."""
391        statement = f"update {table} set {column_to_update} = ?"
392        if match_criteria:
393            if self.count(table, match_criteria) == 0:
394                self.logger.info(
395                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
396                )
397                return False
398            conditions = self._get_conditions(match_criteria)
399            statement += f" where {conditions}"
400        else:
401            conditions = None
402        try:
403            self.cursor.execute(
404                statement,
405                (new_value,),
406            )
407            self.logger.info(
408                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
409            )
410            return True
411        except UnboundLocalError:
412            table_filter_string = "\n".join(
413                table_filter for table_filter in match_criteria
414            )
415            self.logger.error(
416                f"No records found matching filters: {table_filter_string}"
417            )
418            return False
419        except Exception as e:
420            self.logger.error(
421                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
422            )
423            return False

Update row value for entry matched with match_criteria.

Parameters
  • column_to_update: The column to be updated in the matched row.

  • new_value: The new value to insert.

  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.

Returns True if successful, False if not.

def drop_table(self, table: str) -> bool:
425    @_connect
426    def drop_table(self, table: str) -> bool:
427        """Drop a table from the database.
428
429        Returns True if successful, False if not."""
430        try:
431            self.cursor.execute(f"drop Table {table}")
432            self.logger.info(f'Dropped table "{table}"')
433        except Exception as e:
434            print(e)
435            self.logger.error(f'Failed to drop table "{table}"')

Drop a table from the database.

Returns True if successful, False if not.

def add_column(self, table: str, column: str, _type: str, default_value: str = None):
437    @_connect
438    def add_column(
439        self, table: str, column: str, _type: str, default_value: str = None
440    ):
441        """Add a new column to table.
442
443        :param column: Name of the column to add.
444
445        :param _type: The data type of the new column.
446
447        :param default_value: Optional default value for the column."""
448        try:
449            if default_value:
450                self.cursor.execute(
451                    f"alter table {table} add column {column} {_type} default {default_value}"
452                )
453            else:
454                self.cursor.execute(f"alter table {table} add column {column} {_type}")
455            self.logger.info(f'Added column "{column}" to "{table}" table.')
456        except Exception as e:
457            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

Parameters
  • column: Name of the column to add.

  • _type: The data type of the new column.

  • default_value: Optional default value for the column.

def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
460def data_to_string(
461    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
462) -> str:
463    """Uses tabulate to produce pretty string output
464    from a list of dictionaries.
465
466    :param data: Assumes all dictionaries in list have the same set of keys.
467
468    :param sort_key: Optional dictionary key to sort data with.
469
470    :param wrap_to_terminal: If True, the table width will be wrapped
471    to fit within the current terminal window. Set to False
472    if the output is going into something like a txt file."""
473    if len(data) == 0:
474        return ""
475    if sort_key:
476        data = sorted(data, key=lambda d: d[sort_key])
477    for i, d in enumerate(data):
478        for k in d:
479            data[i][k] = str(data[i][k])
480    if wrap_to_terminal:
481        terminal_width = os.get_terminal_size().columns
482        max_col_widths = terminal_width
483        """ Reducing the column width by tabulating one row at a time
484        and then reducing further by tabulating the whole set proved to be 
485        faster than going straight to tabulating the whole set and reducing
486        the column width."""
487        too_wide = True
488        while too_wide and max_col_widths > 1:
489            for i, row in enumerate(data):
490                output = tabulate(
491                    [row],
492                    headers="keys",
493                    disable_numparse=True,
494                    tablefmt="grid",
495                    maxcolwidths=max_col_widths,
496                )
497                if output.index("\n") > terminal_width:
498                    max_col_widths -= 2
499                    too_wide = True
500                    break
501                too_wide = False
502    else:
503        max_col_widths = None
504    output = tabulate(
505        data,
506        headers="keys",
507        disable_numparse=True,
508        tablefmt="grid",
509        maxcolwidths=max_col_widths,
510    )
511    # trim max column width until the output string is less wide than the current terminal width.
512    if wrap_to_terminal:
513        while output.index("\n") > terminal_width and max_col_widths > 1:
514            max_col_widths -= 2
515            max_col_widths = max(1, max_col_widths)
516            output = tabulate(
517                data,
518                headers="keys",
519                disable_numparse=True,
520                tablefmt="grid",
521                maxcolwidths=max_col_widths,
522            )
523    return output

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.