databased.databased

  1import logging
  2import os
  3import sqlite3
  4from datetime import datetime
  5from functools import wraps
  6from pathlib import Path
  7from typing import Any
  8
  9from tabulate import tabulate
 10
 11
 12class DataBased:
 13    """Sqli wrapper so queries don't need to be written except table definitions.
 14
 15    Supports saving and reading dates as datetime objects.
 16
 17    Supports using a context manager."""
 18
 19    def __init__(
 20        self,
 21        dbpath: str | Path,
 22        logger_encoding: str = "utf-8",
 23        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 24    ):
 25        """
 26        :param dbpath: String or Path object to database file.
 27        If a relative path is given, it will be relative to the
 28        current working directory. The log file will be saved to the
 29        same directory.
 30
 31        :param logger_message_format: '{' style format string
 32        for the logger object."""
 33        self.dbpath = Path(dbpath)
 34        self.dbname = Path(dbpath).name
 35        self._logger_init(
 36            encoding=logger_encoding, message_format=logger_message_format
 37        )
 38        self.connection_open = False
 39        self.create_manager()
 40
 41    def __enter__(self):
 42        self.open()
 43        return self
 44
 45    def __exit__(self, exception_type, exception_value, exception_traceback):
 46        self.close()
 47
 48    def create_manager(self):
 49        """Create dbManager.py in the same directory
 50        as the database file if it doesn't exist."""
 51        manager_template = Path(__file__).parent / "dbmanager.py"
 52        manager_path = self.dbpath.parent / "dbmanager.py"
 53        if not manager_path.exists():
 54            manager_path.write_text(manager_template.read_text())
 55
 56    def open(self):
 57        """Open connection to db."""
 58        self.connection = sqlite3.connect(
 59            self.dbpath,
 60            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 61            timeout=10,
 62        )
 63        self.connection.execute("pragma foreign_keys = 1")
 64        self.cursor = self.connection.cursor()
 65        self.connection_open = True
 66
 67    def close(self):
 68        """Save and close connection to db.
 69
 70        Call this as soon as you are done using the database if you have
 71        multiple threads or processes using the same database."""
 72        if self.connection_open:
 73            self.connection.commit()
 74            self.connection.close()
 75            self.connection_open = False
 76
 77    def _connect(func):
 78        """Decorator to open db connection if it isn't already open."""
 79
 80        @wraps(func)
 81        def inner(*args, **kwargs):
 82            self = args[0]
 83            if not self.connection_open:
 84                self.open()
 85            results = func(*args, **kwargs)
 86            return results
 87
 88        return inner
 89
 90    def _logger_init(
 91        self,
 92        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 93        encoding: str = "utf-8",
 94    ):
 95        """:param message_format: '{' style format string"""
 96        self.logger = logging.getLogger(self.dbname)
 97        if not self.logger.hasHandlers():
 98            handler = logging.FileHandler(
 99                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
100            )
101            handler.setFormatter(
102                logging.Formatter(
103                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
104                )
105            )
106            self.logger.addHandler(handler)
107            self.logger.setLevel(logging.INFO)
108
109    def _get_dict(
110        self, table: str, values: list, columns_to_return: list[str] = None
111    ) -> dict:
112        """Converts the values of a row into a dictionary with column names as keys.
113
114        :param table: The table that values were pulled from.
115
116        :param values: List of values expected to be the same quantity
117        and in the same order as the column names of table.
118
119        :param columns_to_return: An optional list of column names.
120        If given, only these columns will be included in the returned dictionary.
121        Otherwise all columns and values are returned."""
122        return {
123            column: value
124            for column, value in zip(self.get_column_names(table), values)
125            if not columns_to_return or column in columns_to_return
126        }
127
128    def _get_conditions(
129        self, match_criteria: list[tuple] | dict, exact_match: bool = True
130    ) -> str:
131        """Builds and returns the conditional portion of a query.
132
133        :param match_criteria: Can be a list of 2-tuples where each
134        tuple is (columnName, rowValue) or a dictionary where
135        keys are column names and values are row values.
136
137        :param exact_match: If False, the rowValue for a give column
138        will be matched as a substring.
139
140        Usage e.g.:
141
142        self.cursor.execute(f'select * from {table} where {conditions}')"""
143        if type(match_criteria) == dict:
144            match_criteria = [(k, v) for k, v in match_criteria.items()]
145        if exact_match:
146            conditions = " and ".join(
147                f'"{column_row[0]}" = "{column_row[1]}"'
148                for column_row in match_criteria
149            )
150        else:
151            conditions = " and ".join(
152                f'"{column_row[0]}" like "%{column_row[1]}%"'
153                for column_row in match_criteria
154            )
155        return f"({conditions})"
156
157    @_connect
158    def create_tables(self, table_statements: list[str] = []):
159        """Create tables if they don't exist.
160
161        :param table_statements: Each statement should be
162        in the form 'tableName(columnDefinitions)'"""
163        if len(table_statements) > 0:
164            table_names = self.get_table_names()
165            for table in table_statements:
166                if table.split("(")[0].strip() not in table_names:
167                    self.cursor.execute(f"create table {table}")
168                    self.logger.info(f'{table.split("(")[0]} table created.')
169
170    @_connect
171    def create_table(self, table: str, column_defs: list[str]):
172        """Create a table if it doesn't exist.
173
174        :param table: Name of the table to create.
175
176        :param column_defs: List of column definitions in
177        proper Sqlite3 sytax.
178        i.e. "columnName text unique" or "columnName int primary key" etc."""
179        if table not in self.get_table_names():
180            statement = f"create table {table}({', '.join(column_defs)})"
181            self.cursor.execute(statement)
182            self.logger.info(f"'{table}' table created.")
183
184    @_connect
185    def get_table_names(self) -> list[str]:
186        """Returns a list of table names from database."""
187        self.cursor.execute(
188            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
189        )
190        return [result[0] for result in self.cursor.fetchall()]
191
192    @_connect
193    def get_column_names(self, table: str) -> list[str]:
194        """Return a list of column names from a table."""
195        self.cursor.execute(f"select * from {table} where 1=0")
196        return [description[0] for description in self.cursor.description]
197
198    @_connect
199    def count(
200        self,
201        table: str,
202        match_criteria: list[tuple] | dict = None,
203        exact_match: bool = True,
204    ) -> int:
205        """Return number of items in table.
206
207        :param match_criteria: Can be a list of 2-tuples where each
208        tuple is (columnName, rowValue) or a dictionary where
209        keys are column names and values are row values.
210        If None, all rows from the table will be counted.
211
212        :param exact_match: If False, the row value for a give column
213        in match_criteria will be matched as a substring. Has no effect if
214        match_criteria is None.
215        """
216        statement = f"select count(_rowid_) from {table}"
217        try:
218            if match_criteria:
219                self.cursor.execute(
220                    f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
221                )
222            else:
223                self.cursor.execute(f"{statement}")
224            return self.cursor.fetchone()[0]
225        except:
226            return 0
227
228    @_connect
229    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
230        """Add row of values to table.
231
232        :param table: The table to insert into.
233
234        :param values: A tuple of values to be inserted into the table.
235
236        :param columns: If None, values param is expected to supply
237        a value for every column in the table. If columns is
238        provided, it should contain the same number of elements as values."""
239        parameterizer = ", ".join("?" for _ in values)
240        logger_values = ", ".join(str(value) for value in values)
241        try:
242            if columns:
243                columns = ", ".join(column for column in columns)
244                self.cursor.execute(
245                    f"insert into {table} ({columns}) values({parameterizer})", values
246                )
247            else:
248                self.cursor.execute(
249                    f"insert into {table} values({parameterizer})", values
250                )
251            self.logger.info(f'Added "{logger_values}" to {table} table.')
252        except Exception as e:
253            if "constraint" not in str(e).lower():
254                self.logger.exception(
255                    f'Error adding "{logger_values}" to {table} table.'
256                )
257            else:
258                self.logger.debug(str(e))
259
260    @_connect
261    def get_rows(
262        self,
263        table: str,
264        match_criteria: list[tuple] | dict = None,
265        exact_match: bool = True,
266        sort_by_column: str = None,
267        columns_to_return: list[str] = None,
268        values_only: bool = False,
269    ) -> tuple[dict] | tuple[tuple]:
270        """Returns rows from table as a list of dictionaries
271        where the key-value pairs of the dictionaries are
272        column name: row value.
273
274        :param match_criteria: Can be a list of 2-tuples where each
275        tuple is (columnName, rowValue) or a dictionary where
276        keys are column names and values are row values.
277
278        :param exact_match: If False, the rowValue for a give column
279        will be matched as a substring.
280
281        :param sort_by_column: A column name to sort the results by.
282
283        :param columns_to_return: Optional list of column names.
284        If provided, the dictionaries returned by get_rows() will
285        only contain the provided columns. Otherwise every column
286        in the row is returned.
287
288        :param values_only: Return the results as a tuple of tuples
289        instead of a tuple of dictionaries that have column names as keys.
290        The results will still be sorted according to sort_by_column if
291        one is provided.
292        """
293        if type(columns_to_return) is str:
294            columns_to_return = [columns_to_return]
295        statement = f"select * from {table}"
296        matches = []
297        if not match_criteria:
298            self.cursor.execute(statement)
299        else:
300            self.cursor.execute(
301                f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
302            )
303        matches = self.cursor.fetchall()
304        results = tuple(
305            self._get_dict(table, match, columns_to_return) for match in matches
306        )
307        if sort_by_column:
308            results = tuple(sorted(results, key=lambda x: x[sort_by_column]))
309        if values_only:
310            return tuple(tuple(row.values()) for row in results)
311        else:
312            return results
313
314    @_connect
315    def find(
316        self, table: str, query_string: str, columns: list[str] = None
317    ) -> tuple[dict]:
318        """Search for rows that contain query_string as a substring
319        of any column.
320
321        :param table: The table to search.
322
323        :param query_string: The substring to search for in all columns.
324
325        :param columns: A list of columns to search for query_string.
326        If None, all columns in the table will be searched.
327        """
328        if type(columns) is str:
329            columns = [columns]
330        results = []
331        if not columns:
332            columns = self.get_column_names(table)
333        for column in columns:
334            results.extend(
335                [
336                    row
337                    for row in self.get_rows(
338                        table, [(column, query_string)], exact_match=False
339                    )
340                    if row not in results
341                ]
342            )
343        return tuple(results)
344
345    @_connect
346    def delete(
347        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
348    ) -> int:
349        """Delete records from table.
350
351        Returns number of deleted records.
352
353        :param match_criteria: Can be a list of 2-tuples where each
354        tuple is (columnName, rowValue) or a dictionary where
355        keys are column names and values are row values.
356
357        :param exact_match: If False, the rowValue for a give column
358        will be matched as a substring.
359        """
360        num_matches = self.count(table, match_criteria, exact_match)
361        conditions = self._get_conditions(match_criteria, exact_match)
362        try:
363            self.cursor.execute(f"delete from {table} where {conditions}")
364            self.logger.info(
365                f'Deleted {num_matches} from "{table}" where {conditions}".'
366            )
367            return num_matches
368        except Exception as e:
369            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
370            return 0
371
372    @_connect
373    def update(
374        self,
375        table: str,
376        column_to_update: str,
377        new_value: Any,
378        match_criteria: list[tuple] | dict = None,
379    ) -> bool:
380        """Update row value for entry matched with match_criteria.
381
382        :param column_to_update: The column to be updated in the matched row.
383
384        :param new_value: The new value to insert.
385
386        :param match_criteria: Can be a list of 2-tuples where each
387        tuple is (columnName, rowValue) or a dictionary where
388        keys are column names and values are row values.
389        If None, every row will be updated.
390
391        Returns True if successful, False if not."""
392        statement = f"update {table} set {column_to_update} = ?"
393        if match_criteria:
394            if self.count(table, match_criteria) == 0:
395                self.logger.info(
396                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
397                )
398                return False
399            conditions = self._get_conditions(match_criteria)
400            statement += f" where {conditions}"
401        else:
402            conditions = None
403        try:
404            self.cursor.execute(
405                statement,
406                (new_value,),
407            )
408            self.logger.info(
409                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
410            )
411            return True
412        except UnboundLocalError:
413            table_filter_string = "\n".join(
414                table_filter for table_filter in match_criteria
415            )
416            self.logger.error(
417                f"No records found matching filters: {table_filter_string}"
418            )
419            return False
420        except Exception as e:
421            self.logger.error(
422                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
423            )
424            return False
425
426    @_connect
427    def drop_table(self, table: str) -> bool:
428        """Drop a table from the database.
429
430        Returns True if successful, False if not."""
431        try:
432            self.cursor.execute(f"drop Table {table}")
433            self.logger.info(f'Dropped table "{table}"')
434        except Exception as e:
435            print(e)
436            self.logger.error(f'Failed to drop table "{table}"')
437
438    @_connect
439    def add_column(
440        self, table: str, column: str, _type: str, default_value: str = None
441    ):
442        """Add a new column to table.
443
444        :param column: Name of the column to add.
445
446        :param _type: The data type of the new column.
447
448        :param default_value: Optional default value for the column."""
449        try:
450            if default_value:
451                self.cursor.execute(
452                    f"alter table {table} add column {column} {_type} default {default_value}"
453                )
454            else:
455                self.cursor.execute(f"alter table {table} add column {column} {_type}")
456            self.logger.info(f'Added column "{column}" to "{table}" table.')
457        except Exception as e:
458            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
459
460
461def data_to_string(
462    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
463) -> str:
464    """Uses tabulate to produce pretty string output
465    from a list of dictionaries.
466
467    :param data: Assumes all dictionaries in list have the same set of keys.
468
469    :param sort_key: Optional dictionary key to sort data with.
470
471    :param wrap_to_terminal: If True, the table width will be wrapped
472    to fit within the current terminal window. Set to False
473    if the output is going into something like a txt file."""
474    if len(data) == 0:
475        return ""
476    if sort_key:
477        data = sorted(data, key=lambda d: d[sort_key])
478    for i, d in enumerate(data):
479        for k in d:
480            data[i][k] = str(data[i][k])
481    if wrap_to_terminal:
482        terminal_width = os.get_terminal_size().columns
483        max_col_widths = terminal_width
484        """ Reducing the column width by tabulating one row at a time
485        and then reducing further by tabulating the whole set proved to be 
486        faster than going straight to tabulating the whole set and reducing
487        the column width."""
488        too_wide = True
489        while too_wide and max_col_widths > 1:
490            for i, row in enumerate(data):
491                output = tabulate(
492                    [row],
493                    headers="keys",
494                    disable_numparse=True,
495                    tablefmt="grid",
496                    maxcolwidths=max_col_widths,
497                )
498                if output.index("\n") > terminal_width:
499                    max_col_widths -= 2
500                    too_wide = True
501                    break
502                too_wide = False
503    else:
504        max_col_widths = None
505    output = tabulate(
506        data,
507        headers="keys",
508        disable_numparse=True,
509        tablefmt="grid",
510        maxcolwidths=max_col_widths,
511    )
512    # trim max column width until the output string is less wide than the current terminal width.
513    if wrap_to_terminal:
514        while output.index("\n") > terminal_width and max_col_widths > 1:
515            max_col_widths -= 2
516            max_col_widths = max(1, max_col_widths)
517            output = tabulate(
518                data,
519                headers="keys",
520                disable_numparse=True,
521                tablefmt="grid",
522                maxcolwidths=max_col_widths,
523            )
524    return output
class DataBased:
 13class DataBased:
 14    """Sqli wrapper so queries don't need to be written except table definitions.
 15
 16    Supports saving and reading dates as datetime objects.
 17
 18    Supports using a context manager."""
 19
 20    def __init__(
 21        self,
 22        dbpath: str | Path,
 23        logger_encoding: str = "utf-8",
 24        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
 25    ):
 26        """
 27        :param dbpath: String or Path object to database file.
 28        If a relative path is given, it will be relative to the
 29        current working directory. The log file will be saved to the
 30        same directory.
 31
 32        :param logger_message_format: '{' style format string
 33        for the logger object."""
 34        self.dbpath = Path(dbpath)
 35        self.dbname = Path(dbpath).name
 36        self._logger_init(
 37            encoding=logger_encoding, message_format=logger_message_format
 38        )
 39        self.connection_open = False
 40        self.create_manager()
 41
 42    def __enter__(self):
 43        self.open()
 44        return self
 45
 46    def __exit__(self, exception_type, exception_value, exception_traceback):
 47        self.close()
 48
 49    def create_manager(self):
 50        """Create dbManager.py in the same directory
 51        as the database file if it doesn't exist."""
 52        manager_template = Path(__file__).parent / "dbmanager.py"
 53        manager_path = self.dbpath.parent / "dbmanager.py"
 54        if not manager_path.exists():
 55            manager_path.write_text(manager_template.read_text())
 56
 57    def open(self):
 58        """Open connection to db."""
 59        self.connection = sqlite3.connect(
 60            self.dbpath,
 61            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
 62            timeout=10,
 63        )
 64        self.connection.execute("pragma foreign_keys = 1")
 65        self.cursor = self.connection.cursor()
 66        self.connection_open = True
 67
 68    def close(self):
 69        """Save and close connection to db.
 70
 71        Call this as soon as you are done using the database if you have
 72        multiple threads or processes using the same database."""
 73        if self.connection_open:
 74            self.connection.commit()
 75            self.connection.close()
 76            self.connection_open = False
 77
 78    def _connect(func):
 79        """Decorator to open db connection if it isn't already open."""
 80
 81        @wraps(func)
 82        def inner(*args, **kwargs):
 83            self = args[0]
 84            if not self.connection_open:
 85                self.open()
 86            results = func(*args, **kwargs)
 87            return results
 88
 89        return inner
 90
 91    def _logger_init(
 92        self,
 93        message_format: str = "{levelname}|-|{asctime}|-|{message}",
 94        encoding: str = "utf-8",
 95    ):
 96        """:param message_format: '{' style format string"""
 97        self.logger = logging.getLogger(self.dbname)
 98        if not self.logger.hasHandlers():
 99            handler = logging.FileHandler(
100                str(self.dbpath).replace(".", "") + ".log", encoding=encoding
101            )
102            handler.setFormatter(
103                logging.Formatter(
104                    message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p"
105                )
106            )
107            self.logger.addHandler(handler)
108            self.logger.setLevel(logging.INFO)
109
110    def _get_dict(
111        self, table: str, values: list, columns_to_return: list[str] = None
112    ) -> dict:
113        """Converts the values of a row into a dictionary with column names as keys.
114
115        :param table: The table that values were pulled from.
116
117        :param values: List of values expected to be the same quantity
118        and in the same order as the column names of table.
119
120        :param columns_to_return: An optional list of column names.
121        If given, only these columns will be included in the returned dictionary.
122        Otherwise all columns and values are returned."""
123        return {
124            column: value
125            for column, value in zip(self.get_column_names(table), values)
126            if not columns_to_return or column in columns_to_return
127        }
128
129    def _get_conditions(
130        self, match_criteria: list[tuple] | dict, exact_match: bool = True
131    ) -> str:
132        """Builds and returns the conditional portion of a query.
133
134        :param match_criteria: Can be a list of 2-tuples where each
135        tuple is (columnName, rowValue) or a dictionary where
136        keys are column names and values are row values.
137
138        :param exact_match: If False, the rowValue for a give column
139        will be matched as a substring.
140
141        Usage e.g.:
142
143        self.cursor.execute(f'select * from {table} where {conditions}')"""
144        if type(match_criteria) == dict:
145            match_criteria = [(k, v) for k, v in match_criteria.items()]
146        if exact_match:
147            conditions = " and ".join(
148                f'"{column_row[0]}" = "{column_row[1]}"'
149                for column_row in match_criteria
150            )
151        else:
152            conditions = " and ".join(
153                f'"{column_row[0]}" like "%{column_row[1]}%"'
154                for column_row in match_criteria
155            )
156        return f"({conditions})"
157
158    @_connect
159    def create_tables(self, table_statements: list[str] = []):
160        """Create tables if they don't exist.
161
162        :param table_statements: Each statement should be
163        in the form 'tableName(columnDefinitions)'"""
164        if len(table_statements) > 0:
165            table_names = self.get_table_names()
166            for table in table_statements:
167                if table.split("(")[0].strip() not in table_names:
168                    self.cursor.execute(f"create table {table}")
169                    self.logger.info(f'{table.split("(")[0]} table created.')
170
171    @_connect
172    def create_table(self, table: str, column_defs: list[str]):
173        """Create a table if it doesn't exist.
174
175        :param table: Name of the table to create.
176
177        :param column_defs: List of column definitions in
178        proper Sqlite3 sytax.
179        i.e. "columnName text unique" or "columnName int primary key" etc."""
180        if table not in self.get_table_names():
181            statement = f"create table {table}({', '.join(column_defs)})"
182            self.cursor.execute(statement)
183            self.logger.info(f"'{table}' table created.")
184
185    @_connect
186    def get_table_names(self) -> list[str]:
187        """Returns a list of table names from database."""
188        self.cursor.execute(
189            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
190        )
191        return [result[0] for result in self.cursor.fetchall()]
192
193    @_connect
194    def get_column_names(self, table: str) -> list[str]:
195        """Return a list of column names from a table."""
196        self.cursor.execute(f"select * from {table} where 1=0")
197        return [description[0] for description in self.cursor.description]
198
199    @_connect
200    def count(
201        self,
202        table: str,
203        match_criteria: list[tuple] | dict = None,
204        exact_match: bool = True,
205    ) -> int:
206        """Return number of items in table.
207
208        :param match_criteria: Can be a list of 2-tuples where each
209        tuple is (columnName, rowValue) or a dictionary where
210        keys are column names and values are row values.
211        If None, all rows from the table will be counted.
212
213        :param exact_match: If False, the row value for a give column
214        in match_criteria will be matched as a substring. Has no effect if
215        match_criteria is None.
216        """
217        statement = f"select count(_rowid_) from {table}"
218        try:
219            if match_criteria:
220                self.cursor.execute(
221                    f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
222                )
223            else:
224                self.cursor.execute(f"{statement}")
225            return self.cursor.fetchone()[0]
226        except:
227            return 0
228
229    @_connect
230    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
231        """Add row of values to table.
232
233        :param table: The table to insert into.
234
235        :param values: A tuple of values to be inserted into the table.
236
237        :param columns: If None, values param is expected to supply
238        a value for every column in the table. If columns is
239        provided, it should contain the same number of elements as values."""
240        parameterizer = ", ".join("?" for _ in values)
241        logger_values = ", ".join(str(value) for value in values)
242        try:
243            if columns:
244                columns = ", ".join(column for column in columns)
245                self.cursor.execute(
246                    f"insert into {table} ({columns}) values({parameterizer})", values
247                )
248            else:
249                self.cursor.execute(
250                    f"insert into {table} values({parameterizer})", values
251                )
252            self.logger.info(f'Added "{logger_values}" to {table} table.')
253        except Exception as e:
254            if "constraint" not in str(e).lower():
255                self.logger.exception(
256                    f'Error adding "{logger_values}" to {table} table.'
257                )
258            else:
259                self.logger.debug(str(e))
260
261    @_connect
262    def get_rows(
263        self,
264        table: str,
265        match_criteria: list[tuple] | dict = None,
266        exact_match: bool = True,
267        sort_by_column: str = None,
268        columns_to_return: list[str] = None,
269        values_only: bool = False,
270    ) -> tuple[dict] | tuple[tuple]:
271        """Returns rows from table as a list of dictionaries
272        where the key-value pairs of the dictionaries are
273        column name: row value.
274
275        :param match_criteria: Can be a list of 2-tuples where each
276        tuple is (columnName, rowValue) or a dictionary where
277        keys are column names and values are row values.
278
279        :param exact_match: If False, the rowValue for a give column
280        will be matched as a substring.
281
282        :param sort_by_column: A column name to sort the results by.
283
284        :param columns_to_return: Optional list of column names.
285        If provided, the dictionaries returned by get_rows() will
286        only contain the provided columns. Otherwise every column
287        in the row is returned.
288
289        :param values_only: Return the results as a tuple of tuples
290        instead of a tuple of dictionaries that have column names as keys.
291        The results will still be sorted according to sort_by_column if
292        one is provided.
293        """
294        if type(columns_to_return) is str:
295            columns_to_return = [columns_to_return]
296        statement = f"select * from {table}"
297        matches = []
298        if not match_criteria:
299            self.cursor.execute(statement)
300        else:
301            self.cursor.execute(
302                f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
303            )
304        matches = self.cursor.fetchall()
305        results = tuple(
306            self._get_dict(table, match, columns_to_return) for match in matches
307        )
308        if sort_by_column:
309            results = tuple(sorted(results, key=lambda x: x[sort_by_column]))
310        if values_only:
311            return tuple(tuple(row.values()) for row in results)
312        else:
313            return results
314
315    @_connect
316    def find(
317        self, table: str, query_string: str, columns: list[str] = None
318    ) -> tuple[dict]:
319        """Search for rows that contain query_string as a substring
320        of any column.
321
322        :param table: The table to search.
323
324        :param query_string: The substring to search for in all columns.
325
326        :param columns: A list of columns to search for query_string.
327        If None, all columns in the table will be searched.
328        """
329        if type(columns) is str:
330            columns = [columns]
331        results = []
332        if not columns:
333            columns = self.get_column_names(table)
334        for column in columns:
335            results.extend(
336                [
337                    row
338                    for row in self.get_rows(
339                        table, [(column, query_string)], exact_match=False
340                    )
341                    if row not in results
342                ]
343            )
344        return tuple(results)
345
346    @_connect
347    def delete(
348        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
349    ) -> int:
350        """Delete records from table.
351
352        Returns number of deleted records.
353
354        :param match_criteria: Can be a list of 2-tuples where each
355        tuple is (columnName, rowValue) or a dictionary where
356        keys are column names and values are row values.
357
358        :param exact_match: If False, the rowValue for a give column
359        will be matched as a substring.
360        """
361        num_matches = self.count(table, match_criteria, exact_match)
362        conditions = self._get_conditions(match_criteria, exact_match)
363        try:
364            self.cursor.execute(f"delete from {table} where {conditions}")
365            self.logger.info(
366                f'Deleted {num_matches} from "{table}" where {conditions}".'
367            )
368            return num_matches
369        except Exception as e:
370            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
371            return 0
372
373    @_connect
374    def update(
375        self,
376        table: str,
377        column_to_update: str,
378        new_value: Any,
379        match_criteria: list[tuple] | dict = None,
380    ) -> bool:
381        """Update row value for entry matched with match_criteria.
382
383        :param column_to_update: The column to be updated in the matched row.
384
385        :param new_value: The new value to insert.
386
387        :param match_criteria: Can be a list of 2-tuples where each
388        tuple is (columnName, rowValue) or a dictionary where
389        keys are column names and values are row values.
390        If None, every row will be updated.
391
392        Returns True if successful, False if not."""
393        statement = f"update {table} set {column_to_update} = ?"
394        if match_criteria:
395            if self.count(table, match_criteria) == 0:
396                self.logger.info(
397                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
398                )
399                return False
400            conditions = self._get_conditions(match_criteria)
401            statement += f" where {conditions}"
402        else:
403            conditions = None
404        try:
405            self.cursor.execute(
406                statement,
407                (new_value,),
408            )
409            self.logger.info(
410                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
411            )
412            return True
413        except UnboundLocalError:
414            table_filter_string = "\n".join(
415                table_filter for table_filter in match_criteria
416            )
417            self.logger.error(
418                f"No records found matching filters: {table_filter_string}"
419            )
420            return False
421        except Exception as e:
422            self.logger.error(
423                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
424            )
425            return False
426
427    @_connect
428    def drop_table(self, table: str) -> bool:
429        """Drop a table from the database.
430
431        Returns True if successful, False if not."""
432        try:
433            self.cursor.execute(f"drop Table {table}")
434            self.logger.info(f'Dropped table "{table}"')
435        except Exception as e:
436            print(e)
437            self.logger.error(f'Failed to drop table "{table}"')
438
439    @_connect
440    def add_column(
441        self, table: str, column: str, _type: str, default_value: str = None
442    ):
443        """Add a new column to table.
444
445        :param column: Name of the column to add.
446
447        :param _type: The data type of the new column.
448
449        :param default_value: Optional default value for the column."""
450        try:
451            if default_value:
452                self.cursor.execute(
453                    f"alter table {table} add column {column} {_type} default {default_value}"
454                )
455            else:
456                self.cursor.execute(f"alter table {table} add column {column} {_type}")
457            self.logger.info(f'Added column "{column}" to "{table}" table.')
458        except Exception as e:
459            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Sqli wrapper so queries don't need to be written except table definitions.

Supports saving and reading dates as datetime objects.

Supports using a context manager.

DataBased( dbpath: str | pathlib.Path, logger_encoding: str = 'utf-8', logger_message_format: str = '{levelname}|-|{asctime}|-|{message}')
20    def __init__(
21        self,
22        dbpath: str | Path,
23        logger_encoding: str = "utf-8",
24        logger_message_format: str = "{levelname}|-|{asctime}|-|{message}",
25    ):
26        """
27        :param dbpath: String or Path object to database file.
28        If a relative path is given, it will be relative to the
29        current working directory. The log file will be saved to the
30        same directory.
31
32        :param logger_message_format: '{' style format string
33        for the logger object."""
34        self.dbpath = Path(dbpath)
35        self.dbname = Path(dbpath).name
36        self._logger_init(
37            encoding=logger_encoding, message_format=logger_message_format
38        )
39        self.connection_open = False
40        self.create_manager()
Parameters
  • dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.

  • logger_message_format: '{' style format string for the logger object.

def create_manager(self):
49    def create_manager(self):
50        """Create dbManager.py in the same directory
51        as the database file if it doesn't exist."""
52        manager_template = Path(__file__).parent / "dbmanager.py"
53        manager_path = self.dbpath.parent / "dbmanager.py"
54        if not manager_path.exists():
55            manager_path.write_text(manager_template.read_text())

Create dbManager.py in the same directory as the database file if it doesn't exist.

def open(self):
57    def open(self):
58        """Open connection to db."""
59        self.connection = sqlite3.connect(
60            self.dbpath,
61            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
62            timeout=10,
63        )
64        self.connection.execute("pragma foreign_keys = 1")
65        self.cursor = self.connection.cursor()
66        self.connection_open = True

Open connection to db.

def close(self):
68    def close(self):
69        """Save and close connection to db.
70
71        Call this as soon as you are done using the database if you have
72        multiple threads or processes using the same database."""
73        if self.connection_open:
74            self.connection.commit()
75            self.connection.close()
76            self.connection_open = False

Save and close connection to db.

Call this as soon as you are done using the database if you have multiple threads or processes using the same database.

def create_tables(self, table_statements: list[str] = []):
158    @_connect
159    def create_tables(self, table_statements: list[str] = []):
160        """Create tables if they don't exist.
161
162        :param table_statements: Each statement should be
163        in the form 'tableName(columnDefinitions)'"""
164        if len(table_statements) > 0:
165            table_names = self.get_table_names()
166            for table in table_statements:
167                if table.split("(")[0].strip() not in table_names:
168                    self.cursor.execute(f"create table {table}")
169                    self.logger.info(f'{table.split("(")[0]} table created.')

Create tables if they don't exist.

Parameters
  • table_statements: Each statement should be in the form 'tableName(columnDefinitions)'
def create_table(self, table: str, column_defs: list[str]):
171    @_connect
172    def create_table(self, table: str, column_defs: list[str]):
173        """Create a table if it doesn't exist.
174
175        :param table: Name of the table to create.
176
177        :param column_defs: List of column definitions in
178        proper Sqlite3 sytax.
179        i.e. "columnName text unique" or "columnName int primary key" etc."""
180        if table not in self.get_table_names():
181            statement = f"create table {table}({', '.join(column_defs)})"
182            self.cursor.execute(statement)
183            self.logger.info(f"'{table}' table created.")

Create a table if it doesn't exist.

Parameters
  • table: Name of the table to create.

  • column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.

def get_table_names(self) -> list[str]:
185    @_connect
186    def get_table_names(self) -> list[str]:
187        """Returns a list of table names from database."""
188        self.cursor.execute(
189            'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"'
190        )
191        return [result[0] for result in self.cursor.fetchall()]

Returns a list of table names from database.

def get_column_names(self, table: str) -> list[str]:
193    @_connect
194    def get_column_names(self, table: str) -> list[str]:
195        """Return a list of column names from a table."""
196        self.cursor.execute(f"select * from {table} where 1=0")
197        return [description[0] for description in self.cursor.description]

Return a list of column names from a table.

def count( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True) -> int:
199    @_connect
200    def count(
201        self,
202        table: str,
203        match_criteria: list[tuple] | dict = None,
204        exact_match: bool = True,
205    ) -> int:
206        """Return number of items in table.
207
208        :param match_criteria: Can be a list of 2-tuples where each
209        tuple is (columnName, rowValue) or a dictionary where
210        keys are column names and values are row values.
211        If None, all rows from the table will be counted.
212
213        :param exact_match: If False, the row value for a give column
214        in match_criteria will be matched as a substring. Has no effect if
215        match_criteria is None.
216        """
217        statement = f"select count(_rowid_) from {table}"
218        try:
219            if match_criteria:
220                self.cursor.execute(
221                    f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
222                )
223            else:
224                self.cursor.execute(f"{statement}")
225            return self.cursor.fetchone()[0]
226        except:
227            return 0

Return number of items in table.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.

  • exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.

def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
229    @_connect
230    def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None):
231        """Add row of values to table.
232
233        :param table: The table to insert into.
234
235        :param values: A tuple of values to be inserted into the table.
236
237        :param columns: If None, values param is expected to supply
238        a value for every column in the table. If columns is
239        provided, it should contain the same number of elements as values."""
240        parameterizer = ", ".join("?" for _ in values)
241        logger_values = ", ".join(str(value) for value in values)
242        try:
243            if columns:
244                columns = ", ".join(column for column in columns)
245                self.cursor.execute(
246                    f"insert into {table} ({columns}) values({parameterizer})", values
247                )
248            else:
249                self.cursor.execute(
250                    f"insert into {table} values({parameterizer})", values
251                )
252            self.logger.info(f'Added "{logger_values}" to {table} table.')
253        except Exception as e:
254            if "constraint" not in str(e).lower():
255                self.logger.exception(
256                    f'Error adding "{logger_values}" to {table} table.'
257                )
258            else:
259                self.logger.debug(str(e))

Add row of values to table.

Parameters
  • table: The table to insert into.

  • values: A tuple of values to be inserted into the table.

  • columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.

def get_rows( self, table: str, match_criteria: list[tuple] | dict = None, exact_match: bool = True, sort_by_column: str = None, columns_to_return: list[str] = None, values_only: bool = False) -> tuple[dict] | tuple[tuple]:
261    @_connect
262    def get_rows(
263        self,
264        table: str,
265        match_criteria: list[tuple] | dict = None,
266        exact_match: bool = True,
267        sort_by_column: str = None,
268        columns_to_return: list[str] = None,
269        values_only: bool = False,
270    ) -> tuple[dict] | tuple[tuple]:
271        """Returns rows from table as a list of dictionaries
272        where the key-value pairs of the dictionaries are
273        column name: row value.
274
275        :param match_criteria: Can be a list of 2-tuples where each
276        tuple is (columnName, rowValue) or a dictionary where
277        keys are column names and values are row values.
278
279        :param exact_match: If False, the rowValue for a give column
280        will be matched as a substring.
281
282        :param sort_by_column: A column name to sort the results by.
283
284        :param columns_to_return: Optional list of column names.
285        If provided, the dictionaries returned by get_rows() will
286        only contain the provided columns. Otherwise every column
287        in the row is returned.
288
289        :param values_only: Return the results as a tuple of tuples
290        instead of a tuple of dictionaries that have column names as keys.
291        The results will still be sorted according to sort_by_column if
292        one is provided.
293        """
294        if type(columns_to_return) is str:
295            columns_to_return = [columns_to_return]
296        statement = f"select * from {table}"
297        matches = []
298        if not match_criteria:
299            self.cursor.execute(statement)
300        else:
301            self.cursor.execute(
302                f"{statement} where {self._get_conditions(match_criteria, exact_match)}"
303            )
304        matches = self.cursor.fetchall()
305        results = tuple(
306            self._get_dict(table, match, columns_to_return) for match in matches
307        )
308        if sort_by_column:
309            results = tuple(sorted(results, key=lambda x: x[sort_by_column]))
310        if values_only:
311            return tuple(tuple(row.values()) for row in results)
312        else:
313            return results

Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

  • sort_by_column: A column name to sort the results by.

  • columns_to_return: Optional list of column names. If provided, the dictionaries returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.

  • values_only: Return the results as a tuple of tuples instead of a tuple of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.

def find( self, table: str, query_string: str, columns: list[str] = None) -> tuple[dict]:
315    @_connect
316    def find(
317        self, table: str, query_string: str, columns: list[str] = None
318    ) -> tuple[dict]:
319        """Search for rows that contain query_string as a substring
320        of any column.
321
322        :param table: The table to search.
323
324        :param query_string: The substring to search for in all columns.
325
326        :param columns: A list of columns to search for query_string.
327        If None, all columns in the table will be searched.
328        """
329        if type(columns) is str:
330            columns = [columns]
331        results = []
332        if not columns:
333            columns = self.get_column_names(table)
334        for column in columns:
335            results.extend(
336                [
337                    row
338                    for row in self.get_rows(
339                        table, [(column, query_string)], exact_match=False
340                    )
341                    if row not in results
342                ]
343            )
344        return tuple(results)

Search for rows that contain query_string as a substring of any column.

Parameters
  • table: The table to search.

  • query_string: The substring to search for in all columns.

  • columns: A list of columns to search for query_string. If None, all columns in the table will be searched.

def delete( self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True) -> int:
346    @_connect
347    def delete(
348        self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True
349    ) -> int:
350        """Delete records from table.
351
352        Returns number of deleted records.
353
354        :param match_criteria: Can be a list of 2-tuples where each
355        tuple is (columnName, rowValue) or a dictionary where
356        keys are column names and values are row values.
357
358        :param exact_match: If False, the rowValue for a give column
359        will be matched as a substring.
360        """
361        num_matches = self.count(table, match_criteria, exact_match)
362        conditions = self._get_conditions(match_criteria, exact_match)
363        try:
364            self.cursor.execute(f"delete from {table} where {conditions}")
365            self.logger.info(
366                f'Deleted {num_matches} from "{table}" where {conditions}".'
367            )
368            return num_matches
369        except Exception as e:
370            self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}')
371            return 0

Delete records from table.

Returns number of deleted records.

Parameters
  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.

  • exact_match: If False, the rowValue for a give column will be matched as a substring.

def update( self, table: str, column_to_update: str, new_value: Any, match_criteria: list[tuple] | dict = None) -> bool:
373    @_connect
374    def update(
375        self,
376        table: str,
377        column_to_update: str,
378        new_value: Any,
379        match_criteria: list[tuple] | dict = None,
380    ) -> bool:
381        """Update row value for entry matched with match_criteria.
382
383        :param column_to_update: The column to be updated in the matched row.
384
385        :param new_value: The new value to insert.
386
387        :param match_criteria: Can be a list of 2-tuples where each
388        tuple is (columnName, rowValue) or a dictionary where
389        keys are column names and values are row values.
390        If None, every row will be updated.
391
392        Returns True if successful, False if not."""
393        statement = f"update {table} set {column_to_update} = ?"
394        if match_criteria:
395            if self.count(table, match_criteria) == 0:
396                self.logger.info(
397                    f"Couldn't find matching records in {table} table to update to '{new_value}'"
398                )
399                return False
400            conditions = self._get_conditions(match_criteria)
401            statement += f" where {conditions}"
402        else:
403            conditions = None
404        try:
405            self.cursor.execute(
406                statement,
407                (new_value,),
408            )
409            self.logger.info(
410                f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}'
411            )
412            return True
413        except UnboundLocalError:
414            table_filter_string = "\n".join(
415                table_filter for table_filter in match_criteria
416            )
417            self.logger.error(
418                f"No records found matching filters: {table_filter_string}"
419            )
420            return False
421        except Exception as e:
422            self.logger.error(
423                f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}'
424            )
425            return False

Update row value for entry matched with match_criteria.

Parameters
  • column_to_update: The column to be updated in the matched row.

  • new_value: The new value to insert.

  • match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.

Returns True if successful, False if not.

def drop_table(self, table: str) -> bool:
427    @_connect
428    def drop_table(self, table: str) -> bool:
429        """Drop a table from the database.
430
431        Returns True if successful, False if not."""
432        try:
433            self.cursor.execute(f"drop Table {table}")
434            self.logger.info(f'Dropped table "{table}"')
435        except Exception as e:
436            print(e)
437            self.logger.error(f'Failed to drop table "{table}"')

Drop a table from the database.

Returns True if successful, False if not.

def add_column(self, table: str, column: str, _type: str, default_value: str = None):
439    @_connect
440    def add_column(
441        self, table: str, column: str, _type: str, default_value: str = None
442    ):
443        """Add a new column to table.
444
445        :param column: Name of the column to add.
446
447        :param _type: The data type of the new column.
448
449        :param default_value: Optional default value for the column."""
450        try:
451            if default_value:
452                self.cursor.execute(
453                    f"alter table {table} add column {column} {_type} default {default_value}"
454                )
455            else:
456                self.cursor.execute(f"alter table {table} add column {column} {_type}")
457            self.logger.info(f'Added column "{column}" to "{table}" table.')
458        except Exception as e:
459            self.logger.error(f'Failed to add column "{column}" to "{table}" table.')

Add a new column to table.

Parameters
  • column: Name of the column to add.

  • _type: The data type of the new column.

  • default_value: Optional default value for the column.

def data_to_string( data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True) -> str:
462def data_to_string(
463    data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True
464) -> str:
465    """Uses tabulate to produce pretty string output
466    from a list of dictionaries.
467
468    :param data: Assumes all dictionaries in list have the same set of keys.
469
470    :param sort_key: Optional dictionary key to sort data with.
471
472    :param wrap_to_terminal: If True, the table width will be wrapped
473    to fit within the current terminal window. Set to False
474    if the output is going into something like a txt file."""
475    if len(data) == 0:
476        return ""
477    if sort_key:
478        data = sorted(data, key=lambda d: d[sort_key])
479    for i, d in enumerate(data):
480        for k in d:
481            data[i][k] = str(data[i][k])
482    if wrap_to_terminal:
483        terminal_width = os.get_terminal_size().columns
484        max_col_widths = terminal_width
485        """ Reducing the column width by tabulating one row at a time
486        and then reducing further by tabulating the whole set proved to be 
487        faster than going straight to tabulating the whole set and reducing
488        the column width."""
489        too_wide = True
490        while too_wide and max_col_widths > 1:
491            for i, row in enumerate(data):
492                output = tabulate(
493                    [row],
494                    headers="keys",
495                    disable_numparse=True,
496                    tablefmt="grid",
497                    maxcolwidths=max_col_widths,
498                )
499                if output.index("\n") > terminal_width:
500                    max_col_widths -= 2
501                    too_wide = True
502                    break
503                too_wide = False
504    else:
505        max_col_widths = None
506    output = tabulate(
507        data,
508        headers="keys",
509        disable_numparse=True,
510        tablefmt="grid",
511        maxcolwidths=max_col_widths,
512    )
513    # trim max column width until the output string is less wide than the current terminal width.
514    if wrap_to_terminal:
515        while output.index("\n") > terminal_width and max_col_widths > 1:
516            max_col_widths -= 2
517            max_col_widths = max(1, max_col_widths)
518            output = tabulate(
519                data,
520                headers="keys",
521                disable_numparse=True,
522                tablefmt="grid",
523                maxcolwidths=max_col_widths,
524            )
525    return output

Uses tabulate to produce pretty string output from a list of dictionaries.

Parameters
  • data: Assumes all dictionaries in list have the same set of keys.

  • sort_key: Optional dictionary key to sort data with.

  • wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.