databased.databased
1import logging 2import os 3import sqlite3 4from datetime import datetime 5from functools import wraps 6from pathlib import Path 7from typing import Any 8 9from tabulate import tabulate 10 11 12class DataBased: 13 """Sqli wrapper so queries don't need to be written except table definitions. 14 15 Supports saving and reading dates as datetime objects. 16 17 Supports using a context manager.""" 18 19 def __init__( 20 self, 21 dbpath: str | Path, 22 logger_encoding: str = "utf-8", 23 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 24 ): 25 """ 26 :param dbpath: String or Path object to database file. 27 If a relative path is given, it will be relative to the 28 current working directory. The log file will be saved to the 29 same directory. 30 31 :param logger_message_format: '{' style format string 32 for the logger object.""" 33 self.dbpath = Path(dbpath) 34 self.dbname = Path(dbpath).name 35 self._logger_init( 36 encoding=logger_encoding, message_format=logger_message_format 37 ) 38 self.connection_open = False 39 self.create_manager() 40 41 def __enter__(self): 42 self.open() 43 return self 44 45 def __exit__(self, exception_type, exception_value, exception_traceback): 46 self.close() 47 48 def create_manager(self): 49 """Create dbManager.py in the same directory 50 as the database file if it doesn't exist.""" 51 manager_template = Path(__file__).parent / "dbmanager.py" 52 manager_path = self.dbpath.parent / "dbmanager.py" 53 if not manager_path.exists(): 54 manager_path.write_text(manager_template.read_text()) 55 56 def open(self): 57 """Open connection to db.""" 58 self.connection = sqlite3.connect( 59 self.dbpath, 60 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 61 timeout=10, 62 ) 63 self.connection.execute("pragma foreign_keys = 1") 64 self.cursor = self.connection.cursor() 65 self.connection_open = True 66 67 def close(self): 68 """Save and close connection to db. 69 70 Call this as soon as you are done using the database if you have 71 multiple threads or processes using the same database.""" 72 if self.connection_open: 73 self.connection.commit() 74 self.connection.close() 75 self.connection_open = False 76 77 def _connect(func): 78 """Decorator to open db connection if it isn't already open.""" 79 80 @wraps(func) 81 def inner(*args, **kwargs): 82 self = args[0] 83 if not self.connection_open: 84 self.open() 85 results = func(*args, **kwargs) 86 return results 87 88 return inner 89 90 def _logger_init( 91 self, 92 message_format: str = "{levelname}|-|{asctime}|-|{message}", 93 encoding: str = "utf-8", 94 ): 95 """:param message_format: '{' style format string""" 96 self.logger = logging.getLogger(self.dbname) 97 if not self.logger.hasHandlers(): 98 handler = logging.FileHandler( 99 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 100 ) 101 handler.setFormatter( 102 logging.Formatter( 103 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 104 ) 105 ) 106 self.logger.addHandler(handler) 107 self.logger.setLevel(logging.INFO) 108 109 def _get_dict( 110 self, table: str, values: list, columns_to_return: list[str] = None 111 ) -> dict: 112 """Converts the values of a row into a dictionary with column names as keys. 113 114 :param table: The table that values were pulled from. 115 116 :param values: List of values expected to be the same quantity 117 and in the same order as the column names of table. 118 119 :param columns_to_return: An optional list of column names. 120 If given, only these columns will be included in the returned dictionary. 121 Otherwise all columns and values are returned.""" 122 return { 123 column: value 124 for column, value in zip(self.get_column_names(table), values) 125 if not columns_to_return or column in columns_to_return 126 } 127 128 def _get_conditions( 129 self, match_criteria: list[tuple] | dict, exact_match: bool = True 130 ) -> str: 131 """Builds and returns the conditional portion of a query. 132 133 :param match_criteria: Can be a list of 2-tuples where each 134 tuple is (columnName, rowValue) or a dictionary where 135 keys are column names and values are row values. 136 137 :param exact_match: If False, the rowValue for a give column 138 will be matched as a substring. 139 140 Usage e.g.: 141 142 self.cursor.execute(f'select * from {table} where {conditions}')""" 143 if type(match_criteria) == dict: 144 match_criteria = [(k, v) for k, v in match_criteria.items()] 145 if exact_match: 146 conditions = " and ".join( 147 f'"{column_row[0]}" = "{column_row[1]}"' 148 for column_row in match_criteria 149 ) 150 else: 151 conditions = " and ".join( 152 f'"{column_row[0]}" like "%{column_row[1]}%"' 153 for column_row in match_criteria 154 ) 155 return f"({conditions})" 156 157 @_connect 158 def create_tables(self, table_statements: list[str] = []): 159 """Create tables if they don't exist. 160 161 :param table_statements: Each statement should be 162 in the form 'tableName(columnDefinitions)'""" 163 if len(table_statements) > 0: 164 table_names = self.get_table_names() 165 for table in table_statements: 166 if table.split("(")[0].strip() not in table_names: 167 self.cursor.execute(f"create table {table}") 168 self.logger.info(f'{table.split("(")[0]} table created.') 169 170 @_connect 171 def create_table(self, table: str, column_defs: list[str]): 172 """Create a table if it doesn't exist. 173 174 :param table: Name of the table to create. 175 176 :param column_defs: List of column definitions in 177 proper Sqlite3 sytax. 178 i.e. "columnName text unique" or "columnName int primary key" etc.""" 179 if table not in self.get_table_names(): 180 statement = f"create table {table}({', '.join(column_defs)})" 181 self.cursor.execute(statement) 182 self.logger.info(f"'{table}' table created.") 183 184 @_connect 185 def get_table_names(self) -> list[str]: 186 """Returns a list of table names from database.""" 187 self.cursor.execute( 188 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 189 ) 190 return [result[0] for result in self.cursor.fetchall()] 191 192 @_connect 193 def get_column_names(self, table: str) -> list[str]: 194 """Return a list of column names from a table.""" 195 self.cursor.execute(f"select * from {table} where 1=0") 196 return [description[0] for description in self.cursor.description] 197 198 @_connect 199 def count( 200 self, 201 table: str, 202 match_criteria: list[tuple] | dict = None, 203 exact_match: bool = True, 204 ) -> int: 205 """Return number of items in table. 206 207 :param match_criteria: Can be a list of 2-tuples where each 208 tuple is (columnName, rowValue) or a dictionary where 209 keys are column names and values are row values. 210 If None, all rows from the table will be counted. 211 212 :param exact_match: If False, the row value for a give column 213 in match_criteria will be matched as a substring. Has no effect if 214 match_criteria is None. 215 """ 216 statement = f"select count(_rowid_) from {table}" 217 try: 218 if match_criteria: 219 self.cursor.execute( 220 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 221 ) 222 else: 223 self.cursor.execute(f"{statement}") 224 return self.cursor.fetchone()[0] 225 except: 226 return 0 227 228 @_connect 229 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 230 """Add row of values to table. 231 232 :param table: The table to insert into. 233 234 :param values: A tuple of values to be inserted into the table. 235 236 :param columns: If None, values param is expected to supply 237 a value for every column in the table. If columns is 238 provided, it should contain the same number of elements as values.""" 239 parameterizer = ", ".join("?" for _ in values) 240 logger_values = ", ".join(str(value) for value in values) 241 try: 242 if columns: 243 columns = ", ".join(column for column in columns) 244 self.cursor.execute( 245 f"insert into {table} ({columns}) values({parameterizer})", values 246 ) 247 else: 248 self.cursor.execute( 249 f"insert into {table} values({parameterizer})", values 250 ) 251 self.logger.info(f'Added "{logger_values}" to {table} table.') 252 except Exception as e: 253 if "constraint" not in str(e).lower(): 254 self.logger.exception( 255 f'Error adding "{logger_values}" to {table} table.' 256 ) 257 else: 258 self.logger.debug(str(e)) 259 260 @_connect 261 def get_rows( 262 self, 263 table: str, 264 match_criteria: list[tuple] | dict = None, 265 exact_match: bool = True, 266 sort_by_column: str = None, 267 columns_to_return: list[str] = None, 268 values_only: bool = False, 269 ) -> tuple[dict] | tuple[tuple]: 270 """Returns rows from table as a list of dictionaries 271 where the key-value pairs of the dictionaries are 272 column name: row value. 273 274 :param match_criteria: Can be a list of 2-tuples where each 275 tuple is (columnName, rowValue) or a dictionary where 276 keys are column names and values are row values. 277 278 :param exact_match: If False, the rowValue for a give column 279 will be matched as a substring. 280 281 :param sort_by_column: A column name to sort the results by. 282 283 :param columns_to_return: Optional list of column names. 284 If provided, the dictionaries returned by get_rows() will 285 only contain the provided columns. Otherwise every column 286 in the row is returned. 287 288 :param values_only: Return the results as a tuple of tuples 289 instead of a tuple of dictionaries that have column names as keys. 290 The results will still be sorted according to sort_by_column if 291 one is provided. 292 """ 293 if type(columns_to_return) is str: 294 columns_to_return = [columns_to_return] 295 statement = f"select * from {table}" 296 matches = [] 297 if not match_criteria: 298 self.cursor.execute(statement) 299 else: 300 self.cursor.execute( 301 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 302 ) 303 matches = self.cursor.fetchall() 304 results = tuple( 305 self._get_dict(table, match, columns_to_return) for match in matches 306 ) 307 if sort_by_column: 308 results = tuple(sorted(results, key=lambda x: x[sort_by_column])) 309 if values_only: 310 return tuple(tuple(row.values()) for row in results) 311 else: 312 return results 313 314 @_connect 315 def find( 316 self, table: str, query_string: str, columns: list[str] = None 317 ) -> tuple[dict]: 318 """Search for rows that contain query_string as a substring 319 of any column. 320 321 :param table: The table to search. 322 323 :param query_string: The substring to search for in all columns. 324 325 :param columns: A list of columns to search for query_string. 326 If None, all columns in the table will be searched. 327 """ 328 if type(columns) is str: 329 columns = [columns] 330 results = [] 331 if not columns: 332 columns = self.get_column_names(table) 333 for column in columns: 334 results.extend( 335 [ 336 row 337 for row in self.get_rows( 338 table, [(column, query_string)], exact_match=False 339 ) 340 if row not in results 341 ] 342 ) 343 return tuple(results) 344 345 @_connect 346 def delete( 347 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 348 ) -> int: 349 """Delete records from table. 350 351 Returns number of deleted records. 352 353 :param match_criteria: Can be a list of 2-tuples where each 354 tuple is (columnName, rowValue) or a dictionary where 355 keys are column names and values are row values. 356 357 :param exact_match: If False, the rowValue for a give column 358 will be matched as a substring. 359 """ 360 num_matches = self.count(table, match_criteria, exact_match) 361 conditions = self._get_conditions(match_criteria, exact_match) 362 try: 363 self.cursor.execute(f"delete from {table} where {conditions}") 364 self.logger.info( 365 f'Deleted {num_matches} from "{table}" where {conditions}".' 366 ) 367 return num_matches 368 except Exception as e: 369 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 370 return 0 371 372 @_connect 373 def update( 374 self, 375 table: str, 376 column_to_update: str, 377 new_value: Any, 378 match_criteria: list[tuple] | dict = None, 379 ) -> bool: 380 """Update row value for entry matched with match_criteria. 381 382 :param column_to_update: The column to be updated in the matched row. 383 384 :param new_value: The new value to insert. 385 386 :param match_criteria: Can be a list of 2-tuples where each 387 tuple is (columnName, rowValue) or a dictionary where 388 keys are column names and values are row values. 389 If None, every row will be updated. 390 391 Returns True if successful, False if not.""" 392 statement = f"update {table} set {column_to_update} = ?" 393 if match_criteria: 394 if self.count(table, match_criteria) == 0: 395 self.logger.info( 396 f"Couldn't find matching records in {table} table to update to '{new_value}'" 397 ) 398 return False 399 conditions = self._get_conditions(match_criteria) 400 statement += f" where {conditions}" 401 else: 402 conditions = None 403 try: 404 self.cursor.execute( 405 statement, 406 (new_value,), 407 ) 408 self.logger.info( 409 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 410 ) 411 return True 412 except UnboundLocalError: 413 table_filter_string = "\n".join( 414 table_filter for table_filter in match_criteria 415 ) 416 self.logger.error( 417 f"No records found matching filters: {table_filter_string}" 418 ) 419 return False 420 except Exception as e: 421 self.logger.error( 422 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 423 ) 424 return False 425 426 @_connect 427 def drop_table(self, table: str) -> bool: 428 """Drop a table from the database. 429 430 Returns True if successful, False if not.""" 431 try: 432 self.cursor.execute(f"drop Table {table}") 433 self.logger.info(f'Dropped table "{table}"') 434 except Exception as e: 435 print(e) 436 self.logger.error(f'Failed to drop table "{table}"') 437 438 @_connect 439 def add_column( 440 self, table: str, column: str, _type: str, default_value: str = None 441 ): 442 """Add a new column to table. 443 444 :param column: Name of the column to add. 445 446 :param _type: The data type of the new column. 447 448 :param default_value: Optional default value for the column.""" 449 try: 450 if default_value: 451 self.cursor.execute( 452 f"alter table {table} add column {column} {_type} default {default_value}" 453 ) 454 else: 455 self.cursor.execute(f"alter table {table} add column {column} {_type}") 456 self.logger.info(f'Added column "{column}" to "{table}" table.') 457 except Exception as e: 458 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 459 460 461def data_to_string( 462 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 463) -> str: 464 """Uses tabulate to produce pretty string output 465 from a list of dictionaries. 466 467 :param data: Assumes all dictionaries in list have the same set of keys. 468 469 :param sort_key: Optional dictionary key to sort data with. 470 471 :param wrap_to_terminal: If True, the table width will be wrapped 472 to fit within the current terminal window. Set to False 473 if the output is going into something like a txt file.""" 474 if len(data) == 0: 475 return "" 476 if sort_key: 477 data = sorted(data, key=lambda d: d[sort_key]) 478 for i, d in enumerate(data): 479 for k in d: 480 data[i][k] = str(data[i][k]) 481 if wrap_to_terminal: 482 terminal_width = os.get_terminal_size().columns 483 max_col_widths = terminal_width 484 """ Reducing the column width by tabulating one row at a time 485 and then reducing further by tabulating the whole set proved to be 486 faster than going straight to tabulating the whole set and reducing 487 the column width.""" 488 too_wide = True 489 while too_wide and max_col_widths > 1: 490 for i, row in enumerate(data): 491 output = tabulate( 492 [row], 493 headers="keys", 494 disable_numparse=True, 495 tablefmt="grid", 496 maxcolwidths=max_col_widths, 497 ) 498 if output.index("\n") > terminal_width: 499 max_col_widths -= 2 500 too_wide = True 501 break 502 too_wide = False 503 else: 504 max_col_widths = None 505 output = tabulate( 506 data, 507 headers="keys", 508 disable_numparse=True, 509 tablefmt="grid", 510 maxcolwidths=max_col_widths, 511 ) 512 # trim max column width until the output string is less wide than the current terminal width. 513 if wrap_to_terminal: 514 while output.index("\n") > terminal_width and max_col_widths > 1: 515 max_col_widths -= 2 516 max_col_widths = max(1, max_col_widths) 517 output = tabulate( 518 data, 519 headers="keys", 520 disable_numparse=True, 521 tablefmt="grid", 522 maxcolwidths=max_col_widths, 523 ) 524 return output
13class DataBased: 14 """Sqli wrapper so queries don't need to be written except table definitions. 15 16 Supports saving and reading dates as datetime objects. 17 18 Supports using a context manager.""" 19 20 def __init__( 21 self, 22 dbpath: str | Path, 23 logger_encoding: str = "utf-8", 24 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 25 ): 26 """ 27 :param dbpath: String or Path object to database file. 28 If a relative path is given, it will be relative to the 29 current working directory. The log file will be saved to the 30 same directory. 31 32 :param logger_message_format: '{' style format string 33 for the logger object.""" 34 self.dbpath = Path(dbpath) 35 self.dbname = Path(dbpath).name 36 self._logger_init( 37 encoding=logger_encoding, message_format=logger_message_format 38 ) 39 self.connection_open = False 40 self.create_manager() 41 42 def __enter__(self): 43 self.open() 44 return self 45 46 def __exit__(self, exception_type, exception_value, exception_traceback): 47 self.close() 48 49 def create_manager(self): 50 """Create dbManager.py in the same directory 51 as the database file if it doesn't exist.""" 52 manager_template = Path(__file__).parent / "dbmanager.py" 53 manager_path = self.dbpath.parent / "dbmanager.py" 54 if not manager_path.exists(): 55 manager_path.write_text(manager_template.read_text()) 56 57 def open(self): 58 """Open connection to db.""" 59 self.connection = sqlite3.connect( 60 self.dbpath, 61 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 62 timeout=10, 63 ) 64 self.connection.execute("pragma foreign_keys = 1") 65 self.cursor = self.connection.cursor() 66 self.connection_open = True 67 68 def close(self): 69 """Save and close connection to db. 70 71 Call this as soon as you are done using the database if you have 72 multiple threads or processes using the same database.""" 73 if self.connection_open: 74 self.connection.commit() 75 self.connection.close() 76 self.connection_open = False 77 78 def _connect(func): 79 """Decorator to open db connection if it isn't already open.""" 80 81 @wraps(func) 82 def inner(*args, **kwargs): 83 self = args[0] 84 if not self.connection_open: 85 self.open() 86 results = func(*args, **kwargs) 87 return results 88 89 return inner 90 91 def _logger_init( 92 self, 93 message_format: str = "{levelname}|-|{asctime}|-|{message}", 94 encoding: str = "utf-8", 95 ): 96 """:param message_format: '{' style format string""" 97 self.logger = logging.getLogger(self.dbname) 98 if not self.logger.hasHandlers(): 99 handler = logging.FileHandler( 100 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 101 ) 102 handler.setFormatter( 103 logging.Formatter( 104 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 105 ) 106 ) 107 self.logger.addHandler(handler) 108 self.logger.setLevel(logging.INFO) 109 110 def _get_dict( 111 self, table: str, values: list, columns_to_return: list[str] = None 112 ) -> dict: 113 """Converts the values of a row into a dictionary with column names as keys. 114 115 :param table: The table that values were pulled from. 116 117 :param values: List of values expected to be the same quantity 118 and in the same order as the column names of table. 119 120 :param columns_to_return: An optional list of column names. 121 If given, only these columns will be included in the returned dictionary. 122 Otherwise all columns and values are returned.""" 123 return { 124 column: value 125 for column, value in zip(self.get_column_names(table), values) 126 if not columns_to_return or column in columns_to_return 127 } 128 129 def _get_conditions( 130 self, match_criteria: list[tuple] | dict, exact_match: bool = True 131 ) -> str: 132 """Builds and returns the conditional portion of a query. 133 134 :param match_criteria: Can be a list of 2-tuples where each 135 tuple is (columnName, rowValue) or a dictionary where 136 keys are column names and values are row values. 137 138 :param exact_match: If False, the rowValue for a give column 139 will be matched as a substring. 140 141 Usage e.g.: 142 143 self.cursor.execute(f'select * from {table} where {conditions}')""" 144 if type(match_criteria) == dict: 145 match_criteria = [(k, v) for k, v in match_criteria.items()] 146 if exact_match: 147 conditions = " and ".join( 148 f'"{column_row[0]}" = "{column_row[1]}"' 149 for column_row in match_criteria 150 ) 151 else: 152 conditions = " and ".join( 153 f'"{column_row[0]}" like "%{column_row[1]}%"' 154 for column_row in match_criteria 155 ) 156 return f"({conditions})" 157 158 @_connect 159 def create_tables(self, table_statements: list[str] = []): 160 """Create tables if they don't exist. 161 162 :param table_statements: Each statement should be 163 in the form 'tableName(columnDefinitions)'""" 164 if len(table_statements) > 0: 165 table_names = self.get_table_names() 166 for table in table_statements: 167 if table.split("(")[0].strip() not in table_names: 168 self.cursor.execute(f"create table {table}") 169 self.logger.info(f'{table.split("(")[0]} table created.') 170 171 @_connect 172 def create_table(self, table: str, column_defs: list[str]): 173 """Create a table if it doesn't exist. 174 175 :param table: Name of the table to create. 176 177 :param column_defs: List of column definitions in 178 proper Sqlite3 sytax. 179 i.e. "columnName text unique" or "columnName int primary key" etc.""" 180 if table not in self.get_table_names(): 181 statement = f"create table {table}({', '.join(column_defs)})" 182 self.cursor.execute(statement) 183 self.logger.info(f"'{table}' table created.") 184 185 @_connect 186 def get_table_names(self) -> list[str]: 187 """Returns a list of table names from database.""" 188 self.cursor.execute( 189 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 190 ) 191 return [result[0] for result in self.cursor.fetchall()] 192 193 @_connect 194 def get_column_names(self, table: str) -> list[str]: 195 """Return a list of column names from a table.""" 196 self.cursor.execute(f"select * from {table} where 1=0") 197 return [description[0] for description in self.cursor.description] 198 199 @_connect 200 def count( 201 self, 202 table: str, 203 match_criteria: list[tuple] | dict = None, 204 exact_match: bool = True, 205 ) -> int: 206 """Return number of items in table. 207 208 :param match_criteria: Can be a list of 2-tuples where each 209 tuple is (columnName, rowValue) or a dictionary where 210 keys are column names and values are row values. 211 If None, all rows from the table will be counted. 212 213 :param exact_match: If False, the row value for a give column 214 in match_criteria will be matched as a substring. Has no effect if 215 match_criteria is None. 216 """ 217 statement = f"select count(_rowid_) from {table}" 218 try: 219 if match_criteria: 220 self.cursor.execute( 221 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 222 ) 223 else: 224 self.cursor.execute(f"{statement}") 225 return self.cursor.fetchone()[0] 226 except: 227 return 0 228 229 @_connect 230 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 231 """Add row of values to table. 232 233 :param table: The table to insert into. 234 235 :param values: A tuple of values to be inserted into the table. 236 237 :param columns: If None, values param is expected to supply 238 a value for every column in the table. If columns is 239 provided, it should contain the same number of elements as values.""" 240 parameterizer = ", ".join("?" for _ in values) 241 logger_values = ", ".join(str(value) for value in values) 242 try: 243 if columns: 244 columns = ", ".join(column for column in columns) 245 self.cursor.execute( 246 f"insert into {table} ({columns}) values({parameterizer})", values 247 ) 248 else: 249 self.cursor.execute( 250 f"insert into {table} values({parameterizer})", values 251 ) 252 self.logger.info(f'Added "{logger_values}" to {table} table.') 253 except Exception as e: 254 if "constraint" not in str(e).lower(): 255 self.logger.exception( 256 f'Error adding "{logger_values}" to {table} table.' 257 ) 258 else: 259 self.logger.debug(str(e)) 260 261 @_connect 262 def get_rows( 263 self, 264 table: str, 265 match_criteria: list[tuple] | dict = None, 266 exact_match: bool = True, 267 sort_by_column: str = None, 268 columns_to_return: list[str] = None, 269 values_only: bool = False, 270 ) -> tuple[dict] | tuple[tuple]: 271 """Returns rows from table as a list of dictionaries 272 where the key-value pairs of the dictionaries are 273 column name: row value. 274 275 :param match_criteria: Can be a list of 2-tuples where each 276 tuple is (columnName, rowValue) or a dictionary where 277 keys are column names and values are row values. 278 279 :param exact_match: If False, the rowValue for a give column 280 will be matched as a substring. 281 282 :param sort_by_column: A column name to sort the results by. 283 284 :param columns_to_return: Optional list of column names. 285 If provided, the dictionaries returned by get_rows() will 286 only contain the provided columns. Otherwise every column 287 in the row is returned. 288 289 :param values_only: Return the results as a tuple of tuples 290 instead of a tuple of dictionaries that have column names as keys. 291 The results will still be sorted according to sort_by_column if 292 one is provided. 293 """ 294 if type(columns_to_return) is str: 295 columns_to_return = [columns_to_return] 296 statement = f"select * from {table}" 297 matches = [] 298 if not match_criteria: 299 self.cursor.execute(statement) 300 else: 301 self.cursor.execute( 302 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 303 ) 304 matches = self.cursor.fetchall() 305 results = tuple( 306 self._get_dict(table, match, columns_to_return) for match in matches 307 ) 308 if sort_by_column: 309 results = tuple(sorted(results, key=lambda x: x[sort_by_column])) 310 if values_only: 311 return tuple(tuple(row.values()) for row in results) 312 else: 313 return results 314 315 @_connect 316 def find( 317 self, table: str, query_string: str, columns: list[str] = None 318 ) -> tuple[dict]: 319 """Search for rows that contain query_string as a substring 320 of any column. 321 322 :param table: The table to search. 323 324 :param query_string: The substring to search for in all columns. 325 326 :param columns: A list of columns to search for query_string. 327 If None, all columns in the table will be searched. 328 """ 329 if type(columns) is str: 330 columns = [columns] 331 results = [] 332 if not columns: 333 columns = self.get_column_names(table) 334 for column in columns: 335 results.extend( 336 [ 337 row 338 for row in self.get_rows( 339 table, [(column, query_string)], exact_match=False 340 ) 341 if row not in results 342 ] 343 ) 344 return tuple(results) 345 346 @_connect 347 def delete( 348 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 349 ) -> int: 350 """Delete records from table. 351 352 Returns number of deleted records. 353 354 :param match_criteria: Can be a list of 2-tuples where each 355 tuple is (columnName, rowValue) or a dictionary where 356 keys are column names and values are row values. 357 358 :param exact_match: If False, the rowValue for a give column 359 will be matched as a substring. 360 """ 361 num_matches = self.count(table, match_criteria, exact_match) 362 conditions = self._get_conditions(match_criteria, exact_match) 363 try: 364 self.cursor.execute(f"delete from {table} where {conditions}") 365 self.logger.info( 366 f'Deleted {num_matches} from "{table}" where {conditions}".' 367 ) 368 return num_matches 369 except Exception as e: 370 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 371 return 0 372 373 @_connect 374 def update( 375 self, 376 table: str, 377 column_to_update: str, 378 new_value: Any, 379 match_criteria: list[tuple] | dict = None, 380 ) -> bool: 381 """Update row value for entry matched with match_criteria. 382 383 :param column_to_update: The column to be updated in the matched row. 384 385 :param new_value: The new value to insert. 386 387 :param match_criteria: Can be a list of 2-tuples where each 388 tuple is (columnName, rowValue) or a dictionary where 389 keys are column names and values are row values. 390 If None, every row will be updated. 391 392 Returns True if successful, False if not.""" 393 statement = f"update {table} set {column_to_update} = ?" 394 if match_criteria: 395 if self.count(table, match_criteria) == 0: 396 self.logger.info( 397 f"Couldn't find matching records in {table} table to update to '{new_value}'" 398 ) 399 return False 400 conditions = self._get_conditions(match_criteria) 401 statement += f" where {conditions}" 402 else: 403 conditions = None 404 try: 405 self.cursor.execute( 406 statement, 407 (new_value,), 408 ) 409 self.logger.info( 410 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 411 ) 412 return True 413 except UnboundLocalError: 414 table_filter_string = "\n".join( 415 table_filter for table_filter in match_criteria 416 ) 417 self.logger.error( 418 f"No records found matching filters: {table_filter_string}" 419 ) 420 return False 421 except Exception as e: 422 self.logger.error( 423 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 424 ) 425 return False 426 427 @_connect 428 def drop_table(self, table: str) -> bool: 429 """Drop a table from the database. 430 431 Returns True if successful, False if not.""" 432 try: 433 self.cursor.execute(f"drop Table {table}") 434 self.logger.info(f'Dropped table "{table}"') 435 except Exception as e: 436 print(e) 437 self.logger.error(f'Failed to drop table "{table}"') 438 439 @_connect 440 def add_column( 441 self, table: str, column: str, _type: str, default_value: str = None 442 ): 443 """Add a new column to table. 444 445 :param column: Name of the column to add. 446 447 :param _type: The data type of the new column. 448 449 :param default_value: Optional default value for the column.""" 450 try: 451 if default_value: 452 self.cursor.execute( 453 f"alter table {table} add column {column} {_type} default {default_value}" 454 ) 455 else: 456 self.cursor.execute(f"alter table {table} add column {column} {_type}") 457 self.logger.info(f'Added column "{column}" to "{table}" table.') 458 except Exception as e: 459 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Sqli wrapper so queries don't need to be written except table definitions.
Supports saving and reading dates as datetime objects.
Supports using a context manager.
20 def __init__( 21 self, 22 dbpath: str | Path, 23 logger_encoding: str = "utf-8", 24 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 25 ): 26 """ 27 :param dbpath: String or Path object to database file. 28 If a relative path is given, it will be relative to the 29 current working directory. The log file will be saved to the 30 same directory. 31 32 :param logger_message_format: '{' style format string 33 for the logger object.""" 34 self.dbpath = Path(dbpath) 35 self.dbname = Path(dbpath).name 36 self._logger_init( 37 encoding=logger_encoding, message_format=logger_message_format 38 ) 39 self.connection_open = False 40 self.create_manager()
Parameters
dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.
logger_message_format: '{' style format string for the logger object.
49 def create_manager(self): 50 """Create dbManager.py in the same directory 51 as the database file if it doesn't exist.""" 52 manager_template = Path(__file__).parent / "dbmanager.py" 53 manager_path = self.dbpath.parent / "dbmanager.py" 54 if not manager_path.exists(): 55 manager_path.write_text(manager_template.read_text())
Create dbManager.py in the same directory as the database file if it doesn't exist.
57 def open(self): 58 """Open connection to db.""" 59 self.connection = sqlite3.connect( 60 self.dbpath, 61 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 62 timeout=10, 63 ) 64 self.connection.execute("pragma foreign_keys = 1") 65 self.cursor = self.connection.cursor() 66 self.connection_open = True
Open connection to db.
68 def close(self): 69 """Save and close connection to db. 70 71 Call this as soon as you are done using the database if you have 72 multiple threads or processes using the same database.""" 73 if self.connection_open: 74 self.connection.commit() 75 self.connection.close() 76 self.connection_open = False
Save and close connection to db.
Call this as soon as you are done using the database if you have multiple threads or processes using the same database.
158 @_connect 159 def create_tables(self, table_statements: list[str] = []): 160 """Create tables if they don't exist. 161 162 :param table_statements: Each statement should be 163 in the form 'tableName(columnDefinitions)'""" 164 if len(table_statements) > 0: 165 table_names = self.get_table_names() 166 for table in table_statements: 167 if table.split("(")[0].strip() not in table_names: 168 self.cursor.execute(f"create table {table}") 169 self.logger.info(f'{table.split("(")[0]} table created.')
Create tables if they don't exist.
Parameters
- table_statements: Each statement should be in the form 'tableName(columnDefinitions)'
171 @_connect 172 def create_table(self, table: str, column_defs: list[str]): 173 """Create a table if it doesn't exist. 174 175 :param table: Name of the table to create. 176 177 :param column_defs: List of column definitions in 178 proper Sqlite3 sytax. 179 i.e. "columnName text unique" or "columnName int primary key" etc.""" 180 if table not in self.get_table_names(): 181 statement = f"create table {table}({', '.join(column_defs)})" 182 self.cursor.execute(statement) 183 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
Parameters
table: Name of the table to create.
column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.
185 @_connect 186 def get_table_names(self) -> list[str]: 187 """Returns a list of table names from database.""" 188 self.cursor.execute( 189 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 190 ) 191 return [result[0] for result in self.cursor.fetchall()]
Returns a list of table names from database.
193 @_connect 194 def get_column_names(self, table: str) -> list[str]: 195 """Return a list of column names from a table.""" 196 self.cursor.execute(f"select * from {table} where 1=0") 197 return [description[0] for description in self.cursor.description]
Return a list of column names from a table.
199 @_connect 200 def count( 201 self, 202 table: str, 203 match_criteria: list[tuple] | dict = None, 204 exact_match: bool = True, 205 ) -> int: 206 """Return number of items in table. 207 208 :param match_criteria: Can be a list of 2-tuples where each 209 tuple is (columnName, rowValue) or a dictionary where 210 keys are column names and values are row values. 211 If None, all rows from the table will be counted. 212 213 :param exact_match: If False, the row value for a give column 214 in match_criteria will be matched as a substring. Has no effect if 215 match_criteria is None. 216 """ 217 statement = f"select count(_rowid_) from {table}" 218 try: 219 if match_criteria: 220 self.cursor.execute( 221 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 222 ) 223 else: 224 self.cursor.execute(f"{statement}") 225 return self.cursor.fetchone()[0] 226 except: 227 return 0
Return number of items in table.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.
exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.
229 @_connect 230 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 231 """Add row of values to table. 232 233 :param table: The table to insert into. 234 235 :param values: A tuple of values to be inserted into the table. 236 237 :param columns: If None, values param is expected to supply 238 a value for every column in the table. If columns is 239 provided, it should contain the same number of elements as values.""" 240 parameterizer = ", ".join("?" for _ in values) 241 logger_values = ", ".join(str(value) for value in values) 242 try: 243 if columns: 244 columns = ", ".join(column for column in columns) 245 self.cursor.execute( 246 f"insert into {table} ({columns}) values({parameterizer})", values 247 ) 248 else: 249 self.cursor.execute( 250 f"insert into {table} values({parameterizer})", values 251 ) 252 self.logger.info(f'Added "{logger_values}" to {table} table.') 253 except Exception as e: 254 if "constraint" not in str(e).lower(): 255 self.logger.exception( 256 f'Error adding "{logger_values}" to {table} table.' 257 ) 258 else: 259 self.logger.debug(str(e))
Add row of values to table.
Parameters
table: The table to insert into.
values: A tuple of values to be inserted into the table.
columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.
261 @_connect 262 def get_rows( 263 self, 264 table: str, 265 match_criteria: list[tuple] | dict = None, 266 exact_match: bool = True, 267 sort_by_column: str = None, 268 columns_to_return: list[str] = None, 269 values_only: bool = False, 270 ) -> tuple[dict] | tuple[tuple]: 271 """Returns rows from table as a list of dictionaries 272 where the key-value pairs of the dictionaries are 273 column name: row value. 274 275 :param match_criteria: Can be a list of 2-tuples where each 276 tuple is (columnName, rowValue) or a dictionary where 277 keys are column names and values are row values. 278 279 :param exact_match: If False, the rowValue for a give column 280 will be matched as a substring. 281 282 :param sort_by_column: A column name to sort the results by. 283 284 :param columns_to_return: Optional list of column names. 285 If provided, the dictionaries returned by get_rows() will 286 only contain the provided columns. Otherwise every column 287 in the row is returned. 288 289 :param values_only: Return the results as a tuple of tuples 290 instead of a tuple of dictionaries that have column names as keys. 291 The results will still be sorted according to sort_by_column if 292 one is provided. 293 """ 294 if type(columns_to_return) is str: 295 columns_to_return = [columns_to_return] 296 statement = f"select * from {table}" 297 matches = [] 298 if not match_criteria: 299 self.cursor.execute(statement) 300 else: 301 self.cursor.execute( 302 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 303 ) 304 matches = self.cursor.fetchall() 305 results = tuple( 306 self._get_dict(table, match, columns_to_return) for match in matches 307 ) 308 if sort_by_column: 309 results = tuple(sorted(results, key=lambda x: x[sort_by_column])) 310 if values_only: 311 return tuple(tuple(row.values()) for row in results) 312 else: 313 return results
Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
sort_by_column: A column name to sort the results by.
columns_to_return: Optional list of column names. If provided, the dictionaries returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.
values_only: Return the results as a tuple of tuples instead of a tuple of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.
315 @_connect 316 def find( 317 self, table: str, query_string: str, columns: list[str] = None 318 ) -> tuple[dict]: 319 """Search for rows that contain query_string as a substring 320 of any column. 321 322 :param table: The table to search. 323 324 :param query_string: The substring to search for in all columns. 325 326 :param columns: A list of columns to search for query_string. 327 If None, all columns in the table will be searched. 328 """ 329 if type(columns) is str: 330 columns = [columns] 331 results = [] 332 if not columns: 333 columns = self.get_column_names(table) 334 for column in columns: 335 results.extend( 336 [ 337 row 338 for row in self.get_rows( 339 table, [(column, query_string)], exact_match=False 340 ) 341 if row not in results 342 ] 343 ) 344 return tuple(results)
Search for rows that contain query_string as a substring of any column.
Parameters
table: The table to search.
query_string: The substring to search for in all columns.
columns: A list of columns to search for query_string. If None, all columns in the table will be searched.
346 @_connect 347 def delete( 348 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 349 ) -> int: 350 """Delete records from table. 351 352 Returns number of deleted records. 353 354 :param match_criteria: Can be a list of 2-tuples where each 355 tuple is (columnName, rowValue) or a dictionary where 356 keys are column names and values are row values. 357 358 :param exact_match: If False, the rowValue for a give column 359 will be matched as a substring. 360 """ 361 num_matches = self.count(table, match_criteria, exact_match) 362 conditions = self._get_conditions(match_criteria, exact_match) 363 try: 364 self.cursor.execute(f"delete from {table} where {conditions}") 365 self.logger.info( 366 f'Deleted {num_matches} from "{table}" where {conditions}".' 367 ) 368 return num_matches 369 except Exception as e: 370 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 371 return 0
Delete records from table.
Returns number of deleted records.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
373 @_connect 374 def update( 375 self, 376 table: str, 377 column_to_update: str, 378 new_value: Any, 379 match_criteria: list[tuple] | dict = None, 380 ) -> bool: 381 """Update row value for entry matched with match_criteria. 382 383 :param column_to_update: The column to be updated in the matched row. 384 385 :param new_value: The new value to insert. 386 387 :param match_criteria: Can be a list of 2-tuples where each 388 tuple is (columnName, rowValue) or a dictionary where 389 keys are column names and values are row values. 390 If None, every row will be updated. 391 392 Returns True if successful, False if not.""" 393 statement = f"update {table} set {column_to_update} = ?" 394 if match_criteria: 395 if self.count(table, match_criteria) == 0: 396 self.logger.info( 397 f"Couldn't find matching records in {table} table to update to '{new_value}'" 398 ) 399 return False 400 conditions = self._get_conditions(match_criteria) 401 statement += f" where {conditions}" 402 else: 403 conditions = None 404 try: 405 self.cursor.execute( 406 statement, 407 (new_value,), 408 ) 409 self.logger.info( 410 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 411 ) 412 return True 413 except UnboundLocalError: 414 table_filter_string = "\n".join( 415 table_filter for table_filter in match_criteria 416 ) 417 self.logger.error( 418 f"No records found matching filters: {table_filter_string}" 419 ) 420 return False 421 except Exception as e: 422 self.logger.error( 423 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 424 ) 425 return False
Update row value for entry matched with match_criteria.
Parameters
column_to_update: The column to be updated in the matched row.
new_value: The new value to insert.
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.
Returns True if successful, False if not.
427 @_connect 428 def drop_table(self, table: str) -> bool: 429 """Drop a table from the database. 430 431 Returns True if successful, False if not.""" 432 try: 433 self.cursor.execute(f"drop Table {table}") 434 self.logger.info(f'Dropped table "{table}"') 435 except Exception as e: 436 print(e) 437 self.logger.error(f'Failed to drop table "{table}"')
Drop a table from the database.
Returns True if successful, False if not.
439 @_connect 440 def add_column( 441 self, table: str, column: str, _type: str, default_value: str = None 442 ): 443 """Add a new column to table. 444 445 :param column: Name of the column to add. 446 447 :param _type: The data type of the new column. 448 449 :param default_value: Optional default value for the column.""" 450 try: 451 if default_value: 452 self.cursor.execute( 453 f"alter table {table} add column {column} {_type} default {default_value}" 454 ) 455 else: 456 self.cursor.execute(f"alter table {table} add column {column} {_type}") 457 self.logger.info(f'Added column "{column}" to "{table}" table.') 458 except Exception as e: 459 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Add a new column to table.
Parameters
column: Name of the column to add.
_type: The data type of the new column.
default_value: Optional default value for the column.
462def data_to_string( 463 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 464) -> str: 465 """Uses tabulate to produce pretty string output 466 from a list of dictionaries. 467 468 :param data: Assumes all dictionaries in list have the same set of keys. 469 470 :param sort_key: Optional dictionary key to sort data with. 471 472 :param wrap_to_terminal: If True, the table width will be wrapped 473 to fit within the current terminal window. Set to False 474 if the output is going into something like a txt file.""" 475 if len(data) == 0: 476 return "" 477 if sort_key: 478 data = sorted(data, key=lambda d: d[sort_key]) 479 for i, d in enumerate(data): 480 for k in d: 481 data[i][k] = str(data[i][k]) 482 if wrap_to_terminal: 483 terminal_width = os.get_terminal_size().columns 484 max_col_widths = terminal_width 485 """ Reducing the column width by tabulating one row at a time 486 and then reducing further by tabulating the whole set proved to be 487 faster than going straight to tabulating the whole set and reducing 488 the column width.""" 489 too_wide = True 490 while too_wide and max_col_widths > 1: 491 for i, row in enumerate(data): 492 output = tabulate( 493 [row], 494 headers="keys", 495 disable_numparse=True, 496 tablefmt="grid", 497 maxcolwidths=max_col_widths, 498 ) 499 if output.index("\n") > terminal_width: 500 max_col_widths -= 2 501 too_wide = True 502 break 503 too_wide = False 504 else: 505 max_col_widths = None 506 output = tabulate( 507 data, 508 headers="keys", 509 disable_numparse=True, 510 tablefmt="grid", 511 maxcolwidths=max_col_widths, 512 ) 513 # trim max column width until the output string is less wide than the current terminal width. 514 if wrap_to_terminal: 515 while output.index("\n") > terminal_width and max_col_widths > 1: 516 max_col_widths -= 2 517 max_col_widths = max(1, max_col_widths) 518 output = tabulate( 519 data, 520 headers="keys", 521 disable_numparse=True, 522 tablefmt="grid", 523 maxcolwidths=max_col_widths, 524 ) 525 return output
Uses tabulate to produce pretty string output from a list of dictionaries.
Parameters
data: Assumes all dictionaries in list have the same set of keys.
sort_key: Optional dictionary key to sort data with.
wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.