databased.databased
1import logging 2import os 3import sqlite3 4from datetime import datetime 5from functools import wraps 6from pathlib import Path 7from typing import Any 8 9from tabulate import tabulate 10 11 12class DataBased: 13 """Sqli wrapper so queries don't need to be written except table definitions. 14 15 Supports saving and reading dates as datetime objects. 16 17 Supports using a context manager.""" 18 19 def __init__( 20 self, 21 dbpath: str | Path, 22 logger_encoding: str = "utf-8", 23 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 24 ): 25 """ 26 :param dbpath: String or Path object to database file. 27 If a relative path is given, it will be relative to the 28 current working directory. The log file will be saved to the 29 same directory. 30 31 :param logger_message_format: '{' style format string 32 for the logger object.""" 33 self.dbpath = Path(dbpath) 34 self.dbname = Path(dbpath).name 35 self._logger_init( 36 encoding=logger_encoding, message_format=logger_message_format 37 ) 38 self.connection_open = False 39 self.create_manager() 40 41 def __enter__(self): 42 self.open() 43 return self 44 45 def __exit__(self, exception_type, exception_value, exception_traceback): 46 self.close() 47 48 def create_manager(self): 49 """Create dbManager.py in the same directory 50 as the database file if it doesn't exist.""" 51 manager_template = Path(__file__).parent / "dbmanager.py" 52 manager_path = self.dbpath.parent / "dbmanager.py" 53 if not manager_path.exists(): 54 manager_path.write_text(manager_template.read_text()) 55 56 def open(self): 57 """Open connection to db.""" 58 self.connection = sqlite3.connect( 59 self.dbpath, 60 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 61 timeout=10, 62 ) 63 self.connection.execute("pragma foreign_keys = 1") 64 self.cursor = self.connection.cursor() 65 self.connection_open = True 66 67 def close(self): 68 """Save and close connection to db. 69 70 Call this as soon as you are done using the database if you have 71 multiple threads or processes using the same database.""" 72 if self.connection_open: 73 self.connection.commit() 74 self.connection.close() 75 self.connection_open = False 76 77 def _connect(func): 78 """Decorator to open db connection if it isn't already open.""" 79 80 @wraps(func) 81 def inner(*args, **kwargs): 82 self = args[0] 83 if not self.connection_open: 84 self.open() 85 results = func(*args, **kwargs) 86 return results 87 88 return inner 89 90 def _logger_init( 91 self, 92 message_format: str = "{levelname}|-|{asctime}|-|{message}", 93 encoding: str = "utf-8", 94 ): 95 """:param message_format: '{' style format string""" 96 self.logger = logging.getLogger(self.dbname) 97 if not self.logger.hasHandlers(): 98 handler = logging.FileHandler( 99 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 100 ) 101 handler.setFormatter( 102 logging.Formatter( 103 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 104 ) 105 ) 106 self.logger.addHandler(handler) 107 self.logger.setLevel(logging.INFO) 108 109 def _get_dict( 110 self, table: str, values: list, columns_to_return: list[str] = None 111 ) -> dict: 112 """Converts the values of a row into a dictionary with column names as keys. 113 114 :param table: The table that values were pulled from. 115 116 :param values: List of values expected to be the same quantity 117 and in the same order as the column names of table. 118 119 :param columns_to_return: An optional list of column names. 120 If given, only these columns will be included in the returned dictionary. 121 Otherwise all columns and values are returned.""" 122 return { 123 column: value 124 for column, value in zip(self.get_column_names(table), values) 125 if not columns_to_return or column in columns_to_return 126 } 127 128 def _get_conditions( 129 self, match_criteria: list[tuple] | dict, exact_match: bool = True 130 ) -> str: 131 """Builds and returns the conditional portion of a query. 132 133 :param match_criteria: Can be a list of 2-tuples where each 134 tuple is (columnName, rowValue) or a dictionary where 135 keys are column names and values are row values. 136 137 :param exact_match: If False, the rowValue for a give column 138 will be matched as a substring. 139 140 Usage e.g.: 141 142 self.cursor.execute(f'select * from {table} where {conditions}')""" 143 if type(match_criteria) == dict: 144 match_criteria = [(k, v) for k, v in match_criteria.items()] 145 if exact_match: 146 conditions = " and ".join( 147 f'"{column_row[0]}" = "{column_row[1]}"' 148 for column_row in match_criteria 149 ) 150 else: 151 conditions = " and ".join( 152 f'"{column_row[0]}" like "%{column_row[1]}%"' 153 for column_row in match_criteria 154 ) 155 return f"({conditions})" 156 157 @_connect 158 def create_tables(self, table_statements: list[str] = []): 159 """Create tables if they don't exist. 160 161 :param table_statements: Each statement should be 162 in the form 'tableName(columnDefinitions)'""" 163 if len(table_statements) > 0: 164 table_names = self.get_table_names() 165 for table in table_statements: 166 if table.split("(")[0].strip() not in table_names: 167 self.cursor.execute(f"create table {table}") 168 self.logger.info(f'{table.split("(")[0]} table created.') 169 170 @_connect 171 def create_table(self, table: str, column_defs: list[str]): 172 """Create a table if it doesn't exist. 173 174 :param table: Name of the table to create. 175 176 :param column_defs: List of column definitions in 177 proper Sqlite3 sytax. 178 i.e. "columnName text unique" or "columnName int primary key" etc.""" 179 if table not in self.get_table_names(): 180 statement = f"create table {table}({', '.join(column_defs)})" 181 self.cursor.execute(statement) 182 self.logger.info(f"'{table}' table created.") 183 184 @_connect 185 def get_table_names(self) -> list[str]: 186 """Returns a list of table names from database.""" 187 self.cursor.execute( 188 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 189 ) 190 return [result[0] for result in self.cursor.fetchall()] 191 192 @_connect 193 def get_column_names(self, table: str) -> list[str]: 194 """Return a list of column names from a table.""" 195 self.cursor.execute(f"select * from {table} where 1=0") 196 return [description[0] for description in self.cursor.description] 197 198 @_connect 199 def count( 200 self, 201 table: str, 202 match_criteria: list[tuple] | dict = None, 203 exact_match: bool = True, 204 ) -> int: 205 """Return number of items in table. 206 207 :param match_criteria: Can be a list of 2-tuples where each 208 tuple is (columnName, rowValue) or a dictionary where 209 keys are column names and values are row values. 210 If None, all rows from the table will be counted. 211 212 :param exact_match: If False, the row value for a give column 213 in match_criteria will be matched as a substring. Has no effect if 214 match_criteria is None. 215 """ 216 statement = f"select count(_rowid_) from {table}" 217 try: 218 if match_criteria: 219 self.cursor.execute( 220 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 221 ) 222 else: 223 self.cursor.execute(f"{statement}") 224 return self.cursor.fetchone()[0] 225 except: 226 return 0 227 228 @_connect 229 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 230 """Add row of values to table. 231 232 :param table: The table to insert into. 233 234 :param values: A tuple of values to be inserted into the table. 235 236 :param columns: If None, values param is expected to supply 237 a value for every column in the table. If columns is 238 provided, it should contain the same number of elements as values.""" 239 parameterizer = ", ".join("?" for _ in values) 240 logger_values = ", ".join(str(value) for value in values) 241 try: 242 if columns: 243 columns = ", ".join(column for column in columns) 244 self.cursor.execute( 245 f"insert into {table} ({columns}) values({parameterizer})", values 246 ) 247 else: 248 self.cursor.execute( 249 f"insert into {table} values({parameterizer})", values 250 ) 251 self.logger.info(f'Added "{logger_values}" to {table} table.') 252 except Exception as e: 253 if "constraint" not in str(e).lower(): 254 self.logger.exception( 255 f'Error adding "{logger_values}" to {table} table.' 256 ) 257 else: 258 self.logger.debug(str(e)) 259 260 @_connect 261 def get_rows( 262 self, 263 table: str, 264 match_criteria: list[tuple] | dict = None, 265 exact_match: bool = True, 266 sort_by_column: str = None, 267 columns_to_return: list[str] = None, 268 values_only: bool = False, 269 ) -> list[dict] | list[tuple]: 270 """Returns rows from table as a list of dictionaries 271 where the key-value pairs of the dictionaries are 272 column name: row value. 273 274 :param match_criteria: Can be a list of 2-tuples where each 275 tuple is (columnName, rowValue) or a dictionary where 276 keys are column names and values are row values. 277 278 :param exact_match: If False, the rowValue for a give column 279 will be matched as a substring. 280 281 :param sort_by_column: A column name to sort the results by. 282 283 :param columns_to_return: Optional list of column names. 284 If provided, the elements returned by get_rows() will 285 only contain the provided columns. Otherwise every column 286 in the row is returned. 287 288 :param values_only: Return the results as a list of tuples 289 instead of a list of dictionaries that have column names as keys. 290 The results will still be sorted according to sort_by_column if 291 one is provided. 292 """ 293 if type(columns_to_return) is str: 294 columns_to_return = [columns_to_return] 295 statement = f"select * from {table}" 296 matches = [] 297 if not match_criteria: 298 self.cursor.execute(statement) 299 else: 300 self.cursor.execute( 301 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 302 ) 303 matches = self.cursor.fetchall() 304 results = [self._get_dict(table, match, columns_to_return) for match in matches] 305 if sort_by_column: 306 results = sorted(results, key=lambda x: x[sort_by_column]) 307 if values_only: 308 return [tuple(row.values()) for row in results] 309 else: 310 return results 311 312 @_connect 313 def find( 314 self, table: str, query_string: str, columns: list[str] = None 315 ) -> tuple[dict]: 316 """Search for rows that contain query_string as a substring 317 of any column. 318 319 :param table: The table to search. 320 321 :param query_string: The substring to search for in all columns. 322 323 :param columns: A list of columns to search for query_string. 324 If None, all columns in the table will be searched. 325 """ 326 if type(columns) is str: 327 columns = [columns] 328 results = [] 329 if not columns: 330 columns = self.get_column_names(table) 331 for column in columns: 332 results.extend( 333 [ 334 row 335 for row in self.get_rows( 336 table, [(column, query_string)], exact_match=False 337 ) 338 if row not in results 339 ] 340 ) 341 return results 342 343 @_connect 344 def delete( 345 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 346 ) -> int: 347 """Delete records from table. 348 349 Returns number of deleted records. 350 351 :param match_criteria: Can be a list of 2-tuples where each 352 tuple is (columnName, rowValue) or a dictionary where 353 keys are column names and values are row values. 354 355 :param exact_match: If False, the rowValue for a give column 356 will be matched as a substring. 357 """ 358 num_matches = self.count(table, match_criteria, exact_match) 359 conditions = self._get_conditions(match_criteria, exact_match) 360 try: 361 self.cursor.execute(f"delete from {table} where {conditions}") 362 self.logger.info( 363 f'Deleted {num_matches} from "{table}" where {conditions}".' 364 ) 365 return num_matches 366 except Exception as e: 367 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 368 return 0 369 370 @_connect 371 def update( 372 self, 373 table: str, 374 column_to_update: str, 375 new_value: Any, 376 match_criteria: list[tuple] | dict = None, 377 ) -> bool: 378 """Update row value for entry matched with match_criteria. 379 380 :param column_to_update: The column to be updated in the matched row. 381 382 :param new_value: The new value to insert. 383 384 :param match_criteria: Can be a list of 2-tuples where each 385 tuple is (columnName, rowValue) or a dictionary where 386 keys are column names and values are row values. 387 If None, every row will be updated. 388 389 Returns True if successful, False if not.""" 390 statement = f"update {table} set {column_to_update} = ?" 391 if match_criteria: 392 if self.count(table, match_criteria) == 0: 393 self.logger.info( 394 f"Couldn't find matching records in {table} table to update to '{new_value}'" 395 ) 396 return False 397 conditions = self._get_conditions(match_criteria) 398 statement += f" where {conditions}" 399 else: 400 conditions = None 401 try: 402 self.cursor.execute( 403 statement, 404 (new_value,), 405 ) 406 self.logger.info( 407 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 408 ) 409 return True 410 except UnboundLocalError: 411 table_filter_string = "\n".join( 412 table_filter for table_filter in match_criteria 413 ) 414 self.logger.error( 415 f"No records found matching filters: {table_filter_string}" 416 ) 417 return False 418 except Exception as e: 419 self.logger.error( 420 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 421 ) 422 return False 423 424 @_connect 425 def drop_table(self, table: str) -> bool: 426 """Drop a table from the database. 427 428 Returns True if successful, False if not.""" 429 try: 430 self.cursor.execute(f"drop Table {table}") 431 self.logger.info(f'Dropped table "{table}"') 432 except Exception as e: 433 print(e) 434 self.logger.error(f'Failed to drop table "{table}"') 435 436 @_connect 437 def add_column( 438 self, table: str, column: str, _type: str, default_value: str = None 439 ): 440 """Add a new column to table. 441 442 :param column: Name of the column to add. 443 444 :param _type: The data type of the new column. 445 446 :param default_value: Optional default value for the column.""" 447 try: 448 if default_value: 449 self.cursor.execute( 450 f"alter table {table} add column {column} {_type} default {default_value}" 451 ) 452 else: 453 self.cursor.execute(f"alter table {table} add column {column} {_type}") 454 self.logger.info(f'Added column "{column}" to "{table}" table.') 455 except Exception as e: 456 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 457 458 459def data_to_string( 460 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 461) -> str: 462 """Uses tabulate to produce pretty string output 463 from a list of dictionaries. 464 465 :param data: Assumes all dictionaries in list have the same set of keys. 466 467 :param sort_key: Optional dictionary key to sort data with. 468 469 :param wrap_to_terminal: If True, the table width will be wrapped 470 to fit within the current terminal window. Set to False 471 if the output is going into something like a txt file.""" 472 if len(data) == 0: 473 return "" 474 if sort_key: 475 data = sorted(data, key=lambda d: d[sort_key]) 476 for i, d in enumerate(data): 477 for k in d: 478 data[i][k] = str(data[i][k]) 479 if wrap_to_terminal: 480 terminal_width = os.get_terminal_size().columns 481 max_col_widths = terminal_width 482 """ Reducing the column width by tabulating one row at a time 483 and then reducing further by tabulating the whole set proved to be 484 faster than going straight to tabulating the whole set and reducing 485 the column width.""" 486 too_wide = True 487 while too_wide and max_col_widths > 1: 488 for i, row in enumerate(data): 489 output = tabulate( 490 [row], 491 headers="keys", 492 disable_numparse=True, 493 tablefmt="grid", 494 maxcolwidths=max_col_widths, 495 ) 496 if output.index("\n") > terminal_width: 497 max_col_widths -= 2 498 too_wide = True 499 break 500 too_wide = False 501 else: 502 max_col_widths = None 503 output = tabulate( 504 data, 505 headers="keys", 506 disable_numparse=True, 507 tablefmt="grid", 508 maxcolwidths=max_col_widths, 509 ) 510 # trim max column width until the output string is less wide than the current terminal width. 511 if wrap_to_terminal: 512 while output.index("\n") > terminal_width and max_col_widths > 1: 513 max_col_widths -= 2 514 max_col_widths = max(1, max_col_widths) 515 output = tabulate( 516 data, 517 headers="keys", 518 disable_numparse=True, 519 tablefmt="grid", 520 maxcolwidths=max_col_widths, 521 ) 522 return output
13class DataBased: 14 """Sqli wrapper so queries don't need to be written except table definitions. 15 16 Supports saving and reading dates as datetime objects. 17 18 Supports using a context manager.""" 19 20 def __init__( 21 self, 22 dbpath: str | Path, 23 logger_encoding: str = "utf-8", 24 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 25 ): 26 """ 27 :param dbpath: String or Path object to database file. 28 If a relative path is given, it will be relative to the 29 current working directory. The log file will be saved to the 30 same directory. 31 32 :param logger_message_format: '{' style format string 33 for the logger object.""" 34 self.dbpath = Path(dbpath) 35 self.dbname = Path(dbpath).name 36 self._logger_init( 37 encoding=logger_encoding, message_format=logger_message_format 38 ) 39 self.connection_open = False 40 self.create_manager() 41 42 def __enter__(self): 43 self.open() 44 return self 45 46 def __exit__(self, exception_type, exception_value, exception_traceback): 47 self.close() 48 49 def create_manager(self): 50 """Create dbManager.py in the same directory 51 as the database file if it doesn't exist.""" 52 manager_template = Path(__file__).parent / "dbmanager.py" 53 manager_path = self.dbpath.parent / "dbmanager.py" 54 if not manager_path.exists(): 55 manager_path.write_text(manager_template.read_text()) 56 57 def open(self): 58 """Open connection to db.""" 59 self.connection = sqlite3.connect( 60 self.dbpath, 61 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 62 timeout=10, 63 ) 64 self.connection.execute("pragma foreign_keys = 1") 65 self.cursor = self.connection.cursor() 66 self.connection_open = True 67 68 def close(self): 69 """Save and close connection to db. 70 71 Call this as soon as you are done using the database if you have 72 multiple threads or processes using the same database.""" 73 if self.connection_open: 74 self.connection.commit() 75 self.connection.close() 76 self.connection_open = False 77 78 def _connect(func): 79 """Decorator to open db connection if it isn't already open.""" 80 81 @wraps(func) 82 def inner(*args, **kwargs): 83 self = args[0] 84 if not self.connection_open: 85 self.open() 86 results = func(*args, **kwargs) 87 return results 88 89 return inner 90 91 def _logger_init( 92 self, 93 message_format: str = "{levelname}|-|{asctime}|-|{message}", 94 encoding: str = "utf-8", 95 ): 96 """:param message_format: '{' style format string""" 97 self.logger = logging.getLogger(self.dbname) 98 if not self.logger.hasHandlers(): 99 handler = logging.FileHandler( 100 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 101 ) 102 handler.setFormatter( 103 logging.Formatter( 104 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 105 ) 106 ) 107 self.logger.addHandler(handler) 108 self.logger.setLevel(logging.INFO) 109 110 def _get_dict( 111 self, table: str, values: list, columns_to_return: list[str] = None 112 ) -> dict: 113 """Converts the values of a row into a dictionary with column names as keys. 114 115 :param table: The table that values were pulled from. 116 117 :param values: List of values expected to be the same quantity 118 and in the same order as the column names of table. 119 120 :param columns_to_return: An optional list of column names. 121 If given, only these columns will be included in the returned dictionary. 122 Otherwise all columns and values are returned.""" 123 return { 124 column: value 125 for column, value in zip(self.get_column_names(table), values) 126 if not columns_to_return or column in columns_to_return 127 } 128 129 def _get_conditions( 130 self, match_criteria: list[tuple] | dict, exact_match: bool = True 131 ) -> str: 132 """Builds and returns the conditional portion of a query. 133 134 :param match_criteria: Can be a list of 2-tuples where each 135 tuple is (columnName, rowValue) or a dictionary where 136 keys are column names and values are row values. 137 138 :param exact_match: If False, the rowValue for a give column 139 will be matched as a substring. 140 141 Usage e.g.: 142 143 self.cursor.execute(f'select * from {table} where {conditions}')""" 144 if type(match_criteria) == dict: 145 match_criteria = [(k, v) for k, v in match_criteria.items()] 146 if exact_match: 147 conditions = " and ".join( 148 f'"{column_row[0]}" = "{column_row[1]}"' 149 for column_row in match_criteria 150 ) 151 else: 152 conditions = " and ".join( 153 f'"{column_row[0]}" like "%{column_row[1]}%"' 154 for column_row in match_criteria 155 ) 156 return f"({conditions})" 157 158 @_connect 159 def create_tables(self, table_statements: list[str] = []): 160 """Create tables if they don't exist. 161 162 :param table_statements: Each statement should be 163 in the form 'tableName(columnDefinitions)'""" 164 if len(table_statements) > 0: 165 table_names = self.get_table_names() 166 for table in table_statements: 167 if table.split("(")[0].strip() not in table_names: 168 self.cursor.execute(f"create table {table}") 169 self.logger.info(f'{table.split("(")[0]} table created.') 170 171 @_connect 172 def create_table(self, table: str, column_defs: list[str]): 173 """Create a table if it doesn't exist. 174 175 :param table: Name of the table to create. 176 177 :param column_defs: List of column definitions in 178 proper Sqlite3 sytax. 179 i.e. "columnName text unique" or "columnName int primary key" etc.""" 180 if table not in self.get_table_names(): 181 statement = f"create table {table}({', '.join(column_defs)})" 182 self.cursor.execute(statement) 183 self.logger.info(f"'{table}' table created.") 184 185 @_connect 186 def get_table_names(self) -> list[str]: 187 """Returns a list of table names from database.""" 188 self.cursor.execute( 189 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 190 ) 191 return [result[0] for result in self.cursor.fetchall()] 192 193 @_connect 194 def get_column_names(self, table: str) -> list[str]: 195 """Return a list of column names from a table.""" 196 self.cursor.execute(f"select * from {table} where 1=0") 197 return [description[0] for description in self.cursor.description] 198 199 @_connect 200 def count( 201 self, 202 table: str, 203 match_criteria: list[tuple] | dict = None, 204 exact_match: bool = True, 205 ) -> int: 206 """Return number of items in table. 207 208 :param match_criteria: Can be a list of 2-tuples where each 209 tuple is (columnName, rowValue) or a dictionary where 210 keys are column names and values are row values. 211 If None, all rows from the table will be counted. 212 213 :param exact_match: If False, the row value for a give column 214 in match_criteria will be matched as a substring. Has no effect if 215 match_criteria is None. 216 """ 217 statement = f"select count(_rowid_) from {table}" 218 try: 219 if match_criteria: 220 self.cursor.execute( 221 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 222 ) 223 else: 224 self.cursor.execute(f"{statement}") 225 return self.cursor.fetchone()[0] 226 except: 227 return 0 228 229 @_connect 230 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 231 """Add row of values to table. 232 233 :param table: The table to insert into. 234 235 :param values: A tuple of values to be inserted into the table. 236 237 :param columns: If None, values param is expected to supply 238 a value for every column in the table. If columns is 239 provided, it should contain the same number of elements as values.""" 240 parameterizer = ", ".join("?" for _ in values) 241 logger_values = ", ".join(str(value) for value in values) 242 try: 243 if columns: 244 columns = ", ".join(column for column in columns) 245 self.cursor.execute( 246 f"insert into {table} ({columns}) values({parameterizer})", values 247 ) 248 else: 249 self.cursor.execute( 250 f"insert into {table} values({parameterizer})", values 251 ) 252 self.logger.info(f'Added "{logger_values}" to {table} table.') 253 except Exception as e: 254 if "constraint" not in str(e).lower(): 255 self.logger.exception( 256 f'Error adding "{logger_values}" to {table} table.' 257 ) 258 else: 259 self.logger.debug(str(e)) 260 261 @_connect 262 def get_rows( 263 self, 264 table: str, 265 match_criteria: list[tuple] | dict = None, 266 exact_match: bool = True, 267 sort_by_column: str = None, 268 columns_to_return: list[str] = None, 269 values_only: bool = False, 270 ) -> list[dict] | list[tuple]: 271 """Returns rows from table as a list of dictionaries 272 where the key-value pairs of the dictionaries are 273 column name: row value. 274 275 :param match_criteria: Can be a list of 2-tuples where each 276 tuple is (columnName, rowValue) or a dictionary where 277 keys are column names and values are row values. 278 279 :param exact_match: If False, the rowValue for a give column 280 will be matched as a substring. 281 282 :param sort_by_column: A column name to sort the results by. 283 284 :param columns_to_return: Optional list of column names. 285 If provided, the elements returned by get_rows() will 286 only contain the provided columns. Otherwise every column 287 in the row is returned. 288 289 :param values_only: Return the results as a list of tuples 290 instead of a list of dictionaries that have column names as keys. 291 The results will still be sorted according to sort_by_column if 292 one is provided. 293 """ 294 if type(columns_to_return) is str: 295 columns_to_return = [columns_to_return] 296 statement = f"select * from {table}" 297 matches = [] 298 if not match_criteria: 299 self.cursor.execute(statement) 300 else: 301 self.cursor.execute( 302 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 303 ) 304 matches = self.cursor.fetchall() 305 results = [self._get_dict(table, match, columns_to_return) for match in matches] 306 if sort_by_column: 307 results = sorted(results, key=lambda x: x[sort_by_column]) 308 if values_only: 309 return [tuple(row.values()) for row in results] 310 else: 311 return results 312 313 @_connect 314 def find( 315 self, table: str, query_string: str, columns: list[str] = None 316 ) -> tuple[dict]: 317 """Search for rows that contain query_string as a substring 318 of any column. 319 320 :param table: The table to search. 321 322 :param query_string: The substring to search for in all columns. 323 324 :param columns: A list of columns to search for query_string. 325 If None, all columns in the table will be searched. 326 """ 327 if type(columns) is str: 328 columns = [columns] 329 results = [] 330 if not columns: 331 columns = self.get_column_names(table) 332 for column in columns: 333 results.extend( 334 [ 335 row 336 for row in self.get_rows( 337 table, [(column, query_string)], exact_match=False 338 ) 339 if row not in results 340 ] 341 ) 342 return results 343 344 @_connect 345 def delete( 346 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 347 ) -> int: 348 """Delete records from table. 349 350 Returns number of deleted records. 351 352 :param match_criteria: Can be a list of 2-tuples where each 353 tuple is (columnName, rowValue) or a dictionary where 354 keys are column names and values are row values. 355 356 :param exact_match: If False, the rowValue for a give column 357 will be matched as a substring. 358 """ 359 num_matches = self.count(table, match_criteria, exact_match) 360 conditions = self._get_conditions(match_criteria, exact_match) 361 try: 362 self.cursor.execute(f"delete from {table} where {conditions}") 363 self.logger.info( 364 f'Deleted {num_matches} from "{table}" where {conditions}".' 365 ) 366 return num_matches 367 except Exception as e: 368 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 369 return 0 370 371 @_connect 372 def update( 373 self, 374 table: str, 375 column_to_update: str, 376 new_value: Any, 377 match_criteria: list[tuple] | dict = None, 378 ) -> bool: 379 """Update row value for entry matched with match_criteria. 380 381 :param column_to_update: The column to be updated in the matched row. 382 383 :param new_value: The new value to insert. 384 385 :param match_criteria: Can be a list of 2-tuples where each 386 tuple is (columnName, rowValue) or a dictionary where 387 keys are column names and values are row values. 388 If None, every row will be updated. 389 390 Returns True if successful, False if not.""" 391 statement = f"update {table} set {column_to_update} = ?" 392 if match_criteria: 393 if self.count(table, match_criteria) == 0: 394 self.logger.info( 395 f"Couldn't find matching records in {table} table to update to '{new_value}'" 396 ) 397 return False 398 conditions = self._get_conditions(match_criteria) 399 statement += f" where {conditions}" 400 else: 401 conditions = None 402 try: 403 self.cursor.execute( 404 statement, 405 (new_value,), 406 ) 407 self.logger.info( 408 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 409 ) 410 return True 411 except UnboundLocalError: 412 table_filter_string = "\n".join( 413 table_filter for table_filter in match_criteria 414 ) 415 self.logger.error( 416 f"No records found matching filters: {table_filter_string}" 417 ) 418 return False 419 except Exception as e: 420 self.logger.error( 421 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 422 ) 423 return False 424 425 @_connect 426 def drop_table(self, table: str) -> bool: 427 """Drop a table from the database. 428 429 Returns True if successful, False if not.""" 430 try: 431 self.cursor.execute(f"drop Table {table}") 432 self.logger.info(f'Dropped table "{table}"') 433 except Exception as e: 434 print(e) 435 self.logger.error(f'Failed to drop table "{table}"') 436 437 @_connect 438 def add_column( 439 self, table: str, column: str, _type: str, default_value: str = None 440 ): 441 """Add a new column to table. 442 443 :param column: Name of the column to add. 444 445 :param _type: The data type of the new column. 446 447 :param default_value: Optional default value for the column.""" 448 try: 449 if default_value: 450 self.cursor.execute( 451 f"alter table {table} add column {column} {_type} default {default_value}" 452 ) 453 else: 454 self.cursor.execute(f"alter table {table} add column {column} {_type}") 455 self.logger.info(f'Added column "{column}" to "{table}" table.') 456 except Exception as e: 457 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Sqli wrapper so queries don't need to be written except table definitions.
Supports saving and reading dates as datetime objects.
Supports using a context manager.
20 def __init__( 21 self, 22 dbpath: str | Path, 23 logger_encoding: str = "utf-8", 24 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 25 ): 26 """ 27 :param dbpath: String or Path object to database file. 28 If a relative path is given, it will be relative to the 29 current working directory. The log file will be saved to the 30 same directory. 31 32 :param logger_message_format: '{' style format string 33 for the logger object.""" 34 self.dbpath = Path(dbpath) 35 self.dbname = Path(dbpath).name 36 self._logger_init( 37 encoding=logger_encoding, message_format=logger_message_format 38 ) 39 self.connection_open = False 40 self.create_manager()
Parameters
dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.
logger_message_format: '{' style format string for the logger object.
49 def create_manager(self): 50 """Create dbManager.py in the same directory 51 as the database file if it doesn't exist.""" 52 manager_template = Path(__file__).parent / "dbmanager.py" 53 manager_path = self.dbpath.parent / "dbmanager.py" 54 if not manager_path.exists(): 55 manager_path.write_text(manager_template.read_text())
Create dbManager.py in the same directory as the database file if it doesn't exist.
57 def open(self): 58 """Open connection to db.""" 59 self.connection = sqlite3.connect( 60 self.dbpath, 61 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 62 timeout=10, 63 ) 64 self.connection.execute("pragma foreign_keys = 1") 65 self.cursor = self.connection.cursor() 66 self.connection_open = True
Open connection to db.
68 def close(self): 69 """Save and close connection to db. 70 71 Call this as soon as you are done using the database if you have 72 multiple threads or processes using the same database.""" 73 if self.connection_open: 74 self.connection.commit() 75 self.connection.close() 76 self.connection_open = False
Save and close connection to db.
Call this as soon as you are done using the database if you have multiple threads or processes using the same database.
158 @_connect 159 def create_tables(self, table_statements: list[str] = []): 160 """Create tables if they don't exist. 161 162 :param table_statements: Each statement should be 163 in the form 'tableName(columnDefinitions)'""" 164 if len(table_statements) > 0: 165 table_names = self.get_table_names() 166 for table in table_statements: 167 if table.split("(")[0].strip() not in table_names: 168 self.cursor.execute(f"create table {table}") 169 self.logger.info(f'{table.split("(")[0]} table created.')
Create tables if they don't exist.
Parameters
- table_statements: Each statement should be in the form 'tableName(columnDefinitions)'
171 @_connect 172 def create_table(self, table: str, column_defs: list[str]): 173 """Create a table if it doesn't exist. 174 175 :param table: Name of the table to create. 176 177 :param column_defs: List of column definitions in 178 proper Sqlite3 sytax. 179 i.e. "columnName text unique" or "columnName int primary key" etc.""" 180 if table not in self.get_table_names(): 181 statement = f"create table {table}({', '.join(column_defs)})" 182 self.cursor.execute(statement) 183 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
Parameters
table: Name of the table to create.
column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.
185 @_connect 186 def get_table_names(self) -> list[str]: 187 """Returns a list of table names from database.""" 188 self.cursor.execute( 189 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 190 ) 191 return [result[0] for result in self.cursor.fetchall()]
Returns a list of table names from database.
193 @_connect 194 def get_column_names(self, table: str) -> list[str]: 195 """Return a list of column names from a table.""" 196 self.cursor.execute(f"select * from {table} where 1=0") 197 return [description[0] for description in self.cursor.description]
Return a list of column names from a table.
199 @_connect 200 def count( 201 self, 202 table: str, 203 match_criteria: list[tuple] | dict = None, 204 exact_match: bool = True, 205 ) -> int: 206 """Return number of items in table. 207 208 :param match_criteria: Can be a list of 2-tuples where each 209 tuple is (columnName, rowValue) or a dictionary where 210 keys are column names and values are row values. 211 If None, all rows from the table will be counted. 212 213 :param exact_match: If False, the row value for a give column 214 in match_criteria will be matched as a substring. Has no effect if 215 match_criteria is None. 216 """ 217 statement = f"select count(_rowid_) from {table}" 218 try: 219 if match_criteria: 220 self.cursor.execute( 221 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 222 ) 223 else: 224 self.cursor.execute(f"{statement}") 225 return self.cursor.fetchone()[0] 226 except: 227 return 0
Return number of items in table.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.
exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.
229 @_connect 230 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 231 """Add row of values to table. 232 233 :param table: The table to insert into. 234 235 :param values: A tuple of values to be inserted into the table. 236 237 :param columns: If None, values param is expected to supply 238 a value for every column in the table. If columns is 239 provided, it should contain the same number of elements as values.""" 240 parameterizer = ", ".join("?" for _ in values) 241 logger_values = ", ".join(str(value) for value in values) 242 try: 243 if columns: 244 columns = ", ".join(column for column in columns) 245 self.cursor.execute( 246 f"insert into {table} ({columns}) values({parameterizer})", values 247 ) 248 else: 249 self.cursor.execute( 250 f"insert into {table} values({parameterizer})", values 251 ) 252 self.logger.info(f'Added "{logger_values}" to {table} table.') 253 except Exception as e: 254 if "constraint" not in str(e).lower(): 255 self.logger.exception( 256 f'Error adding "{logger_values}" to {table} table.' 257 ) 258 else: 259 self.logger.debug(str(e))
Add row of values to table.
Parameters
table: The table to insert into.
values: A tuple of values to be inserted into the table.
columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.
261 @_connect 262 def get_rows( 263 self, 264 table: str, 265 match_criteria: list[tuple] | dict = None, 266 exact_match: bool = True, 267 sort_by_column: str = None, 268 columns_to_return: list[str] = None, 269 values_only: bool = False, 270 ) -> list[dict] | list[tuple]: 271 """Returns rows from table as a list of dictionaries 272 where the key-value pairs of the dictionaries are 273 column name: row value. 274 275 :param match_criteria: Can be a list of 2-tuples where each 276 tuple is (columnName, rowValue) or a dictionary where 277 keys are column names and values are row values. 278 279 :param exact_match: If False, the rowValue for a give column 280 will be matched as a substring. 281 282 :param sort_by_column: A column name to sort the results by. 283 284 :param columns_to_return: Optional list of column names. 285 If provided, the elements returned by get_rows() will 286 only contain the provided columns. Otherwise every column 287 in the row is returned. 288 289 :param values_only: Return the results as a list of tuples 290 instead of a list of dictionaries that have column names as keys. 291 The results will still be sorted according to sort_by_column if 292 one is provided. 293 """ 294 if type(columns_to_return) is str: 295 columns_to_return = [columns_to_return] 296 statement = f"select * from {table}" 297 matches = [] 298 if not match_criteria: 299 self.cursor.execute(statement) 300 else: 301 self.cursor.execute( 302 f"{statement} where {self._get_conditions(match_criteria, exact_match)}" 303 ) 304 matches = self.cursor.fetchall() 305 results = [self._get_dict(table, match, columns_to_return) for match in matches] 306 if sort_by_column: 307 results = sorted(results, key=lambda x: x[sort_by_column]) 308 if values_only: 309 return [tuple(row.values()) for row in results] 310 else: 311 return results
Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
sort_by_column: A column name to sort the results by.
columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.
values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.
313 @_connect 314 def find( 315 self, table: str, query_string: str, columns: list[str] = None 316 ) -> tuple[dict]: 317 """Search for rows that contain query_string as a substring 318 of any column. 319 320 :param table: The table to search. 321 322 :param query_string: The substring to search for in all columns. 323 324 :param columns: A list of columns to search for query_string. 325 If None, all columns in the table will be searched. 326 """ 327 if type(columns) is str: 328 columns = [columns] 329 results = [] 330 if not columns: 331 columns = self.get_column_names(table) 332 for column in columns: 333 results.extend( 334 [ 335 row 336 for row in self.get_rows( 337 table, [(column, query_string)], exact_match=False 338 ) 339 if row not in results 340 ] 341 ) 342 return results
Search for rows that contain query_string as a substring of any column.
Parameters
table: The table to search.
query_string: The substring to search for in all columns.
columns: A list of columns to search for query_string. If None, all columns in the table will be searched.
344 @_connect 345 def delete( 346 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 347 ) -> int: 348 """Delete records from table. 349 350 Returns number of deleted records. 351 352 :param match_criteria: Can be a list of 2-tuples where each 353 tuple is (columnName, rowValue) or a dictionary where 354 keys are column names and values are row values. 355 356 :param exact_match: If False, the rowValue for a give column 357 will be matched as a substring. 358 """ 359 num_matches = self.count(table, match_criteria, exact_match) 360 conditions = self._get_conditions(match_criteria, exact_match) 361 try: 362 self.cursor.execute(f"delete from {table} where {conditions}") 363 self.logger.info( 364 f'Deleted {num_matches} from "{table}" where {conditions}".' 365 ) 366 return num_matches 367 except Exception as e: 368 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 369 return 0
Delete records from table.
Returns number of deleted records.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
371 @_connect 372 def update( 373 self, 374 table: str, 375 column_to_update: str, 376 new_value: Any, 377 match_criteria: list[tuple] | dict = None, 378 ) -> bool: 379 """Update row value for entry matched with match_criteria. 380 381 :param column_to_update: The column to be updated in the matched row. 382 383 :param new_value: The new value to insert. 384 385 :param match_criteria: Can be a list of 2-tuples where each 386 tuple is (columnName, rowValue) or a dictionary where 387 keys are column names and values are row values. 388 If None, every row will be updated. 389 390 Returns True if successful, False if not.""" 391 statement = f"update {table} set {column_to_update} = ?" 392 if match_criteria: 393 if self.count(table, match_criteria) == 0: 394 self.logger.info( 395 f"Couldn't find matching records in {table} table to update to '{new_value}'" 396 ) 397 return False 398 conditions = self._get_conditions(match_criteria) 399 statement += f" where {conditions}" 400 else: 401 conditions = None 402 try: 403 self.cursor.execute( 404 statement, 405 (new_value,), 406 ) 407 self.logger.info( 408 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 409 ) 410 return True 411 except UnboundLocalError: 412 table_filter_string = "\n".join( 413 table_filter for table_filter in match_criteria 414 ) 415 self.logger.error( 416 f"No records found matching filters: {table_filter_string}" 417 ) 418 return False 419 except Exception as e: 420 self.logger.error( 421 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 422 ) 423 return False
Update row value for entry matched with match_criteria.
Parameters
column_to_update: The column to be updated in the matched row.
new_value: The new value to insert.
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.
Returns True if successful, False if not.
425 @_connect 426 def drop_table(self, table: str) -> bool: 427 """Drop a table from the database. 428 429 Returns True if successful, False if not.""" 430 try: 431 self.cursor.execute(f"drop Table {table}") 432 self.logger.info(f'Dropped table "{table}"') 433 except Exception as e: 434 print(e) 435 self.logger.error(f'Failed to drop table "{table}"')
Drop a table from the database.
Returns True if successful, False if not.
437 @_connect 438 def add_column( 439 self, table: str, column: str, _type: str, default_value: str = None 440 ): 441 """Add a new column to table. 442 443 :param column: Name of the column to add. 444 445 :param _type: The data type of the new column. 446 447 :param default_value: Optional default value for the column.""" 448 try: 449 if default_value: 450 self.cursor.execute( 451 f"alter table {table} add column {column} {_type} default {default_value}" 452 ) 453 else: 454 self.cursor.execute(f"alter table {table} add column {column} {_type}") 455 self.logger.info(f'Added column "{column}" to "{table}" table.') 456 except Exception as e: 457 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Add a new column to table.
Parameters
column: Name of the column to add.
_type: The data type of the new column.
default_value: Optional default value for the column.
460def data_to_string( 461 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 462) -> str: 463 """Uses tabulate to produce pretty string output 464 from a list of dictionaries. 465 466 :param data: Assumes all dictionaries in list have the same set of keys. 467 468 :param sort_key: Optional dictionary key to sort data with. 469 470 :param wrap_to_terminal: If True, the table width will be wrapped 471 to fit within the current terminal window. Set to False 472 if the output is going into something like a txt file.""" 473 if len(data) == 0: 474 return "" 475 if sort_key: 476 data = sorted(data, key=lambda d: d[sort_key]) 477 for i, d in enumerate(data): 478 for k in d: 479 data[i][k] = str(data[i][k]) 480 if wrap_to_terminal: 481 terminal_width = os.get_terminal_size().columns 482 max_col_widths = terminal_width 483 """ Reducing the column width by tabulating one row at a time 484 and then reducing further by tabulating the whole set proved to be 485 faster than going straight to tabulating the whole set and reducing 486 the column width.""" 487 too_wide = True 488 while too_wide and max_col_widths > 1: 489 for i, row in enumerate(data): 490 output = tabulate( 491 [row], 492 headers="keys", 493 disable_numparse=True, 494 tablefmt="grid", 495 maxcolwidths=max_col_widths, 496 ) 497 if output.index("\n") > terminal_width: 498 max_col_widths -= 2 499 too_wide = True 500 break 501 too_wide = False 502 else: 503 max_col_widths = None 504 output = tabulate( 505 data, 506 headers="keys", 507 disable_numparse=True, 508 tablefmt="grid", 509 maxcolwidths=max_col_widths, 510 ) 511 # trim max column width until the output string is less wide than the current terminal width. 512 if wrap_to_terminal: 513 while output.index("\n") > terminal_width and max_col_widths > 1: 514 max_col_widths -= 2 515 max_col_widths = max(1, max_col_widths) 516 output = tabulate( 517 data, 518 headers="keys", 519 disable_numparse=True, 520 tablefmt="grid", 521 maxcolwidths=max_col_widths, 522 ) 523 return output
Uses tabulate to produce pretty string output from a list of dictionaries.
Parameters
data: Assumes all dictionaries in list have the same set of keys.
sort_key: Optional dictionary key to sort data with.
wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.