databased.databased
1import logging 2import os 3import shutil 4import sqlite3 5from datetime import datetime 6from functools import wraps 7from pathlib import Path 8from typing import Any 9 10import pandas 11from tabulate import tabulate 12 13 14def _connect(func): 15 """Decorator to open db connection if it isn't already open.""" 16 17 @wraps(func) 18 def inner(self, *args, **kwargs): 19 if not self.connection_open: 20 self.open() 21 results = func(self, *args, **kwargs) 22 return results 23 24 return inner 25 26 27class DataBased: 28 """Sqli wrapper so queries don't need to be written except table definitions. 29 30 Supports saving and reading dates as datetime objects. 31 32 Supports using a context manager.""" 33 34 def __init__( 35 self, 36 dbpath: str | Path, 37 logger_encoding: str = "utf-8", 38 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 39 ): 40 """ 41 :param dbpath: String or Path object to database file. 42 If a relative path is given, it will be relative to the 43 current working directory. The log file will be saved to the 44 same directory. 45 46 :param logger_message_format: '{' style format string 47 for the logger object.""" 48 self.dbpath = Path(dbpath) 49 self.dbname = Path(dbpath).name 50 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 51 self._logger_init( 52 encoding=logger_encoding, message_format=logger_message_format 53 ) 54 self.connection_open = False 55 self.create_manager() 56 57 def __enter__(self): 58 self.open() 59 return self 60 61 def __exit__(self, exception_type, exception_value, exception_traceback): 62 self.close() 63 64 def create_manager(self): 65 """Create dbManager.py in the same directory 66 as the database file if it doesn't exist.""" 67 manager_template = Path(__file__).parent / "dbmanager.py" 68 manager_path = self.dbpath.parent / "dbmanager.py" 69 if not manager_path.exists(): 70 shutil.copyfile(manager_template, manager_path) 71 72 def open(self): 73 """Open connection to db.""" 74 self.connection = sqlite3.connect( 75 self.dbpath, 76 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 77 timeout=10, 78 ) 79 self.connection.execute("pragma foreign_keys = 1") 80 self.cursor = self.connection.cursor() 81 self.connection_open = True 82 83 def close(self): 84 """Save and close connection to db. 85 86 Call this as soon as you are done using the database if you have 87 multiple threads or processes using the same database.""" 88 if self.connection_open: 89 self.connection.commit() 90 self.connection.close() 91 self.connection_open = False 92 93 def _logger_init( 94 self, 95 message_format: str = "{levelname}|-|{asctime}|-|{message}", 96 encoding: str = "utf-8", 97 ): 98 """:param message_format: '{' style format string""" 99 self.logger = logging.getLogger(self.dbname) 100 if not self.logger.hasHandlers(): 101 handler = logging.FileHandler( 102 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 103 ) 104 handler.setFormatter( 105 logging.Formatter( 106 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 107 ) 108 ) 109 self.logger.addHandler(handler) 110 self.logger.setLevel(logging.INFO) 111 112 def _get_dict( 113 self, table: str, values: list, columns_to_return: list[str] = None 114 ) -> dict: 115 """Converts the values of a row into a dictionary with column names as keys. 116 117 :param table: The table that values were pulled from. 118 119 :param values: List of values expected to be the same quantity 120 and in the same order as the column names of table. 121 122 :param columns_to_return: An optional list of column names. 123 If given, only these columns will be included in the returned dictionary. 124 Otherwise all columns and values are returned.""" 125 return { 126 column: value 127 for column, value in zip(self.get_column_names(table), values) 128 if not columns_to_return or column in columns_to_return 129 } 130 131 def _get_conditions( 132 self, match_criteria: list[tuple] | dict, exact_match: bool = True 133 ) -> str: 134 """Builds and returns the conditional portion of a query. 135 136 :param match_criteria: Can be a list of 2-tuples where each 137 tuple is (columnName, rowValue) or a dictionary where 138 keys are column names and values are row values. 139 140 :param exact_match: If False, the rowValue for a give column 141 will be matched as a substring. 142 143 Usage e.g.: 144 145 self.cursor.execute(f'select * from {table} where {conditions}')""" 146 if type(match_criteria) == dict: 147 match_criteria = [(k, v) for k, v in match_criteria.items()] 148 if exact_match: 149 conditions = " and ".join( 150 f'"{column_row[0]}" = "{column_row[1]}"' 151 for column_row in match_criteria 152 ) 153 else: 154 conditions = " and ".join( 155 f'"{column_row[0]}" like "%{column_row[1]}%"' 156 for column_row in match_criteria 157 ) 158 return f"({conditions})" 159 160 @_connect 161 def query(self, query_) -> list[Any]: 162 """Execute an arbitrary query and 163 return the results.""" 164 self.cursor.execute(query_) 165 return self.cursor.fetchall() 166 167 @_connect 168 def create_tables(self, table_querys: list[str] = []): 169 """Create tables if they don't exist. 170 171 :param table_querys: Each query should be 172 in the form 'tableName(columnDefinitions)'""" 173 if len(table_querys) > 0: 174 table_names = self.get_table_names() 175 for table in table_querys: 176 if table.split("(")[0].strip() not in table_names: 177 self.cursor.execute(f"create table {table}") 178 self.logger.info(f'{table.split("(")[0]} table created.') 179 180 @_connect 181 def create_table(self, table: str, column_defs: list[str]): 182 """Create a table if it doesn't exist. 183 184 :param table: Name of the table to create. 185 186 :param column_defs: List of column definitions in 187 proper Sqlite3 sytax. 188 i.e. "columnName text unique" or "columnName int primary key" etc.""" 189 if table not in self.get_table_names(): 190 query = f"create table {table}({', '.join(column_defs)})" 191 self.cursor.execute(query) 192 self.logger.info(f"'{table}' table created.") 193 194 @_connect 195 def get_table_names(self) -> list[str]: 196 """Returns a list of table names from database.""" 197 self.cursor.execute( 198 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 199 ) 200 return [result[0] for result in self.cursor.fetchall()] 201 202 @_connect 203 def get_column_names(self, table: str) -> list[str]: 204 """Return a list of column names from a table.""" 205 self.cursor.execute(f"select * from {table} where 1=0") 206 return [description[0] for description in self.cursor.description] 207 208 @_connect 209 def count( 210 self, 211 table: str, 212 match_criteria: list[tuple] | dict = None, 213 exact_match: bool = True, 214 ) -> int: 215 """Return number of items in table. 216 217 :param match_criteria: Can be a list of 2-tuples where each 218 tuple is (columnName, rowValue) or a dictionary where 219 keys are column names and values are row values. 220 If None, all rows from the table will be counted. 221 222 :param exact_match: If False, the row value for a give column 223 in match_criteria will be matched as a substring. Has no effect if 224 match_criteria is None. 225 """ 226 query = f"select count(_rowid_) from {table}" 227 try: 228 if match_criteria: 229 self.cursor.execute( 230 f"{query} where {self._get_conditions(match_criteria, exact_match)}" 231 ) 232 else: 233 self.cursor.execute(f"{query}") 234 return self.cursor.fetchone()[0] 235 except: 236 return 0 237 238 @_connect 239 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 240 """Add row of values to table. 241 242 :param table: The table to insert into. 243 244 :param values: A tuple of values to be inserted into the table. 245 246 :param columns: If None, values param is expected to supply 247 a value for every column in the table. If columns is 248 provided, it should contain the same number of elements as values.""" 249 parameterizer = ", ".join("?" for _ in values) 250 logger_values = ", ".join(str(value) for value in values) 251 try: 252 if columns: 253 columns = ", ".join(column for column in columns) 254 self.cursor.execute( 255 f"insert into {table} ({columns}) values({parameterizer})", values 256 ) 257 else: 258 self.cursor.execute( 259 f"insert into {table} values({parameterizer})", values 260 ) 261 self.logger.info(f'Added "{logger_values}" to {table} table.') 262 except Exception as e: 263 if "constraint" not in str(e).lower(): 264 self.logger.exception( 265 f'Error adding "{logger_values}" to {table} table.' 266 ) 267 else: 268 self.logger.debug(str(e)) 269 270 @_connect 271 def get_rows( 272 self, 273 table: str, 274 match_criteria: list[tuple] | dict = None, 275 exact_match: bool = True, 276 sort_by_column: str = None, 277 columns_to_return: list[str] = None, 278 return_as_dataframe: bool = False, 279 values_only: bool = False, 280 order_by: str = None, 281 limit: str | int = None, 282 ) -> list[dict] | list[tuple] | pandas.DataFrame: 283 """Returns rows from table as a list of dictionaries 284 where the key-value pairs of the dictionaries are 285 column name: row value. 286 287 :param match_criteria: Can be a list of 2-tuples where each 288 tuple is (columnName, rowValue) or a dictionary where 289 keys are column names and values are row values. 290 291 :param exact_match: If False, the rowValue for a give column 292 will be matched as a substring. 293 294 :param sort_by_column: A column name to sort the results by. 295 This will sort results in Python after retrieving them from the db. 296 Use the 'order_by' param to use SQLite engine for ordering. 297 298 :param columns_to_return: Optional list of column names. 299 If provided, the elements returned by get_rows() will 300 only contain the provided columns. Otherwise every column 301 in the row is returned. 302 303 :param return_as_dataframe: If True, 304 the results will be returned as a pandas.DataFrame object. 305 306 :param values_only: Return the results as a list of tuples 307 instead of a list of dictionaries that have column names as keys. 308 The results will still be sorted according to sort_by_column if 309 one is provided. 310 311 :param order_by: If given, a 'order by {order_by}' clause 312 will be added to the select query. 313 314 :param limit: If given, a 'limit {limit}' clause will be 315 added to the select query. 316 """ 317 318 if type(columns_to_return) is str: 319 columns_to_return = [columns_to_return] 320 query = f"select * from {table}" 321 matches = [] 322 if match_criteria: 323 query += f" where {self._get_conditions(match_criteria, exact_match)}" 324 if order_by: 325 query += f" order by {order_by}" 326 if limit: 327 query += f" limit {limit}" 328 query += ";" 329 self.cursor.execute(query) 330 matches = self.cursor.fetchall() 331 results = [self._get_dict(table, match, columns_to_return) for match in matches] 332 if sort_by_column: 333 results = sorted(results, key=lambda x: x[sort_by_column]) 334 if return_as_dataframe: 335 return pandas.DataFrame(results) 336 if values_only: 337 return [tuple(row.values()) for row in results] 338 else: 339 return results 340 341 @_connect 342 def find( 343 self, table: str, query_string: str, columns: list[str] = None 344 ) -> tuple[dict]: 345 """Search for rows that contain query_string as a substring 346 of any column. 347 348 :param table: The table to search. 349 350 :param query_string: The substring to search for in all columns. 351 352 :param columns: A list of columns to search for query_string. 353 If None, all columns in the table will be searched. 354 """ 355 if type(columns) is str: 356 columns = [columns] 357 results = [] 358 if not columns: 359 columns = self.get_column_names(table) 360 for column in columns: 361 results.extend( 362 [ 363 row 364 for row in self.get_rows( 365 table, [(column, query_string)], exact_match=False 366 ) 367 if row not in results 368 ] 369 ) 370 return results 371 372 @_connect 373 def delete( 374 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 375 ) -> int: 376 """Delete records from table. 377 378 Returns number of deleted records. 379 380 :param match_criteria: Can be a list of 2-tuples where each 381 tuple is (columnName, rowValue) or a dictionary where 382 keys are column names and values are row values. 383 384 :param exact_match: If False, the rowValue for a give column 385 will be matched as a substring. 386 """ 387 num_matches = self.count(table, match_criteria, exact_match) 388 conditions = self._get_conditions(match_criteria, exact_match) 389 try: 390 self.cursor.execute(f"delete from {table} where {conditions}") 391 self.logger.info( 392 f'Deleted {num_matches} from "{table}" where {conditions}".' 393 ) 394 return num_matches 395 except Exception as e: 396 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 397 return 0 398 399 @_connect 400 def update( 401 self, 402 table: str, 403 column_to_update: str, 404 new_value: Any, 405 match_criteria: list[tuple] | dict = None, 406 ) -> bool: 407 """Update row value for entry matched with match_criteria. 408 409 :param column_to_update: The column to be updated in the matched row. 410 411 :param new_value: The new value to insert. 412 413 :param match_criteria: Can be a list of 2-tuples where each 414 tuple is (columnName, rowValue) or a dictionary where 415 keys are column names and values are row values. 416 If None, every row will be updated. 417 418 Returns True if successful, False if not.""" 419 query = f"update {table} set {column_to_update} = ?" 420 if match_criteria: 421 if self.count(table, match_criteria) == 0: 422 self.logger.info( 423 f"Couldn't find matching records in {table} table to update to '{new_value}'" 424 ) 425 return False 426 conditions = self._get_conditions(match_criteria) 427 query += f" where {conditions}" 428 else: 429 conditions = None 430 try: 431 self.cursor.execute( 432 query, 433 (new_value,), 434 ) 435 self.logger.info( 436 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 437 ) 438 return True 439 except UnboundLocalError: 440 table_filter_string = "\n".join( 441 table_filter for table_filter in match_criteria 442 ) 443 self.logger.error( 444 f"No records found matching filters: {table_filter_string}" 445 ) 446 return False 447 except Exception as e: 448 self.logger.error( 449 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 450 ) 451 return False 452 453 @_connect 454 def drop_table(self, table: str) -> bool: 455 """Drop a table from the database. 456 457 Returns True if successful, False if not.""" 458 try: 459 self.cursor.execute(f"drop Table {table}") 460 self.logger.info(f'Dropped table "{table}"') 461 except Exception as e: 462 print(e) 463 self.logger.error(f'Failed to drop table "{table}"') 464 465 @_connect 466 def add_column( 467 self, table: str, column: str, _type: str, default_value: str = None 468 ): 469 """Add a new column to table. 470 471 :param column: Name of the column to add. 472 473 :param _type: The data type of the new column. 474 475 :param default_value: Optional default value for the column.""" 476 try: 477 if default_value: 478 self.cursor.execute( 479 f"alter table {table} add column {column} {_type} default {default_value}" 480 ) 481 else: 482 self.cursor.execute(f"alter table {table} add column {column} {_type}") 483 self.logger.info(f'Added column "{column}" to "{table}" table.') 484 except Exception as e: 485 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 486 487 @staticmethod 488 def data_to_string( 489 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 490 ) -> str: 491 """Uses tabulate to produce pretty string output 492 from a list of dictionaries. 493 494 :param data: Assumes all dictionaries in list have the same set of keys. 495 496 :param sort_key: Optional dictionary key to sort data with. 497 498 :param wrap_to_terminal: If True, the table width will be wrapped 499 to fit within the current terminal window. Set to False 500 if the output is going into something like a txt file.""" 501 return data_to_string(data, sort_key, wrap_to_terminal) 502 503 504def data_to_string( 505 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 506) -> str: 507 """Uses tabulate to produce pretty string output 508 from a list of dictionaries. 509 510 :param data: Assumes all dictionaries in list have the same set of keys. 511 512 :param sort_key: Optional dictionary key to sort data with. 513 514 :param wrap_to_terminal: If True, the table width will be wrapped 515 to fit within the current terminal window. Set to False 516 if the output is going into something like a txt file.""" 517 if len(data) == 0: 518 return "" 519 if sort_key: 520 data = sorted(data, key=lambda d: d[sort_key]) 521 for i, d in enumerate(data): 522 for k in d: 523 data[i][k] = str(data[i][k]) 524 if wrap_to_terminal: 525 terminal_width = os.get_terminal_size().columns 526 max_col_widths = terminal_width 527 """ Reducing the column width by tabulating one row at a time 528 and then reducing further by tabulating the whole set proved to be 529 faster than going straight to tabulating the whole set and reducing 530 the column width.""" 531 too_wide = True 532 while too_wide and max_col_widths > 1: 533 for i, row in enumerate(data): 534 output = tabulate( 535 [row], 536 headers="keys", 537 disable_numparse=True, 538 tablefmt="grid", 539 maxcolwidths=max_col_widths, 540 ) 541 if output.index("\n") > terminal_width: 542 max_col_widths -= 2 543 too_wide = True 544 break 545 too_wide = False 546 else: 547 max_col_widths = None 548 output = tabulate( 549 data, 550 headers="keys", 551 disable_numparse=True, 552 tablefmt="grid", 553 maxcolwidths=max_col_widths, 554 ) 555 # trim max column width until the output string is less wide than the current terminal width. 556 if wrap_to_terminal: 557 while output.index("\n") > terminal_width and max_col_widths > 1: 558 max_col_widths -= 2 559 max_col_widths = max(1, max_col_widths) 560 output = tabulate( 561 data, 562 headers="keys", 563 disable_numparse=True, 564 tablefmt="grid", 565 maxcolwidths=max_col_widths, 566 ) 567 return output
28class DataBased: 29 """Sqli wrapper so queries don't need to be written except table definitions. 30 31 Supports saving and reading dates as datetime objects. 32 33 Supports using a context manager.""" 34 35 def __init__( 36 self, 37 dbpath: str | Path, 38 logger_encoding: str = "utf-8", 39 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 40 ): 41 """ 42 :param dbpath: String or Path object to database file. 43 If a relative path is given, it will be relative to the 44 current working directory. The log file will be saved to the 45 same directory. 46 47 :param logger_message_format: '{' style format string 48 for the logger object.""" 49 self.dbpath = Path(dbpath) 50 self.dbname = Path(dbpath).name 51 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 52 self._logger_init( 53 encoding=logger_encoding, message_format=logger_message_format 54 ) 55 self.connection_open = False 56 self.create_manager() 57 58 def __enter__(self): 59 self.open() 60 return self 61 62 def __exit__(self, exception_type, exception_value, exception_traceback): 63 self.close() 64 65 def create_manager(self): 66 """Create dbManager.py in the same directory 67 as the database file if it doesn't exist.""" 68 manager_template = Path(__file__).parent / "dbmanager.py" 69 manager_path = self.dbpath.parent / "dbmanager.py" 70 if not manager_path.exists(): 71 shutil.copyfile(manager_template, manager_path) 72 73 def open(self): 74 """Open connection to db.""" 75 self.connection = sqlite3.connect( 76 self.dbpath, 77 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 78 timeout=10, 79 ) 80 self.connection.execute("pragma foreign_keys = 1") 81 self.cursor = self.connection.cursor() 82 self.connection_open = True 83 84 def close(self): 85 """Save and close connection to db. 86 87 Call this as soon as you are done using the database if you have 88 multiple threads or processes using the same database.""" 89 if self.connection_open: 90 self.connection.commit() 91 self.connection.close() 92 self.connection_open = False 93 94 def _logger_init( 95 self, 96 message_format: str = "{levelname}|-|{asctime}|-|{message}", 97 encoding: str = "utf-8", 98 ): 99 """:param message_format: '{' style format string""" 100 self.logger = logging.getLogger(self.dbname) 101 if not self.logger.hasHandlers(): 102 handler = logging.FileHandler( 103 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 104 ) 105 handler.setFormatter( 106 logging.Formatter( 107 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 108 ) 109 ) 110 self.logger.addHandler(handler) 111 self.logger.setLevel(logging.INFO) 112 113 def _get_dict( 114 self, table: str, values: list, columns_to_return: list[str] = None 115 ) -> dict: 116 """Converts the values of a row into a dictionary with column names as keys. 117 118 :param table: The table that values were pulled from. 119 120 :param values: List of values expected to be the same quantity 121 and in the same order as the column names of table. 122 123 :param columns_to_return: An optional list of column names. 124 If given, only these columns will be included in the returned dictionary. 125 Otherwise all columns and values are returned.""" 126 return { 127 column: value 128 for column, value in zip(self.get_column_names(table), values) 129 if not columns_to_return or column in columns_to_return 130 } 131 132 def _get_conditions( 133 self, match_criteria: list[tuple] | dict, exact_match: bool = True 134 ) -> str: 135 """Builds and returns the conditional portion of a query. 136 137 :param match_criteria: Can be a list of 2-tuples where each 138 tuple is (columnName, rowValue) or a dictionary where 139 keys are column names and values are row values. 140 141 :param exact_match: If False, the rowValue for a give column 142 will be matched as a substring. 143 144 Usage e.g.: 145 146 self.cursor.execute(f'select * from {table} where {conditions}')""" 147 if type(match_criteria) == dict: 148 match_criteria = [(k, v) for k, v in match_criteria.items()] 149 if exact_match: 150 conditions = " and ".join( 151 f'"{column_row[0]}" = "{column_row[1]}"' 152 for column_row in match_criteria 153 ) 154 else: 155 conditions = " and ".join( 156 f'"{column_row[0]}" like "%{column_row[1]}%"' 157 for column_row in match_criteria 158 ) 159 return f"({conditions})" 160 161 @_connect 162 def query(self, query_) -> list[Any]: 163 """Execute an arbitrary query and 164 return the results.""" 165 self.cursor.execute(query_) 166 return self.cursor.fetchall() 167 168 @_connect 169 def create_tables(self, table_querys: list[str] = []): 170 """Create tables if they don't exist. 171 172 :param table_querys: Each query should be 173 in the form 'tableName(columnDefinitions)'""" 174 if len(table_querys) > 0: 175 table_names = self.get_table_names() 176 for table in table_querys: 177 if table.split("(")[0].strip() not in table_names: 178 self.cursor.execute(f"create table {table}") 179 self.logger.info(f'{table.split("(")[0]} table created.') 180 181 @_connect 182 def create_table(self, table: str, column_defs: list[str]): 183 """Create a table if it doesn't exist. 184 185 :param table: Name of the table to create. 186 187 :param column_defs: List of column definitions in 188 proper Sqlite3 sytax. 189 i.e. "columnName text unique" or "columnName int primary key" etc.""" 190 if table not in self.get_table_names(): 191 query = f"create table {table}({', '.join(column_defs)})" 192 self.cursor.execute(query) 193 self.logger.info(f"'{table}' table created.") 194 195 @_connect 196 def get_table_names(self) -> list[str]: 197 """Returns a list of table names from database.""" 198 self.cursor.execute( 199 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 200 ) 201 return [result[0] for result in self.cursor.fetchall()] 202 203 @_connect 204 def get_column_names(self, table: str) -> list[str]: 205 """Return a list of column names from a table.""" 206 self.cursor.execute(f"select * from {table} where 1=0") 207 return [description[0] for description in self.cursor.description] 208 209 @_connect 210 def count( 211 self, 212 table: str, 213 match_criteria: list[tuple] | dict = None, 214 exact_match: bool = True, 215 ) -> int: 216 """Return number of items in table. 217 218 :param match_criteria: Can be a list of 2-tuples where each 219 tuple is (columnName, rowValue) or a dictionary where 220 keys are column names and values are row values. 221 If None, all rows from the table will be counted. 222 223 :param exact_match: If False, the row value for a give column 224 in match_criteria will be matched as a substring. Has no effect if 225 match_criteria is None. 226 """ 227 query = f"select count(_rowid_) from {table}" 228 try: 229 if match_criteria: 230 self.cursor.execute( 231 f"{query} where {self._get_conditions(match_criteria, exact_match)}" 232 ) 233 else: 234 self.cursor.execute(f"{query}") 235 return self.cursor.fetchone()[0] 236 except: 237 return 0 238 239 @_connect 240 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 241 """Add row of values to table. 242 243 :param table: The table to insert into. 244 245 :param values: A tuple of values to be inserted into the table. 246 247 :param columns: If None, values param is expected to supply 248 a value for every column in the table. If columns is 249 provided, it should contain the same number of elements as values.""" 250 parameterizer = ", ".join("?" for _ in values) 251 logger_values = ", ".join(str(value) for value in values) 252 try: 253 if columns: 254 columns = ", ".join(column for column in columns) 255 self.cursor.execute( 256 f"insert into {table} ({columns}) values({parameterizer})", values 257 ) 258 else: 259 self.cursor.execute( 260 f"insert into {table} values({parameterizer})", values 261 ) 262 self.logger.info(f'Added "{logger_values}" to {table} table.') 263 except Exception as e: 264 if "constraint" not in str(e).lower(): 265 self.logger.exception( 266 f'Error adding "{logger_values}" to {table} table.' 267 ) 268 else: 269 self.logger.debug(str(e)) 270 271 @_connect 272 def get_rows( 273 self, 274 table: str, 275 match_criteria: list[tuple] | dict = None, 276 exact_match: bool = True, 277 sort_by_column: str = None, 278 columns_to_return: list[str] = None, 279 return_as_dataframe: bool = False, 280 values_only: bool = False, 281 order_by: str = None, 282 limit: str | int = None, 283 ) -> list[dict] | list[tuple] | pandas.DataFrame: 284 """Returns rows from table as a list of dictionaries 285 where the key-value pairs of the dictionaries are 286 column name: row value. 287 288 :param match_criteria: Can be a list of 2-tuples where each 289 tuple is (columnName, rowValue) or a dictionary where 290 keys are column names and values are row values. 291 292 :param exact_match: If False, the rowValue for a give column 293 will be matched as a substring. 294 295 :param sort_by_column: A column name to sort the results by. 296 This will sort results in Python after retrieving them from the db. 297 Use the 'order_by' param to use SQLite engine for ordering. 298 299 :param columns_to_return: Optional list of column names. 300 If provided, the elements returned by get_rows() will 301 only contain the provided columns. Otherwise every column 302 in the row is returned. 303 304 :param return_as_dataframe: If True, 305 the results will be returned as a pandas.DataFrame object. 306 307 :param values_only: Return the results as a list of tuples 308 instead of a list of dictionaries that have column names as keys. 309 The results will still be sorted according to sort_by_column if 310 one is provided. 311 312 :param order_by: If given, a 'order by {order_by}' clause 313 will be added to the select query. 314 315 :param limit: If given, a 'limit {limit}' clause will be 316 added to the select query. 317 """ 318 319 if type(columns_to_return) is str: 320 columns_to_return = [columns_to_return] 321 query = f"select * from {table}" 322 matches = [] 323 if match_criteria: 324 query += f" where {self._get_conditions(match_criteria, exact_match)}" 325 if order_by: 326 query += f" order by {order_by}" 327 if limit: 328 query += f" limit {limit}" 329 query += ";" 330 self.cursor.execute(query) 331 matches = self.cursor.fetchall() 332 results = [self._get_dict(table, match, columns_to_return) for match in matches] 333 if sort_by_column: 334 results = sorted(results, key=lambda x: x[sort_by_column]) 335 if return_as_dataframe: 336 return pandas.DataFrame(results) 337 if values_only: 338 return [tuple(row.values()) for row in results] 339 else: 340 return results 341 342 @_connect 343 def find( 344 self, table: str, query_string: str, columns: list[str] = None 345 ) -> tuple[dict]: 346 """Search for rows that contain query_string as a substring 347 of any column. 348 349 :param table: The table to search. 350 351 :param query_string: The substring to search for in all columns. 352 353 :param columns: A list of columns to search for query_string. 354 If None, all columns in the table will be searched. 355 """ 356 if type(columns) is str: 357 columns = [columns] 358 results = [] 359 if not columns: 360 columns = self.get_column_names(table) 361 for column in columns: 362 results.extend( 363 [ 364 row 365 for row in self.get_rows( 366 table, [(column, query_string)], exact_match=False 367 ) 368 if row not in results 369 ] 370 ) 371 return results 372 373 @_connect 374 def delete( 375 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 376 ) -> int: 377 """Delete records from table. 378 379 Returns number of deleted records. 380 381 :param match_criteria: Can be a list of 2-tuples where each 382 tuple is (columnName, rowValue) or a dictionary where 383 keys are column names and values are row values. 384 385 :param exact_match: If False, the rowValue for a give column 386 will be matched as a substring. 387 """ 388 num_matches = self.count(table, match_criteria, exact_match) 389 conditions = self._get_conditions(match_criteria, exact_match) 390 try: 391 self.cursor.execute(f"delete from {table} where {conditions}") 392 self.logger.info( 393 f'Deleted {num_matches} from "{table}" where {conditions}".' 394 ) 395 return num_matches 396 except Exception as e: 397 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 398 return 0 399 400 @_connect 401 def update( 402 self, 403 table: str, 404 column_to_update: str, 405 new_value: Any, 406 match_criteria: list[tuple] | dict = None, 407 ) -> bool: 408 """Update row value for entry matched with match_criteria. 409 410 :param column_to_update: The column to be updated in the matched row. 411 412 :param new_value: The new value to insert. 413 414 :param match_criteria: Can be a list of 2-tuples where each 415 tuple is (columnName, rowValue) or a dictionary where 416 keys are column names and values are row values. 417 If None, every row will be updated. 418 419 Returns True if successful, False if not.""" 420 query = f"update {table} set {column_to_update} = ?" 421 if match_criteria: 422 if self.count(table, match_criteria) == 0: 423 self.logger.info( 424 f"Couldn't find matching records in {table} table to update to '{new_value}'" 425 ) 426 return False 427 conditions = self._get_conditions(match_criteria) 428 query += f" where {conditions}" 429 else: 430 conditions = None 431 try: 432 self.cursor.execute( 433 query, 434 (new_value,), 435 ) 436 self.logger.info( 437 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 438 ) 439 return True 440 except UnboundLocalError: 441 table_filter_string = "\n".join( 442 table_filter for table_filter in match_criteria 443 ) 444 self.logger.error( 445 f"No records found matching filters: {table_filter_string}" 446 ) 447 return False 448 except Exception as e: 449 self.logger.error( 450 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 451 ) 452 return False 453 454 @_connect 455 def drop_table(self, table: str) -> bool: 456 """Drop a table from the database. 457 458 Returns True if successful, False if not.""" 459 try: 460 self.cursor.execute(f"drop Table {table}") 461 self.logger.info(f'Dropped table "{table}"') 462 except Exception as e: 463 print(e) 464 self.logger.error(f'Failed to drop table "{table}"') 465 466 @_connect 467 def add_column( 468 self, table: str, column: str, _type: str, default_value: str = None 469 ): 470 """Add a new column to table. 471 472 :param column: Name of the column to add. 473 474 :param _type: The data type of the new column. 475 476 :param default_value: Optional default value for the column.""" 477 try: 478 if default_value: 479 self.cursor.execute( 480 f"alter table {table} add column {column} {_type} default {default_value}" 481 ) 482 else: 483 self.cursor.execute(f"alter table {table} add column {column} {_type}") 484 self.logger.info(f'Added column "{column}" to "{table}" table.') 485 except Exception as e: 486 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 487 488 @staticmethod 489 def data_to_string( 490 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 491 ) -> str: 492 """Uses tabulate to produce pretty string output 493 from a list of dictionaries. 494 495 :param data: Assumes all dictionaries in list have the same set of keys. 496 497 :param sort_key: Optional dictionary key to sort data with. 498 499 :param wrap_to_terminal: If True, the table width will be wrapped 500 to fit within the current terminal window. Set to False 501 if the output is going into something like a txt file.""" 502 return data_to_string(data, sort_key, wrap_to_terminal)
Sqli wrapper so queries don't need to be written except table definitions.
Supports saving and reading dates as datetime objects.
Supports using a context manager.
35 def __init__( 36 self, 37 dbpath: str | Path, 38 logger_encoding: str = "utf-8", 39 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 40 ): 41 """ 42 :param dbpath: String or Path object to database file. 43 If a relative path is given, it will be relative to the 44 current working directory. The log file will be saved to the 45 same directory. 46 47 :param logger_message_format: '{' style format string 48 for the logger object.""" 49 self.dbpath = Path(dbpath) 50 self.dbname = Path(dbpath).name 51 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 52 self._logger_init( 53 encoding=logger_encoding, message_format=logger_message_format 54 ) 55 self.connection_open = False 56 self.create_manager()
Parameters
dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.
logger_message_format: '{' style format string for the logger object.
65 def create_manager(self): 66 """Create dbManager.py in the same directory 67 as the database file if it doesn't exist.""" 68 manager_template = Path(__file__).parent / "dbmanager.py" 69 manager_path = self.dbpath.parent / "dbmanager.py" 70 if not manager_path.exists(): 71 shutil.copyfile(manager_template, manager_path)
Create dbManager.py in the same directory as the database file if it doesn't exist.
73 def open(self): 74 """Open connection to db.""" 75 self.connection = sqlite3.connect( 76 self.dbpath, 77 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 78 timeout=10, 79 ) 80 self.connection.execute("pragma foreign_keys = 1") 81 self.cursor = self.connection.cursor() 82 self.connection_open = True
Open connection to db.
84 def close(self): 85 """Save and close connection to db. 86 87 Call this as soon as you are done using the database if you have 88 multiple threads or processes using the same database.""" 89 if self.connection_open: 90 self.connection.commit() 91 self.connection.close() 92 self.connection_open = False
Save and close connection to db.
Call this as soon as you are done using the database if you have multiple threads or processes using the same database.
161 @_connect 162 def query(self, query_) -> list[Any]: 163 """Execute an arbitrary query and 164 return the results.""" 165 self.cursor.execute(query_) 166 return self.cursor.fetchall()
Execute an arbitrary query and return the results.
168 @_connect 169 def create_tables(self, table_querys: list[str] = []): 170 """Create tables if they don't exist. 171 172 :param table_querys: Each query should be 173 in the form 'tableName(columnDefinitions)'""" 174 if len(table_querys) > 0: 175 table_names = self.get_table_names() 176 for table in table_querys: 177 if table.split("(")[0].strip() not in table_names: 178 self.cursor.execute(f"create table {table}") 179 self.logger.info(f'{table.split("(")[0]} table created.')
Create tables if they don't exist.
Parameters
- table_querys: Each query should be in the form 'tableName(columnDefinitions)'
181 @_connect 182 def create_table(self, table: str, column_defs: list[str]): 183 """Create a table if it doesn't exist. 184 185 :param table: Name of the table to create. 186 187 :param column_defs: List of column definitions in 188 proper Sqlite3 sytax. 189 i.e. "columnName text unique" or "columnName int primary key" etc.""" 190 if table not in self.get_table_names(): 191 query = f"create table {table}({', '.join(column_defs)})" 192 self.cursor.execute(query) 193 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
Parameters
table: Name of the table to create.
column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.
195 @_connect 196 def get_table_names(self) -> list[str]: 197 """Returns a list of table names from database.""" 198 self.cursor.execute( 199 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 200 ) 201 return [result[0] for result in self.cursor.fetchall()]
Returns a list of table names from database.
203 @_connect 204 def get_column_names(self, table: str) -> list[str]: 205 """Return a list of column names from a table.""" 206 self.cursor.execute(f"select * from {table} where 1=0") 207 return [description[0] for description in self.cursor.description]
Return a list of column names from a table.
209 @_connect 210 def count( 211 self, 212 table: str, 213 match_criteria: list[tuple] | dict = None, 214 exact_match: bool = True, 215 ) -> int: 216 """Return number of items in table. 217 218 :param match_criteria: Can be a list of 2-tuples where each 219 tuple is (columnName, rowValue) or a dictionary where 220 keys are column names and values are row values. 221 If None, all rows from the table will be counted. 222 223 :param exact_match: If False, the row value for a give column 224 in match_criteria will be matched as a substring. Has no effect if 225 match_criteria is None. 226 """ 227 query = f"select count(_rowid_) from {table}" 228 try: 229 if match_criteria: 230 self.cursor.execute( 231 f"{query} where {self._get_conditions(match_criteria, exact_match)}" 232 ) 233 else: 234 self.cursor.execute(f"{query}") 235 return self.cursor.fetchone()[0] 236 except: 237 return 0
Return number of items in table.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.
exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.
239 @_connect 240 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 241 """Add row of values to table. 242 243 :param table: The table to insert into. 244 245 :param values: A tuple of values to be inserted into the table. 246 247 :param columns: If None, values param is expected to supply 248 a value for every column in the table. If columns is 249 provided, it should contain the same number of elements as values.""" 250 parameterizer = ", ".join("?" for _ in values) 251 logger_values = ", ".join(str(value) for value in values) 252 try: 253 if columns: 254 columns = ", ".join(column for column in columns) 255 self.cursor.execute( 256 f"insert into {table} ({columns}) values({parameterizer})", values 257 ) 258 else: 259 self.cursor.execute( 260 f"insert into {table} values({parameterizer})", values 261 ) 262 self.logger.info(f'Added "{logger_values}" to {table} table.') 263 except Exception as e: 264 if "constraint" not in str(e).lower(): 265 self.logger.exception( 266 f'Error adding "{logger_values}" to {table} table.' 267 ) 268 else: 269 self.logger.debug(str(e))
Add row of values to table.
Parameters
table: The table to insert into.
values: A tuple of values to be inserted into the table.
columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.
271 @_connect 272 def get_rows( 273 self, 274 table: str, 275 match_criteria: list[tuple] | dict = None, 276 exact_match: bool = True, 277 sort_by_column: str = None, 278 columns_to_return: list[str] = None, 279 return_as_dataframe: bool = False, 280 values_only: bool = False, 281 order_by: str = None, 282 limit: str | int = None, 283 ) -> list[dict] | list[tuple] | pandas.DataFrame: 284 """Returns rows from table as a list of dictionaries 285 where the key-value pairs of the dictionaries are 286 column name: row value. 287 288 :param match_criteria: Can be a list of 2-tuples where each 289 tuple is (columnName, rowValue) or a dictionary where 290 keys are column names and values are row values. 291 292 :param exact_match: If False, the rowValue for a give column 293 will be matched as a substring. 294 295 :param sort_by_column: A column name to sort the results by. 296 This will sort results in Python after retrieving them from the db. 297 Use the 'order_by' param to use SQLite engine for ordering. 298 299 :param columns_to_return: Optional list of column names. 300 If provided, the elements returned by get_rows() will 301 only contain the provided columns. Otherwise every column 302 in the row is returned. 303 304 :param return_as_dataframe: If True, 305 the results will be returned as a pandas.DataFrame object. 306 307 :param values_only: Return the results as a list of tuples 308 instead of a list of dictionaries that have column names as keys. 309 The results will still be sorted according to sort_by_column if 310 one is provided. 311 312 :param order_by: If given, a 'order by {order_by}' clause 313 will be added to the select query. 314 315 :param limit: If given, a 'limit {limit}' clause will be 316 added to the select query. 317 """ 318 319 if type(columns_to_return) is str: 320 columns_to_return = [columns_to_return] 321 query = f"select * from {table}" 322 matches = [] 323 if match_criteria: 324 query += f" where {self._get_conditions(match_criteria, exact_match)}" 325 if order_by: 326 query += f" order by {order_by}" 327 if limit: 328 query += f" limit {limit}" 329 query += ";" 330 self.cursor.execute(query) 331 matches = self.cursor.fetchall() 332 results = [self._get_dict(table, match, columns_to_return) for match in matches] 333 if sort_by_column: 334 results = sorted(results, key=lambda x: x[sort_by_column]) 335 if return_as_dataframe: 336 return pandas.DataFrame(results) 337 if values_only: 338 return [tuple(row.values()) for row in results] 339 else: 340 return results
Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.
columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.
return_as_dataframe: If True, the results will be returned as a pandas.DataFrame object.
values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.
order_by: If given, a 'order by {order_by}' clause will be added to the select query.
limit: If given, a 'limit {limit}' clause will be added to the select query.
342 @_connect 343 def find( 344 self, table: str, query_string: str, columns: list[str] = None 345 ) -> tuple[dict]: 346 """Search for rows that contain query_string as a substring 347 of any column. 348 349 :param table: The table to search. 350 351 :param query_string: The substring to search for in all columns. 352 353 :param columns: A list of columns to search for query_string. 354 If None, all columns in the table will be searched. 355 """ 356 if type(columns) is str: 357 columns = [columns] 358 results = [] 359 if not columns: 360 columns = self.get_column_names(table) 361 for column in columns: 362 results.extend( 363 [ 364 row 365 for row in self.get_rows( 366 table, [(column, query_string)], exact_match=False 367 ) 368 if row not in results 369 ] 370 ) 371 return results
Search for rows that contain query_string as a substring of any column.
Parameters
table: The table to search.
query_string: The substring to search for in all columns.
columns: A list of columns to search for query_string. If None, all columns in the table will be searched.
373 @_connect 374 def delete( 375 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 376 ) -> int: 377 """Delete records from table. 378 379 Returns number of deleted records. 380 381 :param match_criteria: Can be a list of 2-tuples where each 382 tuple is (columnName, rowValue) or a dictionary where 383 keys are column names and values are row values. 384 385 :param exact_match: If False, the rowValue for a give column 386 will be matched as a substring. 387 """ 388 num_matches = self.count(table, match_criteria, exact_match) 389 conditions = self._get_conditions(match_criteria, exact_match) 390 try: 391 self.cursor.execute(f"delete from {table} where {conditions}") 392 self.logger.info( 393 f'Deleted {num_matches} from "{table}" where {conditions}".' 394 ) 395 return num_matches 396 except Exception as e: 397 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 398 return 0
Delete records from table.
Returns number of deleted records.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
400 @_connect 401 def update( 402 self, 403 table: str, 404 column_to_update: str, 405 new_value: Any, 406 match_criteria: list[tuple] | dict = None, 407 ) -> bool: 408 """Update row value for entry matched with match_criteria. 409 410 :param column_to_update: The column to be updated in the matched row. 411 412 :param new_value: The new value to insert. 413 414 :param match_criteria: Can be a list of 2-tuples where each 415 tuple is (columnName, rowValue) or a dictionary where 416 keys are column names and values are row values. 417 If None, every row will be updated. 418 419 Returns True if successful, False if not.""" 420 query = f"update {table} set {column_to_update} = ?" 421 if match_criteria: 422 if self.count(table, match_criteria) == 0: 423 self.logger.info( 424 f"Couldn't find matching records in {table} table to update to '{new_value}'" 425 ) 426 return False 427 conditions = self._get_conditions(match_criteria) 428 query += f" where {conditions}" 429 else: 430 conditions = None 431 try: 432 self.cursor.execute( 433 query, 434 (new_value,), 435 ) 436 self.logger.info( 437 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 438 ) 439 return True 440 except UnboundLocalError: 441 table_filter_string = "\n".join( 442 table_filter for table_filter in match_criteria 443 ) 444 self.logger.error( 445 f"No records found matching filters: {table_filter_string}" 446 ) 447 return False 448 except Exception as e: 449 self.logger.error( 450 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 451 ) 452 return False
Update row value for entry matched with match_criteria.
Parameters
column_to_update: The column to be updated in the matched row.
new_value: The new value to insert.
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.
Returns True if successful, False if not.
454 @_connect 455 def drop_table(self, table: str) -> bool: 456 """Drop a table from the database. 457 458 Returns True if successful, False if not.""" 459 try: 460 self.cursor.execute(f"drop Table {table}") 461 self.logger.info(f'Dropped table "{table}"') 462 except Exception as e: 463 print(e) 464 self.logger.error(f'Failed to drop table "{table}"')
Drop a table from the database.
Returns True if successful, False if not.
466 @_connect 467 def add_column( 468 self, table: str, column: str, _type: str, default_value: str = None 469 ): 470 """Add a new column to table. 471 472 :param column: Name of the column to add. 473 474 :param _type: The data type of the new column. 475 476 :param default_value: Optional default value for the column.""" 477 try: 478 if default_value: 479 self.cursor.execute( 480 f"alter table {table} add column {column} {_type} default {default_value}" 481 ) 482 else: 483 self.cursor.execute(f"alter table {table} add column {column} {_type}") 484 self.logger.info(f'Added column "{column}" to "{table}" table.') 485 except Exception as e: 486 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Add a new column to table.
Parameters
column: Name of the column to add.
_type: The data type of the new column.
default_value: Optional default value for the column.
488 @staticmethod 489 def data_to_string( 490 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 491 ) -> str: 492 """Uses tabulate to produce pretty string output 493 from a list of dictionaries. 494 495 :param data: Assumes all dictionaries in list have the same set of keys. 496 497 :param sort_key: Optional dictionary key to sort data with. 498 499 :param wrap_to_terminal: If True, the table width will be wrapped 500 to fit within the current terminal window. Set to False 501 if the output is going into something like a txt file.""" 502 return data_to_string(data, sort_key, wrap_to_terminal)
Uses tabulate to produce pretty string output from a list of dictionaries.
Parameters
data: Assumes all dictionaries in list have the same set of keys.
sort_key: Optional dictionary key to sort data with.
wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.
505def data_to_string( 506 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 507) -> str: 508 """Uses tabulate to produce pretty string output 509 from a list of dictionaries. 510 511 :param data: Assumes all dictionaries in list have the same set of keys. 512 513 :param sort_key: Optional dictionary key to sort data with. 514 515 :param wrap_to_terminal: If True, the table width will be wrapped 516 to fit within the current terminal window. Set to False 517 if the output is going into something like a txt file.""" 518 if len(data) == 0: 519 return "" 520 if sort_key: 521 data = sorted(data, key=lambda d: d[sort_key]) 522 for i, d in enumerate(data): 523 for k in d: 524 data[i][k] = str(data[i][k]) 525 if wrap_to_terminal: 526 terminal_width = os.get_terminal_size().columns 527 max_col_widths = terminal_width 528 """ Reducing the column width by tabulating one row at a time 529 and then reducing further by tabulating the whole set proved to be 530 faster than going straight to tabulating the whole set and reducing 531 the column width.""" 532 too_wide = True 533 while too_wide and max_col_widths > 1: 534 for i, row in enumerate(data): 535 output = tabulate( 536 [row], 537 headers="keys", 538 disable_numparse=True, 539 tablefmt="grid", 540 maxcolwidths=max_col_widths, 541 ) 542 if output.index("\n") > terminal_width: 543 max_col_widths -= 2 544 too_wide = True 545 break 546 too_wide = False 547 else: 548 max_col_widths = None 549 output = tabulate( 550 data, 551 headers="keys", 552 disable_numparse=True, 553 tablefmt="grid", 554 maxcolwidths=max_col_widths, 555 ) 556 # trim max column width until the output string is less wide than the current terminal width. 557 if wrap_to_terminal: 558 while output.index("\n") > terminal_width and max_col_widths > 1: 559 max_col_widths -= 2 560 max_col_widths = max(1, max_col_widths) 561 output = tabulate( 562 data, 563 headers="keys", 564 disable_numparse=True, 565 tablefmt="grid", 566 maxcolwidths=max_col_widths, 567 ) 568 return output
Uses tabulate to produce pretty string output from a list of dictionaries.
Parameters
data: Assumes all dictionaries in list have the same set of keys.
sort_key: Optional dictionary key to sort data with.
wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.