databased.databased
1import logging 2import os 3import sqlite3 4from datetime import datetime 5from functools import wraps 6from pathlib import Path 7from typing import Any 8 9import pandas 10from tabulate import tabulate 11 12 13def _connect(func): 14 """Decorator to open db connection if it isn't already open.""" 15 16 @wraps(func) 17 def inner(self, *args, **kwargs): 18 if not self.connection_open: 19 self.open() 20 results = func(self, *args, **kwargs) 21 return results 22 23 return inner 24 25 26class DataBased: 27 """Sqli wrapper so queries don't need to be written except table definitions. 28 29 Supports saving and reading dates as datetime objects. 30 31 Supports using a context manager.""" 32 33 def __init__( 34 self, 35 dbpath: str | Path, 36 logger_encoding: str = "utf-8", 37 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 38 ): 39 """ 40 :param dbpath: String or Path object to database file. 41 If a relative path is given, it will be relative to the 42 current working directory. The log file will be saved to the 43 same directory. 44 45 :param logger_message_format: '{' style format string 46 for the logger object.""" 47 self.dbpath = Path(dbpath) 48 self.dbname = Path(dbpath).name 49 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 50 self._logger_init( 51 encoding=logger_encoding, message_format=logger_message_format 52 ) 53 self.connection_open = False 54 self.create_manager() 55 56 def __enter__(self): 57 self.open() 58 return self 59 60 def __exit__(self, exception_type, exception_value, exception_traceback): 61 self.close() 62 63 def create_manager(self): 64 """Create dbmanager.py in the same directory 65 as the database file if it doesn't exist.""" 66 manager_path = self.dbpath.parent / "dbmanager.py" 67 if not manager_path.exists(): 68 manager_template = (Path(__file__).parent / "dbmanager.py").read_text() 69 manager_path.write_text(manager_template.replace("$dbname", self.dbname)) 70 71 def open(self): 72 """Open connection to db.""" 73 self.connection = sqlite3.connect( 74 self.dbpath, 75 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 76 timeout=10, 77 ) 78 self.connection.execute("pragma foreign_keys = 1") 79 self.cursor = self.connection.cursor() 80 self.connection_open = True 81 82 def close(self): 83 """Save and close connection to db. 84 85 Call this as soon as you are done using the database if you have 86 multiple threads or processes using the same database.""" 87 if self.connection_open: 88 self.connection.commit() 89 self.connection.close() 90 self.connection_open = False 91 92 def _logger_init( 93 self, 94 message_format: str = "{levelname}|-|{asctime}|-|{message}", 95 encoding: str = "utf-8", 96 ): 97 """:param message_format: '{' style format string""" 98 self.logger = logging.getLogger(self.dbname) 99 if not self.logger.hasHandlers(): 100 handler = logging.FileHandler( 101 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 102 ) 103 handler.setFormatter( 104 logging.Formatter( 105 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 106 ) 107 ) 108 self.logger.addHandler(handler) 109 self.logger.setLevel(logging.INFO) 110 111 def _get_dict( 112 self, table: str, values: list, columns_to_return: list[str] = None 113 ) -> dict: 114 """Converts the values of a row into a dictionary with column names as keys. 115 116 :param table: The table that values were pulled from. 117 118 :param values: List of values expected to be the same quantity 119 and in the same order as the column names of table. 120 121 :param columns_to_return: An optional list of column names. 122 If given, only these columns will be included in the returned dictionary. 123 Otherwise all columns and values are returned.""" 124 return { 125 column: value 126 for column, value in zip(self.get_column_names(table), values) 127 if not columns_to_return or column in columns_to_return 128 } 129 130 def _get_conditions( 131 self, match_criteria: list[tuple] | dict, exact_match: bool = True 132 ) -> str: 133 """Builds and returns the conditional portion of a query. 134 135 :param match_criteria: Can be a list of 2-tuples where each 136 tuple is (columnName, rowValue) or a dictionary where 137 keys are column names and values are row values. 138 139 :param exact_match: If False, the rowValue for a give column 140 will be matched as a substring. 141 142 Usage e.g.: 143 144 self.cursor.execute(f'select * from {table} where {conditions}')""" 145 if type(match_criteria) == dict: 146 match_criteria = [(k, v) for k, v in match_criteria.items()] 147 if exact_match: 148 conditions = " and ".join( 149 f'"{column_row[0]}" = "{column_row[1]}"' 150 for column_row in match_criteria 151 ) 152 else: 153 conditions = " and ".join( 154 f'"{column_row[0]}" like "%{column_row[1]}%"' 155 for column_row in match_criteria 156 ) 157 return f"({conditions})" 158 159 @_connect 160 def query(self, query_) -> list[Any]: 161 """Execute an arbitrary query and 162 return the results.""" 163 self.cursor.execute(query_) 164 return self.cursor.fetchall() 165 166 @_connect 167 def create_tables(self, table_querys: list[str] = []): 168 """Create tables if they don't exist. 169 170 :param table_querys: Each query should be 171 in the form 'tableName(columnDefinitions)'""" 172 if len(table_querys) > 0: 173 table_names = self.get_table_names() 174 for table in table_querys: 175 if table.split("(")[0].strip() not in table_names: 176 self.cursor.execute(f"create table {table}") 177 self.logger.info(f'{table.split("(")[0]} table created.') 178 179 @_connect 180 def create_table(self, table: str, column_defs: list[str]): 181 """Create a table if it doesn't exist. 182 183 :param table: Name of the table to create. 184 185 :param column_defs: List of column definitions in 186 proper Sqlite3 sytax. 187 i.e. "columnName text unique" or "columnName int primary key" etc.""" 188 if table not in self.get_table_names(): 189 query = f"create table {table}({', '.join(column_defs)})" 190 self.cursor.execute(query) 191 self.logger.info(f"'{table}' table created.") 192 193 @_connect 194 def get_table_names(self) -> list[str]: 195 """Returns a list of table names from database.""" 196 self.cursor.execute( 197 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 198 ) 199 return [result[0] for result in self.cursor.fetchall()] 200 201 @_connect 202 def get_column_names(self, table: str) -> list[str]: 203 """Return a list of column names from a table.""" 204 self.cursor.execute(f"select * from {table} where 1=0") 205 return [description[0] for description in self.cursor.description] 206 207 @_connect 208 def count( 209 self, 210 table: str, 211 match_criteria: list[tuple] | dict = None, 212 exact_match: bool = True, 213 ) -> int: 214 """Return number of items in table. 215 216 :param match_criteria: Can be a list of 2-tuples where each 217 tuple is (columnName, rowValue) or a dictionary where 218 keys are column names and values are row values. 219 If None, all rows from the table will be counted. 220 221 :param exact_match: If False, the row value for a give column 222 in match_criteria will be matched as a substring. Has no effect if 223 match_criteria is None. 224 """ 225 query = f"select count(_rowid_) from {table}" 226 try: 227 if match_criteria: 228 self.cursor.execute( 229 f"{query} where {self._get_conditions(match_criteria, exact_match)}" 230 ) 231 else: 232 self.cursor.execute(f"{query}") 233 return self.cursor.fetchone()[0] 234 except: 235 return 0 236 237 @_connect 238 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 239 """Add row of values to table. 240 241 :param table: The table to insert into. 242 243 :param values: A tuple of values to be inserted into the table. 244 245 :param columns: If None, values param is expected to supply 246 a value for every column in the table. If columns is 247 provided, it should contain the same number of elements as values.""" 248 parameterizer = ", ".join("?" for _ in values) 249 logger_values = ", ".join(str(value) for value in values) 250 try: 251 if columns: 252 columns = ", ".join(column for column in columns) 253 self.cursor.execute( 254 f"insert into {table} ({columns}) values({parameterizer})", values 255 ) 256 else: 257 self.cursor.execute( 258 f"insert into {table} values({parameterizer})", values 259 ) 260 self.logger.info(f'Added "{logger_values}" to {table} table.') 261 except Exception as e: 262 if "constraint" not in str(e).lower(): 263 self.logger.exception( 264 f'Error adding "{logger_values}" to {table} table.' 265 ) 266 else: 267 self.logger.debug(str(e)) 268 269 @_connect 270 def get_rows( 271 self, 272 table: str, 273 match_criteria: list[tuple] | dict = None, 274 exact_match: bool = True, 275 sort_by_column: str = None, 276 columns_to_return: list[str] = None, 277 return_as_dataframe: bool = False, 278 values_only: bool = False, 279 order_by: str = None, 280 limit: str | int = None, 281 ) -> list[dict] | list[tuple] | pandas.DataFrame: 282 """Returns rows from table as a list of dictionaries 283 where the key-value pairs of the dictionaries are 284 column name: row value. 285 286 :param match_criteria: Can be a list of 2-tuples where each 287 tuple is (columnName, rowValue) or a dictionary where 288 keys are column names and values are row values. 289 290 :param exact_match: If False, the rowValue for a give column 291 will be matched as a substring. 292 293 :param sort_by_column: A column name to sort the results by. 294 This will sort results in Python after retrieving them from the db. 295 Use the 'order_by' param to use SQLite engine for ordering. 296 297 :param columns_to_return: Optional list of column names. 298 If provided, the elements returned by get_rows() will 299 only contain the provided columns. Otherwise every column 300 in the row is returned. 301 302 :param return_as_dataframe: If True, 303 the results will be returned as a pandas.DataFrame object. 304 305 :param values_only: Return the results as a list of tuples 306 instead of a list of dictionaries that have column names as keys. 307 The results will still be sorted according to sort_by_column if 308 one is provided. 309 310 :param order_by: If given, a 'order by {order_by}' clause 311 will be added to the select query. 312 313 :param limit: If given, a 'limit {limit}' clause will be 314 added to the select query. 315 """ 316 317 if type(columns_to_return) is str: 318 columns_to_return = [columns_to_return] 319 query = f"select * from {table}" 320 matches = [] 321 if match_criteria: 322 query += f" where {self._get_conditions(match_criteria, exact_match)}" 323 if order_by: 324 query += f" order by {order_by}" 325 if limit: 326 query += f" limit {limit}" 327 query += ";" 328 self.cursor.execute(query) 329 matches = self.cursor.fetchall() 330 results = [self._get_dict(table, match, columns_to_return) for match in matches] 331 if sort_by_column: 332 results = sorted(results, key=lambda x: x[sort_by_column]) 333 if return_as_dataframe: 334 return pandas.DataFrame(results) 335 if values_only: 336 return [tuple(row.values()) for row in results] 337 else: 338 return results 339 340 @_connect 341 def find( 342 self, table: str, query_string: str, columns: list[str] = None 343 ) -> tuple[dict]: 344 """Search for rows that contain query_string as a substring 345 of any column. 346 347 :param table: The table to search. 348 349 :param query_string: The substring to search for in all columns. 350 351 :param columns: A list of columns to search for query_string. 352 If None, all columns in the table will be searched. 353 """ 354 if type(columns) is str: 355 columns = [columns] 356 results = [] 357 if not columns: 358 columns = self.get_column_names(table) 359 for column in columns: 360 results.extend( 361 [ 362 row 363 for row in self.get_rows( 364 table, [(column, query_string)], exact_match=False 365 ) 366 if row not in results 367 ] 368 ) 369 return results 370 371 @_connect 372 def delete( 373 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 374 ) -> int: 375 """Delete records from table. 376 377 Returns number of deleted records. 378 379 :param match_criteria: Can be a list of 2-tuples where each 380 tuple is (columnName, rowValue) or a dictionary where 381 keys are column names and values are row values. 382 383 :param exact_match: If False, the rowValue for a give column 384 will be matched as a substring. 385 """ 386 num_matches = self.count(table, match_criteria, exact_match) 387 conditions = self._get_conditions(match_criteria, exact_match) 388 try: 389 self.cursor.execute(f"delete from {table} where {conditions}") 390 self.logger.info( 391 f'Deleted {num_matches} from "{table}" where {conditions}".' 392 ) 393 return num_matches 394 except Exception as e: 395 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 396 return 0 397 398 @_connect 399 def update( 400 self, 401 table: str, 402 column_to_update: str, 403 new_value: Any, 404 match_criteria: list[tuple] | dict = None, 405 ) -> bool: 406 """Update row value for entry matched with match_criteria. 407 408 :param column_to_update: The column to be updated in the matched row. 409 410 :param new_value: The new value to insert. 411 412 :param match_criteria: Can be a list of 2-tuples where each 413 tuple is (columnName, rowValue) or a dictionary where 414 keys are column names and values are row values. 415 If None, every row will be updated. 416 417 Returns True if successful, False if not.""" 418 query = f"update {table} set {column_to_update} = ?" 419 if match_criteria: 420 if self.count(table, match_criteria) == 0: 421 self.logger.info( 422 f"Couldn't find matching records in {table} table to update to '{new_value}'" 423 ) 424 return False 425 conditions = self._get_conditions(match_criteria) 426 query += f" where {conditions}" 427 else: 428 conditions = None 429 try: 430 self.cursor.execute( 431 query, 432 (new_value,), 433 ) 434 self.logger.info( 435 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 436 ) 437 return True 438 except UnboundLocalError: 439 table_filter_string = "\n".join( 440 table_filter for table_filter in match_criteria 441 ) 442 self.logger.error( 443 f"No records found matching filters: {table_filter_string}" 444 ) 445 return False 446 except Exception as e: 447 self.logger.error( 448 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 449 ) 450 return False 451 452 @_connect 453 def drop_table(self, table: str) -> bool: 454 """Drop a table from the database. 455 456 Returns True if successful, False if not.""" 457 try: 458 self.cursor.execute(f"drop Table {table}") 459 self.logger.info(f'Dropped table "{table}"') 460 except Exception as e: 461 print(e) 462 self.logger.error(f'Failed to drop table "{table}"') 463 464 @_connect 465 def add_column( 466 self, table: str, column: str, _type: str, default_value: str = None 467 ): 468 """Add a new column to table. 469 470 :param column: Name of the column to add. 471 472 :param _type: The data type of the new column. 473 474 :param default_value: Optional default value for the column.""" 475 try: 476 if default_value: 477 self.cursor.execute( 478 f"alter table {table} add column {column} {_type} default {default_value}" 479 ) 480 else: 481 self.cursor.execute(f"alter table {table} add column {column} {_type}") 482 self.logger.info(f'Added column "{column}" to "{table}" table.') 483 except Exception as e: 484 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 485 486 @staticmethod 487 def data_to_string( 488 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 489 ) -> str: 490 """Uses tabulate to produce pretty string output 491 from a list of dictionaries. 492 493 :param data: Assumes all dictionaries in list have the same set of keys. 494 495 :param sort_key: Optional dictionary key to sort data with. 496 497 :param wrap_to_terminal: If True, the table width will be wrapped 498 to fit within the current terminal window. Set to False 499 if the output is going into something like a txt file.""" 500 return data_to_string(data, sort_key, wrap_to_terminal) 501 502 503def data_to_string( 504 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 505) -> str: 506 """Use tabulate to produce grid output from a list of dictionaries. 507 508 :param data: Assumes all dictionaries in list have the same set of keys. 509 510 :param sort_key: Optional dictionary key to sort data with. 511 512 :param wrap_to_terminal: If True, the column widths will be reduced so the grid fits 513 within the current terminal window without wrapping. If the column widths have reduced to 1 514 and the grid is still too wide, str(data) will be returned.""" 515 if len(data) == 0: 516 return "" 517 if sort_key: 518 data = sorted(data, key=lambda d: d[sort_key]) 519 for i, d in enumerate(data): 520 for k in d: 521 data[i][k] = str(data[i][k]) 522 523 too_wide = True 524 terminal_width = os.get_terminal_size().columns 525 max_col_widths = terminal_width 526 # Make an output with effectively unrestricted column widths 527 # to see if shrinking is necessary 528 output = tabulate( 529 data, 530 headers="keys", 531 disable_numparse=True, 532 tablefmt="grid", 533 maxcolwidths=max_col_widths, 534 ) 535 current_width = output.index("\n") 536 if current_width < terminal_width: 537 too_wide = False 538 if wrap_to_terminal and too_wide: 539 print("Resizing grid to fit within the terminal...") 540 previous_col_widths = max_col_widths 541 acceptable_width = terminal_width - 10 542 while too_wide and max_col_widths > 1: 543 if current_width > terminal_width: 544 previous_col_widths = max_col_widths 545 max_col_widths = int(max_col_widths * 0.5) 546 elif current_width < terminal_width: 547 # Without lowering acceptable_width, this condition will cause infinite loop 548 if max_col_widths == previous_col_widths - 1: 549 acceptable_width -= 10 550 max_col_widths = int( 551 max_col_widths + (0.5 * (previous_col_widths - max_col_widths)) 552 ) 553 output = tabulate( 554 data, 555 headers="keys", 556 disable_numparse=True, 557 tablefmt="grid", 558 maxcolwidths=max_col_widths, 559 ) 560 current_width = output.index("\n") 561 if acceptable_width < current_width < terminal_width: 562 too_wide = False 563 if too_wide: 564 print("Couldn't resize grid to fit within the terminal.") 565 return str(data) 566 return output
27class DataBased: 28 """Sqli wrapper so queries don't need to be written except table definitions. 29 30 Supports saving and reading dates as datetime objects. 31 32 Supports using a context manager.""" 33 34 def __init__( 35 self, 36 dbpath: str | Path, 37 logger_encoding: str = "utf-8", 38 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 39 ): 40 """ 41 :param dbpath: String or Path object to database file. 42 If a relative path is given, it will be relative to the 43 current working directory. The log file will be saved to the 44 same directory. 45 46 :param logger_message_format: '{' style format string 47 for the logger object.""" 48 self.dbpath = Path(dbpath) 49 self.dbname = Path(dbpath).name 50 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 51 self._logger_init( 52 encoding=logger_encoding, message_format=logger_message_format 53 ) 54 self.connection_open = False 55 self.create_manager() 56 57 def __enter__(self): 58 self.open() 59 return self 60 61 def __exit__(self, exception_type, exception_value, exception_traceback): 62 self.close() 63 64 def create_manager(self): 65 """Create dbmanager.py in the same directory 66 as the database file if it doesn't exist.""" 67 manager_path = self.dbpath.parent / "dbmanager.py" 68 if not manager_path.exists(): 69 manager_template = (Path(__file__).parent / "dbmanager.py").read_text() 70 manager_path.write_text(manager_template.replace("$dbname", self.dbname)) 71 72 def open(self): 73 """Open connection to db.""" 74 self.connection = sqlite3.connect( 75 self.dbpath, 76 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 77 timeout=10, 78 ) 79 self.connection.execute("pragma foreign_keys = 1") 80 self.cursor = self.connection.cursor() 81 self.connection_open = True 82 83 def close(self): 84 """Save and close connection to db. 85 86 Call this as soon as you are done using the database if you have 87 multiple threads or processes using the same database.""" 88 if self.connection_open: 89 self.connection.commit() 90 self.connection.close() 91 self.connection_open = False 92 93 def _logger_init( 94 self, 95 message_format: str = "{levelname}|-|{asctime}|-|{message}", 96 encoding: str = "utf-8", 97 ): 98 """:param message_format: '{' style format string""" 99 self.logger = logging.getLogger(self.dbname) 100 if not self.logger.hasHandlers(): 101 handler = logging.FileHandler( 102 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 103 ) 104 handler.setFormatter( 105 logging.Formatter( 106 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 107 ) 108 ) 109 self.logger.addHandler(handler) 110 self.logger.setLevel(logging.INFO) 111 112 def _get_dict( 113 self, table: str, values: list, columns_to_return: list[str] = None 114 ) -> dict: 115 """Converts the values of a row into a dictionary with column names as keys. 116 117 :param table: The table that values were pulled from. 118 119 :param values: List of values expected to be the same quantity 120 and in the same order as the column names of table. 121 122 :param columns_to_return: An optional list of column names. 123 If given, only these columns will be included in the returned dictionary. 124 Otherwise all columns and values are returned.""" 125 return { 126 column: value 127 for column, value in zip(self.get_column_names(table), values) 128 if not columns_to_return or column in columns_to_return 129 } 130 131 def _get_conditions( 132 self, match_criteria: list[tuple] | dict, exact_match: bool = True 133 ) -> str: 134 """Builds and returns the conditional portion of a query. 135 136 :param match_criteria: Can be a list of 2-tuples where each 137 tuple is (columnName, rowValue) or a dictionary where 138 keys are column names and values are row values. 139 140 :param exact_match: If False, the rowValue for a give column 141 will be matched as a substring. 142 143 Usage e.g.: 144 145 self.cursor.execute(f'select * from {table} where {conditions}')""" 146 if type(match_criteria) == dict: 147 match_criteria = [(k, v) for k, v in match_criteria.items()] 148 if exact_match: 149 conditions = " and ".join( 150 f'"{column_row[0]}" = "{column_row[1]}"' 151 for column_row in match_criteria 152 ) 153 else: 154 conditions = " and ".join( 155 f'"{column_row[0]}" like "%{column_row[1]}%"' 156 for column_row in match_criteria 157 ) 158 return f"({conditions})" 159 160 @_connect 161 def query(self, query_) -> list[Any]: 162 """Execute an arbitrary query and 163 return the results.""" 164 self.cursor.execute(query_) 165 return self.cursor.fetchall() 166 167 @_connect 168 def create_tables(self, table_querys: list[str] = []): 169 """Create tables if they don't exist. 170 171 :param table_querys: Each query should be 172 in the form 'tableName(columnDefinitions)'""" 173 if len(table_querys) > 0: 174 table_names = self.get_table_names() 175 for table in table_querys: 176 if table.split("(")[0].strip() not in table_names: 177 self.cursor.execute(f"create table {table}") 178 self.logger.info(f'{table.split("(")[0]} table created.') 179 180 @_connect 181 def create_table(self, table: str, column_defs: list[str]): 182 """Create a table if it doesn't exist. 183 184 :param table: Name of the table to create. 185 186 :param column_defs: List of column definitions in 187 proper Sqlite3 sytax. 188 i.e. "columnName text unique" or "columnName int primary key" etc.""" 189 if table not in self.get_table_names(): 190 query = f"create table {table}({', '.join(column_defs)})" 191 self.cursor.execute(query) 192 self.logger.info(f"'{table}' table created.") 193 194 @_connect 195 def get_table_names(self) -> list[str]: 196 """Returns a list of table names from database.""" 197 self.cursor.execute( 198 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 199 ) 200 return [result[0] for result in self.cursor.fetchall()] 201 202 @_connect 203 def get_column_names(self, table: str) -> list[str]: 204 """Return a list of column names from a table.""" 205 self.cursor.execute(f"select * from {table} where 1=0") 206 return [description[0] for description in self.cursor.description] 207 208 @_connect 209 def count( 210 self, 211 table: str, 212 match_criteria: list[tuple] | dict = None, 213 exact_match: bool = True, 214 ) -> int: 215 """Return number of items in table. 216 217 :param match_criteria: Can be a list of 2-tuples where each 218 tuple is (columnName, rowValue) or a dictionary where 219 keys are column names and values are row values. 220 If None, all rows from the table will be counted. 221 222 :param exact_match: If False, the row value for a give column 223 in match_criteria will be matched as a substring. Has no effect if 224 match_criteria is None. 225 """ 226 query = f"select count(_rowid_) from {table}" 227 try: 228 if match_criteria: 229 self.cursor.execute( 230 f"{query} where {self._get_conditions(match_criteria, exact_match)}" 231 ) 232 else: 233 self.cursor.execute(f"{query}") 234 return self.cursor.fetchone()[0] 235 except: 236 return 0 237 238 @_connect 239 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 240 """Add row of values to table. 241 242 :param table: The table to insert into. 243 244 :param values: A tuple of values to be inserted into the table. 245 246 :param columns: If None, values param is expected to supply 247 a value for every column in the table. If columns is 248 provided, it should contain the same number of elements as values.""" 249 parameterizer = ", ".join("?" for _ in values) 250 logger_values = ", ".join(str(value) for value in values) 251 try: 252 if columns: 253 columns = ", ".join(column for column in columns) 254 self.cursor.execute( 255 f"insert into {table} ({columns}) values({parameterizer})", values 256 ) 257 else: 258 self.cursor.execute( 259 f"insert into {table} values({parameterizer})", values 260 ) 261 self.logger.info(f'Added "{logger_values}" to {table} table.') 262 except Exception as e: 263 if "constraint" not in str(e).lower(): 264 self.logger.exception( 265 f'Error adding "{logger_values}" to {table} table.' 266 ) 267 else: 268 self.logger.debug(str(e)) 269 270 @_connect 271 def get_rows( 272 self, 273 table: str, 274 match_criteria: list[tuple] | dict = None, 275 exact_match: bool = True, 276 sort_by_column: str = None, 277 columns_to_return: list[str] = None, 278 return_as_dataframe: bool = False, 279 values_only: bool = False, 280 order_by: str = None, 281 limit: str | int = None, 282 ) -> list[dict] | list[tuple] | pandas.DataFrame: 283 """Returns rows from table as a list of dictionaries 284 where the key-value pairs of the dictionaries are 285 column name: row value. 286 287 :param match_criteria: Can be a list of 2-tuples where each 288 tuple is (columnName, rowValue) or a dictionary where 289 keys are column names and values are row values. 290 291 :param exact_match: If False, the rowValue for a give column 292 will be matched as a substring. 293 294 :param sort_by_column: A column name to sort the results by. 295 This will sort results in Python after retrieving them from the db. 296 Use the 'order_by' param to use SQLite engine for ordering. 297 298 :param columns_to_return: Optional list of column names. 299 If provided, the elements returned by get_rows() will 300 only contain the provided columns. Otherwise every column 301 in the row is returned. 302 303 :param return_as_dataframe: If True, 304 the results will be returned as a pandas.DataFrame object. 305 306 :param values_only: Return the results as a list of tuples 307 instead of a list of dictionaries that have column names as keys. 308 The results will still be sorted according to sort_by_column if 309 one is provided. 310 311 :param order_by: If given, a 'order by {order_by}' clause 312 will be added to the select query. 313 314 :param limit: If given, a 'limit {limit}' clause will be 315 added to the select query. 316 """ 317 318 if type(columns_to_return) is str: 319 columns_to_return = [columns_to_return] 320 query = f"select * from {table}" 321 matches = [] 322 if match_criteria: 323 query += f" where {self._get_conditions(match_criteria, exact_match)}" 324 if order_by: 325 query += f" order by {order_by}" 326 if limit: 327 query += f" limit {limit}" 328 query += ";" 329 self.cursor.execute(query) 330 matches = self.cursor.fetchall() 331 results = [self._get_dict(table, match, columns_to_return) for match in matches] 332 if sort_by_column: 333 results = sorted(results, key=lambda x: x[sort_by_column]) 334 if return_as_dataframe: 335 return pandas.DataFrame(results) 336 if values_only: 337 return [tuple(row.values()) for row in results] 338 else: 339 return results 340 341 @_connect 342 def find( 343 self, table: str, query_string: str, columns: list[str] = None 344 ) -> tuple[dict]: 345 """Search for rows that contain query_string as a substring 346 of any column. 347 348 :param table: The table to search. 349 350 :param query_string: The substring to search for in all columns. 351 352 :param columns: A list of columns to search for query_string. 353 If None, all columns in the table will be searched. 354 """ 355 if type(columns) is str: 356 columns = [columns] 357 results = [] 358 if not columns: 359 columns = self.get_column_names(table) 360 for column in columns: 361 results.extend( 362 [ 363 row 364 for row in self.get_rows( 365 table, [(column, query_string)], exact_match=False 366 ) 367 if row not in results 368 ] 369 ) 370 return results 371 372 @_connect 373 def delete( 374 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 375 ) -> int: 376 """Delete records from table. 377 378 Returns number of deleted records. 379 380 :param match_criteria: Can be a list of 2-tuples where each 381 tuple is (columnName, rowValue) or a dictionary where 382 keys are column names and values are row values. 383 384 :param exact_match: If False, the rowValue for a give column 385 will be matched as a substring. 386 """ 387 num_matches = self.count(table, match_criteria, exact_match) 388 conditions = self._get_conditions(match_criteria, exact_match) 389 try: 390 self.cursor.execute(f"delete from {table} where {conditions}") 391 self.logger.info( 392 f'Deleted {num_matches} from "{table}" where {conditions}".' 393 ) 394 return num_matches 395 except Exception as e: 396 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 397 return 0 398 399 @_connect 400 def update( 401 self, 402 table: str, 403 column_to_update: str, 404 new_value: Any, 405 match_criteria: list[tuple] | dict = None, 406 ) -> bool: 407 """Update row value for entry matched with match_criteria. 408 409 :param column_to_update: The column to be updated in the matched row. 410 411 :param new_value: The new value to insert. 412 413 :param match_criteria: Can be a list of 2-tuples where each 414 tuple is (columnName, rowValue) or a dictionary where 415 keys are column names and values are row values. 416 If None, every row will be updated. 417 418 Returns True if successful, False if not.""" 419 query = f"update {table} set {column_to_update} = ?" 420 if match_criteria: 421 if self.count(table, match_criteria) == 0: 422 self.logger.info( 423 f"Couldn't find matching records in {table} table to update to '{new_value}'" 424 ) 425 return False 426 conditions = self._get_conditions(match_criteria) 427 query += f" where {conditions}" 428 else: 429 conditions = None 430 try: 431 self.cursor.execute( 432 query, 433 (new_value,), 434 ) 435 self.logger.info( 436 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 437 ) 438 return True 439 except UnboundLocalError: 440 table_filter_string = "\n".join( 441 table_filter for table_filter in match_criteria 442 ) 443 self.logger.error( 444 f"No records found matching filters: {table_filter_string}" 445 ) 446 return False 447 except Exception as e: 448 self.logger.error( 449 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 450 ) 451 return False 452 453 @_connect 454 def drop_table(self, table: str) -> bool: 455 """Drop a table from the database. 456 457 Returns True if successful, False if not.""" 458 try: 459 self.cursor.execute(f"drop Table {table}") 460 self.logger.info(f'Dropped table "{table}"') 461 except Exception as e: 462 print(e) 463 self.logger.error(f'Failed to drop table "{table}"') 464 465 @_connect 466 def add_column( 467 self, table: str, column: str, _type: str, default_value: str = None 468 ): 469 """Add a new column to table. 470 471 :param column: Name of the column to add. 472 473 :param _type: The data type of the new column. 474 475 :param default_value: Optional default value for the column.""" 476 try: 477 if default_value: 478 self.cursor.execute( 479 f"alter table {table} add column {column} {_type} default {default_value}" 480 ) 481 else: 482 self.cursor.execute(f"alter table {table} add column {column} {_type}") 483 self.logger.info(f'Added column "{column}" to "{table}" table.') 484 except Exception as e: 485 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 486 487 @staticmethod 488 def data_to_string( 489 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 490 ) -> str: 491 """Uses tabulate to produce pretty string output 492 from a list of dictionaries. 493 494 :param data: Assumes all dictionaries in list have the same set of keys. 495 496 :param sort_key: Optional dictionary key to sort data with. 497 498 :param wrap_to_terminal: If True, the table width will be wrapped 499 to fit within the current terminal window. Set to False 500 if the output is going into something like a txt file.""" 501 return data_to_string(data, sort_key, wrap_to_terminal)
Sqli wrapper so queries don't need to be written except table definitions.
Supports saving and reading dates as datetime objects.
Supports using a context manager.
34 def __init__( 35 self, 36 dbpath: str | Path, 37 logger_encoding: str = "utf-8", 38 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 39 ): 40 """ 41 :param dbpath: String or Path object to database file. 42 If a relative path is given, it will be relative to the 43 current working directory. The log file will be saved to the 44 same directory. 45 46 :param logger_message_format: '{' style format string 47 for the logger object.""" 48 self.dbpath = Path(dbpath) 49 self.dbname = Path(dbpath).name 50 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 51 self._logger_init( 52 encoding=logger_encoding, message_format=logger_message_format 53 ) 54 self.connection_open = False 55 self.create_manager()
Parameters
dbpath: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.
logger_message_format: '{' style format string for the logger object.
64 def create_manager(self): 65 """Create dbmanager.py in the same directory 66 as the database file if it doesn't exist.""" 67 manager_path = self.dbpath.parent / "dbmanager.py" 68 if not manager_path.exists(): 69 manager_template = (Path(__file__).parent / "dbmanager.py").read_text() 70 manager_path.write_text(manager_template.replace("$dbname", self.dbname))
Create dbmanager.py in the same directory as the database file if it doesn't exist.
72 def open(self): 73 """Open connection to db.""" 74 self.connection = sqlite3.connect( 75 self.dbpath, 76 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 77 timeout=10, 78 ) 79 self.connection.execute("pragma foreign_keys = 1") 80 self.cursor = self.connection.cursor() 81 self.connection_open = True
Open connection to db.
83 def close(self): 84 """Save and close connection to db. 85 86 Call this as soon as you are done using the database if you have 87 multiple threads or processes using the same database.""" 88 if self.connection_open: 89 self.connection.commit() 90 self.connection.close() 91 self.connection_open = False
Save and close connection to db.
Call this as soon as you are done using the database if you have multiple threads or processes using the same database.
160 @_connect 161 def query(self, query_) -> list[Any]: 162 """Execute an arbitrary query and 163 return the results.""" 164 self.cursor.execute(query_) 165 return self.cursor.fetchall()
Execute an arbitrary query and return the results.
167 @_connect 168 def create_tables(self, table_querys: list[str] = []): 169 """Create tables if they don't exist. 170 171 :param table_querys: Each query should be 172 in the form 'tableName(columnDefinitions)'""" 173 if len(table_querys) > 0: 174 table_names = self.get_table_names() 175 for table in table_querys: 176 if table.split("(")[0].strip() not in table_names: 177 self.cursor.execute(f"create table {table}") 178 self.logger.info(f'{table.split("(")[0]} table created.')
Create tables if they don't exist.
Parameters
- table_querys: Each query should be in the form 'tableName(columnDefinitions)'
180 @_connect 181 def create_table(self, table: str, column_defs: list[str]): 182 """Create a table if it doesn't exist. 183 184 :param table: Name of the table to create. 185 186 :param column_defs: List of column definitions in 187 proper Sqlite3 sytax. 188 i.e. "columnName text unique" or "columnName int primary key" etc.""" 189 if table not in self.get_table_names(): 190 query = f"create table {table}({', '.join(column_defs)})" 191 self.cursor.execute(query) 192 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
Parameters
table: Name of the table to create.
column_defs: List of column definitions in proper Sqlite3 sytax. i.e. "columnName text unique" or "columnName int primary key" etc.
194 @_connect 195 def get_table_names(self) -> list[str]: 196 """Returns a list of table names from database.""" 197 self.cursor.execute( 198 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%"' 199 ) 200 return [result[0] for result in self.cursor.fetchall()]
Returns a list of table names from database.
202 @_connect 203 def get_column_names(self, table: str) -> list[str]: 204 """Return a list of column names from a table.""" 205 self.cursor.execute(f"select * from {table} where 1=0") 206 return [description[0] for description in self.cursor.description]
Return a list of column names from a table.
208 @_connect 209 def count( 210 self, 211 table: str, 212 match_criteria: list[tuple] | dict = None, 213 exact_match: bool = True, 214 ) -> int: 215 """Return number of items in table. 216 217 :param match_criteria: Can be a list of 2-tuples where each 218 tuple is (columnName, rowValue) or a dictionary where 219 keys are column names and values are row values. 220 If None, all rows from the table will be counted. 221 222 :param exact_match: If False, the row value for a give column 223 in match_criteria will be matched as a substring. Has no effect if 224 match_criteria is None. 225 """ 226 query = f"select count(_rowid_) from {table}" 227 try: 228 if match_criteria: 229 self.cursor.execute( 230 f"{query} where {self._get_conditions(match_criteria, exact_match)}" 231 ) 232 else: 233 self.cursor.execute(f"{query}") 234 return self.cursor.fetchone()[0] 235 except: 236 return 0
Return number of items in table.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, all rows from the table will be counted.
exact_match: If False, the row value for a give column in match_criteria will be matched as a substring. Has no effect if match_criteria is None.
238 @_connect 239 def add_row(self, table: str, values: tuple[any], columns: tuple[str] = None): 240 """Add row of values to table. 241 242 :param table: The table to insert into. 243 244 :param values: A tuple of values to be inserted into the table. 245 246 :param columns: If None, values param is expected to supply 247 a value for every column in the table. If columns is 248 provided, it should contain the same number of elements as values.""" 249 parameterizer = ", ".join("?" for _ in values) 250 logger_values = ", ".join(str(value) for value in values) 251 try: 252 if columns: 253 columns = ", ".join(column for column in columns) 254 self.cursor.execute( 255 f"insert into {table} ({columns}) values({parameterizer})", values 256 ) 257 else: 258 self.cursor.execute( 259 f"insert into {table} values({parameterizer})", values 260 ) 261 self.logger.info(f'Added "{logger_values}" to {table} table.') 262 except Exception as e: 263 if "constraint" not in str(e).lower(): 264 self.logger.exception( 265 f'Error adding "{logger_values}" to {table} table.' 266 ) 267 else: 268 self.logger.debug(str(e))
Add row of values to table.
Parameters
table: The table to insert into.
values: A tuple of values to be inserted into the table.
columns: If None, values param is expected to supply a value for every column in the table. If columns is provided, it should contain the same number of elements as values.
270 @_connect 271 def get_rows( 272 self, 273 table: str, 274 match_criteria: list[tuple] | dict = None, 275 exact_match: bool = True, 276 sort_by_column: str = None, 277 columns_to_return: list[str] = None, 278 return_as_dataframe: bool = False, 279 values_only: bool = False, 280 order_by: str = None, 281 limit: str | int = None, 282 ) -> list[dict] | list[tuple] | pandas.DataFrame: 283 """Returns rows from table as a list of dictionaries 284 where the key-value pairs of the dictionaries are 285 column name: row value. 286 287 :param match_criteria: Can be a list of 2-tuples where each 288 tuple is (columnName, rowValue) or a dictionary where 289 keys are column names and values are row values. 290 291 :param exact_match: If False, the rowValue for a give column 292 will be matched as a substring. 293 294 :param sort_by_column: A column name to sort the results by. 295 This will sort results in Python after retrieving them from the db. 296 Use the 'order_by' param to use SQLite engine for ordering. 297 298 :param columns_to_return: Optional list of column names. 299 If provided, the elements returned by get_rows() will 300 only contain the provided columns. Otherwise every column 301 in the row is returned. 302 303 :param return_as_dataframe: If True, 304 the results will be returned as a pandas.DataFrame object. 305 306 :param values_only: Return the results as a list of tuples 307 instead of a list of dictionaries that have column names as keys. 308 The results will still be sorted according to sort_by_column if 309 one is provided. 310 311 :param order_by: If given, a 'order by {order_by}' clause 312 will be added to the select query. 313 314 :param limit: If given, a 'limit {limit}' clause will be 315 added to the select query. 316 """ 317 318 if type(columns_to_return) is str: 319 columns_to_return = [columns_to_return] 320 query = f"select * from {table}" 321 matches = [] 322 if match_criteria: 323 query += f" where {self._get_conditions(match_criteria, exact_match)}" 324 if order_by: 325 query += f" order by {order_by}" 326 if limit: 327 query += f" limit {limit}" 328 query += ";" 329 self.cursor.execute(query) 330 matches = self.cursor.fetchall() 331 results = [self._get_dict(table, match, columns_to_return) for match in matches] 332 if sort_by_column: 333 results = sorted(results, key=lambda x: x[sort_by_column]) 334 if return_as_dataframe: 335 return pandas.DataFrame(results) 336 if values_only: 337 return [tuple(row.values()) for row in results] 338 else: 339 return results
Returns rows from table as a list of dictionaries where the key-value pairs of the dictionaries are column name: row value.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
sort_by_column: A column name to sort the results by. This will sort results in Python after retrieving them from the db. Use the 'order_by' param to use SQLite engine for ordering.
columns_to_return: Optional list of column names. If provided, the elements returned by get_rows() will only contain the provided columns. Otherwise every column in the row is returned.
return_as_dataframe: If True, the results will be returned as a pandas.DataFrame object.
values_only: Return the results as a list of tuples instead of a list of dictionaries that have column names as keys. The results will still be sorted according to sort_by_column if one is provided.
order_by: If given, a 'order by {order_by}' clause will be added to the select query.
limit: If given, a 'limit {limit}' clause will be added to the select query.
341 @_connect 342 def find( 343 self, table: str, query_string: str, columns: list[str] = None 344 ) -> tuple[dict]: 345 """Search for rows that contain query_string as a substring 346 of any column. 347 348 :param table: The table to search. 349 350 :param query_string: The substring to search for in all columns. 351 352 :param columns: A list of columns to search for query_string. 353 If None, all columns in the table will be searched. 354 """ 355 if type(columns) is str: 356 columns = [columns] 357 results = [] 358 if not columns: 359 columns = self.get_column_names(table) 360 for column in columns: 361 results.extend( 362 [ 363 row 364 for row in self.get_rows( 365 table, [(column, query_string)], exact_match=False 366 ) 367 if row not in results 368 ] 369 ) 370 return results
Search for rows that contain query_string as a substring of any column.
Parameters
table: The table to search.
query_string: The substring to search for in all columns.
columns: A list of columns to search for query_string. If None, all columns in the table will be searched.
372 @_connect 373 def delete( 374 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 375 ) -> int: 376 """Delete records from table. 377 378 Returns number of deleted records. 379 380 :param match_criteria: Can be a list of 2-tuples where each 381 tuple is (columnName, rowValue) or a dictionary where 382 keys are column names and values are row values. 383 384 :param exact_match: If False, the rowValue for a give column 385 will be matched as a substring. 386 """ 387 num_matches = self.count(table, match_criteria, exact_match) 388 conditions = self._get_conditions(match_criteria, exact_match) 389 try: 390 self.cursor.execute(f"delete from {table} where {conditions}") 391 self.logger.info( 392 f'Deleted {num_matches} from "{table}" where {conditions}".' 393 ) 394 return num_matches 395 except Exception as e: 396 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 397 return 0
Delete records from table.
Returns number of deleted records.
Parameters
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values.
exact_match: If False, the rowValue for a give column will be matched as a substring.
399 @_connect 400 def update( 401 self, 402 table: str, 403 column_to_update: str, 404 new_value: Any, 405 match_criteria: list[tuple] | dict = None, 406 ) -> bool: 407 """Update row value for entry matched with match_criteria. 408 409 :param column_to_update: The column to be updated in the matched row. 410 411 :param new_value: The new value to insert. 412 413 :param match_criteria: Can be a list of 2-tuples where each 414 tuple is (columnName, rowValue) or a dictionary where 415 keys are column names and values are row values. 416 If None, every row will be updated. 417 418 Returns True if successful, False if not.""" 419 query = f"update {table} set {column_to_update} = ?" 420 if match_criteria: 421 if self.count(table, match_criteria) == 0: 422 self.logger.info( 423 f"Couldn't find matching records in {table} table to update to '{new_value}'" 424 ) 425 return False 426 conditions = self._get_conditions(match_criteria) 427 query += f" where {conditions}" 428 else: 429 conditions = None 430 try: 431 self.cursor.execute( 432 query, 433 (new_value,), 434 ) 435 self.logger.info( 436 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 437 ) 438 return True 439 except UnboundLocalError: 440 table_filter_string = "\n".join( 441 table_filter for table_filter in match_criteria 442 ) 443 self.logger.error( 444 f"No records found matching filters: {table_filter_string}" 445 ) 446 return False 447 except Exception as e: 448 self.logger.error( 449 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 450 ) 451 return False
Update row value for entry matched with match_criteria.
Parameters
column_to_update: The column to be updated in the matched row.
new_value: The new value to insert.
match_criteria: Can be a list of 2-tuples where each tuple is (columnName, rowValue) or a dictionary where keys are column names and values are row values. If None, every row will be updated.
Returns True if successful, False if not.
453 @_connect 454 def drop_table(self, table: str) -> bool: 455 """Drop a table from the database. 456 457 Returns True if successful, False if not.""" 458 try: 459 self.cursor.execute(f"drop Table {table}") 460 self.logger.info(f'Dropped table "{table}"') 461 except Exception as e: 462 print(e) 463 self.logger.error(f'Failed to drop table "{table}"')
Drop a table from the database.
Returns True if successful, False if not.
465 @_connect 466 def add_column( 467 self, table: str, column: str, _type: str, default_value: str = None 468 ): 469 """Add a new column to table. 470 471 :param column: Name of the column to add. 472 473 :param _type: The data type of the new column. 474 475 :param default_value: Optional default value for the column.""" 476 try: 477 if default_value: 478 self.cursor.execute( 479 f"alter table {table} add column {column} {_type} default {default_value}" 480 ) 481 else: 482 self.cursor.execute(f"alter table {table} add column {column} {_type}") 483 self.logger.info(f'Added column "{column}" to "{table}" table.') 484 except Exception as e: 485 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Add a new column to table.
Parameters
column: Name of the column to add.
_type: The data type of the new column.
default_value: Optional default value for the column.
487 @staticmethod 488 def data_to_string( 489 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 490 ) -> str: 491 """Uses tabulate to produce pretty string output 492 from a list of dictionaries. 493 494 :param data: Assumes all dictionaries in list have the same set of keys. 495 496 :param sort_key: Optional dictionary key to sort data with. 497 498 :param wrap_to_terminal: If True, the table width will be wrapped 499 to fit within the current terminal window. Set to False 500 if the output is going into something like a txt file.""" 501 return data_to_string(data, sort_key, wrap_to_terminal)
Uses tabulate to produce pretty string output from a list of dictionaries.
Parameters
data: Assumes all dictionaries in list have the same set of keys.
sort_key: Optional dictionary key to sort data with.
wrap_to_terminal: If True, the table width will be wrapped to fit within the current terminal window. Set to False if the output is going into something like a txt file.
504def data_to_string( 505 data: list[dict], sort_key: str = None, wrap_to_terminal: bool = True 506) -> str: 507 """Use tabulate to produce grid output from a list of dictionaries. 508 509 :param data: Assumes all dictionaries in list have the same set of keys. 510 511 :param sort_key: Optional dictionary key to sort data with. 512 513 :param wrap_to_terminal: If True, the column widths will be reduced so the grid fits 514 within the current terminal window without wrapping. If the column widths have reduced to 1 515 and the grid is still too wide, str(data) will be returned.""" 516 if len(data) == 0: 517 return "" 518 if sort_key: 519 data = sorted(data, key=lambda d: d[sort_key]) 520 for i, d in enumerate(data): 521 for k in d: 522 data[i][k] = str(data[i][k]) 523 524 too_wide = True 525 terminal_width = os.get_terminal_size().columns 526 max_col_widths = terminal_width 527 # Make an output with effectively unrestricted column widths 528 # to see if shrinking is necessary 529 output = tabulate( 530 data, 531 headers="keys", 532 disable_numparse=True, 533 tablefmt="grid", 534 maxcolwidths=max_col_widths, 535 ) 536 current_width = output.index("\n") 537 if current_width < terminal_width: 538 too_wide = False 539 if wrap_to_terminal and too_wide: 540 print("Resizing grid to fit within the terminal...") 541 previous_col_widths = max_col_widths 542 acceptable_width = terminal_width - 10 543 while too_wide and max_col_widths > 1: 544 if current_width > terminal_width: 545 previous_col_widths = max_col_widths 546 max_col_widths = int(max_col_widths * 0.5) 547 elif current_width < terminal_width: 548 # Without lowering acceptable_width, this condition will cause infinite loop 549 if max_col_widths == previous_col_widths - 1: 550 acceptable_width -= 10 551 max_col_widths = int( 552 max_col_widths + (0.5 * (previous_col_widths - max_col_widths)) 553 ) 554 output = tabulate( 555 data, 556 headers="keys", 557 disable_numparse=True, 558 tablefmt="grid", 559 maxcolwidths=max_col_widths, 560 ) 561 current_width = output.index("\n") 562 if acceptable_width < current_width < terminal_width: 563 too_wide = False 564 if too_wide: 565 print("Couldn't resize grid to fit within the terminal.") 566 return str(data) 567 return output
Use tabulate to produce grid output from a list of dictionaries.
Parameters
data: Assumes all dictionaries in list have the same set of keys.
sort_key: Optional dictionary key to sort data with.
wrap_to_terminal: If True, the column widths will be reduced so the grid fits within the current terminal window without wrapping. If the column widths have reduced to 1 and the grid is still too wide, str(data) will be returned.