Coverage for pytest_recap/plugin.py: 71%

217 statements  

« prev     ^ index     » next       coverage.py v7.8.0, created at 2025-06-16 00:19 -0600

1import ast 

2import json 

3import logging 

4import os 

5import platform 

6import socket 

7import uuid 

8from datetime import datetime, timezone 

9from typing import Dict, Generator, List, Optional, Tuple 

10from warnings import WarningMessage 

11 

12import pytest 

13from _pytest.config import Config 

14from _pytest.config.argparsing import Parser 

15from _pytest.nodes import Item 

16from _pytest.reports import TestReport 

17from _pytest.runner import CallInfo 

18from _pytest.terminal import TerminalReporter 

19 

20from pytest_recap.cloud import upload_to_cloud 

21from pytest_recap.models import RecapEvent, RerunTestGroup, TestResult, TestSession, TestSessionStats 

22from pytest_recap.storage import JSONStorage 

23 

24# --- Global warning collection. This is required because Pytest hook pytest-warning-recorded 

25# does not pass the Config object, so it cannot be used to store warnings. 

26_collected_warnings = [] 

27 

28 

29# --- pytest hooks --- # 

30def pytest_addoption(parser: Parser) -> None: 

31 """Add command line options for pytest-recap, supporting environment variable defaults. 

32 

33 Args: 

34 parser (Parser): The pytest parser object. 

35 """ 

36 group = parser.getgroup("Pytest Recap") 

37 recap_env = os.environ.get("RECAP_ENABLE", "0").lower() 

38 recap_default = recap_env in ("1", "true", "yes", "y") 

39 group.addoption( 

40 "--recap", 

41 action="store_true", 

42 default=recap_default, 

43 help="Enable pytest-recap plugin (or set environment variable RECAP_ENABLE)", 

44 ) 

45 recap_dest_env = os.environ.get("RECAP_DESTINATION") 

46 if recap_dest_env: 

47 recap_dest_default = recap_dest_env 

48 else: 

49 timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") 

50 default_dir = os.path.expanduser("~/.pytest-recap-sessions") 

51 os.makedirs(default_dir, exist_ok=True) 

52 recap_dest_default = os.path.join(default_dir, f"{timestamp}-recap.json") 

53 group.addoption( 

54 "--recap-destination", 

55 action="store", 

56 default=recap_dest_default, 

57 help="Specify pytest-recap storage destination (filepath) (or set environment variable RECAP_DESTINATION)", 

58 ) 

59 group.addoption( 

60 "--recap-system-under-test", 

61 action="store", 

62 default=None, 

63 help="JSON or Python dict string for system under test metadata (or set RECAP_SYSTEM_UNDER_TEST)", 

64 ) 

65 group.addoption( 

66 "--recap-testing-system", 

67 action="store", 

68 default=None, 

69 help="JSON or Python dict string for testing system metadata (or set RECAP_TESTING_SYSTEM)", 

70 ) 

71 group.addoption( 

72 "--recap-session-tags", 

73 action="store", 

74 default=None, 

75 help="JSON or Python dict string for session tags (or set RECAP_SESSION_TAGS)", 

76 ) 

77 group.addoption( 

78 "--recap-pretty", 

79 action="store_true", 

80 default=None, 

81 help="Pretty-print recap JSON output (or set RECAP_PRETTY=1, or ini: recap_pretty=1)", 

82 ) 

83 

84 parser.addini("recap_system_under_test", "System under test dict (JSON or Python dict string)", default="") 

85 parser.addini("recap_testing_system", "Testing system dict (JSON or Python dict string)", default="") 

86 parser.addini("recap_session_tags", "Session tags dict (JSON or Python dict string)", default="") 

87 parser.addini("recap_pretty", "Pretty-print recap JSON output (1 for pretty, 0 for minified)", default="0") 

88 

89 

90def pytest_configure(config: Config) -> None: 

91 """Configure pytest-recap plugin. 

92 

93 Args: 

94 config (Config): The pytest Config object. 

95 """ 

96 config._recap_enabled: bool = config.getoption("--recap") 

97 config._recap_destination: str = config.getoption("--recap-destination") 

98 pretty = get_recap_option(config, "recap_pretty", "recap_pretty", "RECAP_PRETTY", default="0") 

99 config._recap_pretty: bool = str(pretty).strip().lower() in {"1", "true", "yes", "y"} 

100 

101 

102def pytest_sessionstart(session): 

103 """Reset collected warnings at the start of each test session.""" 

104 global _collected_warnings 

105 _collected_warnings = [] 

106 

107 

108@pytest.hookimpl(hookwrapper=True) 

109def pytest_runtest_makereport(item: Item, call: CallInfo) -> Generator: 

110 """Hook into pytest's test report generation to generate start and stop times, if not set already.""" 

111 outcome = yield 

112 

113 logger = logging.getLogger(__name__) 

114 

115 report: TestReport = outcome.get_result() 

116 if report.when == "setup" and not hasattr(report, "start"): 

117 logger.warning(f"Setting start time for {report.nodeid} since it was not set previously") 

118 setattr(report, "start", datetime.now(timezone.utc).timestamp()) 

119 

120 if report.when == "teardown" and not hasattr(report, "stop"): 

121 logger.warning(f"Setting stop time for {report.nodeid} since it was not set previously") 

122 setattr(report, "stop", datetime.now(timezone.utc).timestamp()) 

123 

124 

125def pytest_warning_recorded(warning_message: WarningMessage, when: str, nodeid: str, location: tuple): 

126 """Collect warnings during pytest session for recap reporting. 

127 

128 Args: 

129 warning_message (WarningMessage): The warning message object. 

130 when (str): When the warning was recorded (e.g., 'call', 'setup', etc.). 

131 nodeid (str): Node ID of the test (if any). 

132 location (tuple): Location tuple (filename, lineno, function). 

133 """ 

134 _collected_warnings.append( 

135 RecapEvent( 

136 nodeid=nodeid, 

137 when=when, 

138 message=str(warning_message.message), 

139 category=getattr(warning_message.category, "__name__", str(warning_message.category)), 

140 filename=warning_message.filename, 

141 lineno=warning_message.lineno, 

142 location=location, 

143 ) 

144 ) 

145 

146 

147@pytest.hookimpl(hookwrapper=True) 

148def pytest_terminal_summary(terminalreporter: TerminalReporter, exitstatus: int, config: Config) -> None: 

149 """Hook into pytest's terminal summary to collect test results, errors, warnings, and write recap file. 

150 

151 Args: 

152 terminalreporter (TerminalReporter): The pytest terminal reporter object. 

153 exitstatus (int): Exit status of the pytest session. 

154 config (Config): The pytest config object. 

155 """ 

156 yield 

157 

158 if not getattr(config, "_recap_enabled", False): 

159 return 

160 

161 test_results_tuple: Tuple[List[TestResult], datetime, datetime] = collect_test_results_and_session_times( 

162 terminalreporter 

163 ) 

164 test_results, session_start, session_end = test_results_tuple 

165 rerun_groups: List[RerunTestGroup] = build_rerun_groups(test_results) 

166 

167 errors = [ 

168 RecapEvent( 

169 nodeid=getattr(rep, "nodeid", None), 

170 when=getattr(rep, "when", None), 

171 outcome=getattr(rep, "outcome", None), 

172 longrepr=str(getattr(rep, "longrepr", "")), 

173 sections=list(getattr(rep, "sections", [])), 

174 keywords=list(getattr(rep, "keywords", [])), 

175 # message, category, filename, lineno, location use defaults 

176 ) 

177 for rep in terminalreporter.stats.get("error", []) 

178 ] 

179 

180 warnings = _collected_warnings.copy() 

181 

182 session: TestSession = build_recap_session( 

183 test_results, session_start, session_end, rerun_groups, errors, warnings, terminalreporter, config 

184 ) 

185 # Print summary of warnings and errors using RecapEvent helpers 

186 warning_count = sum(bool(w.is_warning()) for w in warnings) 

187 error_count = sum(bool(e.is_error()) for e in errors) 

188 terminalreporter.write_sep("-", f"Recap: {warning_count} warnings, {error_count} errors collected") 

189 

190 # Optionally, print details 

191 if warning_count > 0: 

192 terminalreporter.write_line("\nWarnings:") 

193 for w in warnings: 

194 if w.is_warning(): 

195 terminalreporter.write_line(f" {w.filename}:{w.lineno} [{w.category}] {w.message}") 

196 if error_count > 0: 

197 terminalreporter.write_line("\nErrors:") 

198 for e in errors: 

199 if e.is_error(): 

200 terminalreporter.write_line(f" {e.nodeid} [{e.when}] {e.longrepr}") 

201 

202 write_recap_file(session, getattr(config, "_recap_destination", None), terminalreporter) 

203 

204 

205# --- pytest-recap-specific functions, only used internally --- # 

206def to_datetime(val: Optional[float]) -> Optional[datetime]: 

207 """Convert a timestamp to a datetime object. 

208 

209 Args: 

210 val (Optional[float]): The timestamp to convert. 

211 

212 Returns: 

213 Optional[datetime]: The datetime object, or None if the timestamp is None. 

214 """ 

215 return datetime.fromtimestamp(val, timezone.utc) if val is not None else None 

216 

217 

218def collect_test_results_and_session_times( 

219 terminalreporter: TerminalReporter, 

220) -> Tuple[List[TestResult], datetime, datetime]: 

221 """Collect test results and session times from the terminal reporter. 

222 

223 Args: 

224 terminalreporter (TerminalReporter): The terminal reporter object. 

225 

226 Returns: 

227 tuple: A tuple containing the list of test results, session start time, and session end time. 

228 """ 

229 stats: Dict[str, List[TestReport]] = terminalreporter.stats 

230 test_results: List[TestResult] = [] 

231 session_start: Optional[datetime] = None 

232 session_end: Optional[datetime] = None 

233 

234 for outcome, report_list in stats.items(): 

235 # Skip orocessing '' outcomes (which are for setup phase), and 'warnings' 

236 # (which we count warnings elsewhere, in pytest_warning_recorded) 

237 if not outcome or outcome == "warnings": 

238 continue 

239 

240 for report in report_list: 

241 if not isinstance(report, TestReport): 

242 continue 

243 

244 # Only process 'call' phase, and 'setup'/'teardown' phases for failed, error, or skipped tests 

245 # TODO: why did i do this again? 

246 if report.when == "call" or ( 

247 report.when in ("setup", "teardown") and report.outcome in ("failed", "error", "skipped") 

248 ): 

249 # Get start and end times from TestReport, and store them as datetime objects 

250 report_time = to_datetime(getattr(report, "start", None) or getattr(report, "starttime", None)) 

251 report_end = to_datetime(getattr(report, "stop", None) or getattr(report, "stoptime", None)) 

252 

253 # Update session start and end times if necessary 

254 if session_start is None or (report_time and report_time < session_start): 

255 session_start = report_time 

256 if session_end is None or (report_end and report_end > session_end): 

257 session_end = report_end 

258 

259 # longrepr can be a string, and object or None; whereas capstdout, capstderr, and caplog 

260 # are always strings 

261 longrepr = getattr(report, "longrepr", "") 

262 longreprtext = str(longrepr) if longrepr is not None else "" 

263 capstdout = getattr(report, "capstdout", "") 

264 capstderr = getattr(report, "capstderr", "") 

265 caplog = getattr(report, "caplog", "") 

266 

267 # Create TestResult object and append to list to return 

268 test_results.append( 

269 { 

270 "nodeid": report.nodeid, 

271 "outcome": outcome, 

272 "start_time": report_time, 

273 "stop_time": report_end, 

274 "longreprtext": longreprtext, 

275 "capstdout": capstdout, 

276 "capstderr": capstderr, 

277 "caplog": caplog, 

278 } 

279 ) 

280 

281 # Set session start and end times to either the times found in the test results or the current time 

282 session_start = session_start or datetime.now(timezone.utc) 

283 session_end = session_end or datetime.now(timezone.utc) 

284 

285 return test_results, session_start, session_end 

286 

287 

288def build_rerun_groups(test_results: List[TestResult]) -> List[RerunTestGroup]: 

289 """Build a list of RerunTestGroup objects from a list of test results. 

290 

291 Args: 

292 test_results (list): List of TestResult objects. 

293 

294 Returns: 

295 list: List of RerunTestGroup objects, each containing reruns for a nodeid. 

296 

297 """ 

298 test_result_objs = [ 

299 TestResult( 

300 nodeid=test_result["nodeid"], 

301 outcome=test_result["outcome"], 

302 longreprtext=test_result["longreprtext"], 

303 start_time=test_result["start_time"], 

304 stop_time=test_result["stop_time"], 

305 ) 

306 for test_result in test_results 

307 ] 

308 rerun_test_groups: Dict[str, RerunTestGroup] = {} 

309 for test_result in test_result_objs: 

310 if test_result.nodeid not in rerun_test_groups: 

311 rerun_test_groups[test_result.nodeid] = RerunTestGroup(nodeid=test_result.nodeid) 

312 rerun_test_groups[test_result.nodeid].add_test(test_result) 

313 return [group for group in rerun_test_groups.values() if len(group.tests) > 1] 

314 

315 

316def parse_dict_option( 

317 option_value: str, 

318 default: dict, 

319 option_name: str, 

320 terminalreporter: TerminalReporter, 

321 envvar: str = None, 

322 source: str = None, 

323) -> dict: 

324 """Parse a recap option string value into a Python dict. 

325 Supports both JSON and Python dict literal formats. 

326 Returns the provided default if parsing fails. 

327 """ 

328 if not option_value: 

329 return default 

330 try: 

331 return json.loads(option_value) 

332 except Exception: 

333 pass 

334 try: 

335 return ast.literal_eval(option_value) 

336 except Exception as e: 

337 src = f" from {source}" if source else "" 

338 env_info = f" (env var: {envvar})" if envvar else "" 

339 msg = ( 

340 f"WARNING: Invalid RECAP_{option_name.upper()} value{src}{env_info}: {option_value!r}. " 

341 f"Could not parse as dict: {e}. Using default." 

342 ) 

343 if terminalreporter: 

344 terminalreporter.write_line(msg) 

345 else: 

346 print(msg) 

347 return default 

348 

349 

350def get_recap_option(config: Config, opt: str, ini: str, envvar: str, default: str = "") -> str: 

351 """Retrieve the raw option value for a recap option from CLI, environment variable, pytest.ini, or default. 

352 This function is responsible for determining the source (precedence order: CLI > env > ini > default), 

353 but does NOT parse the value into a dict—it always returns a string. 

354 

355 Args: 

356 config (Config): The pytest Config object. 

357 opt (str): The option name (the pytest cmd-line flag). 

358 ini (str): The ini option name (the pytest ini file option). 

359 envvar (str): The environment variable name. 

360 default (str, optional): The default value. Defaults to "". 

361 

362 Returns: 

363 str: The option value to use. 

364 """ 

365 cli_val = getattr(config.option, opt, None) 

366 if cli_val is not None and str(cli_val).strip() != "": 

367 return cli_val 

368 env_val = os.environ.get(envvar) 

369 if env_val is not None and str(env_val).strip() != "": 

370 return env_val 

371 ini_val = config.getini(ini) 

372 # If ini_val is a list (possible for ini options), join to string 

373 if isinstance(ini_val, list): 

374 ini_val = " ".join(str(x) for x in ini_val).strip() 

375 if ini_val is not None and str(ini_val).strip() != "": 

376 return ini_val.strip() 

377 return default 

378 

379 

380def build_recap_session( 

381 test_results: List[TestResult], 

382 session_start: datetime, 

383 session_end: datetime, 

384 rerun_groups: List[RerunTestGroup], 

385 errors: List[Dict], 

386 warnings: List[Dict], 

387 terminalreporter: TerminalReporter, 

388 config: Config, 

389) -> TestSession: 

390 """Build a TestSession object summarizing the test session. 

391 

392 Args: 

393 test_results (list): List of test result dicts. 

394 session_start (datetime): Session start time. 

395 session_end (datetime): Session end time. 

396 rerun_groups (list): List of RerunTestGroup objects. 

397 terminalreporter: Pytest terminal reporter. 

398 config: Pytest config object. 

399 

400 Returns: 

401 TestSession: The constructed test session object. 

402 

403 Notes: 

404 - session_tags, system_under_test, and testing_system can be set via CLI, env, or pytest.ini. 

405 

406 """ 

407 session_timestamp: str = session_start.strftime("%Y%m%d-%H%M%S") 

408 session_id: str = f"{session_timestamp}-{str(uuid.uuid4())[:8]}".lower() 

409 

410 # Session tags 

411 session_tags = parse_dict_option( 

412 get_recap_option(config, "recap_session_tags", "recap_session_tags", "RECAP_SESSION_TAGS"), 

413 {}, 

414 "session_tags", 

415 terminalreporter, 

416 ) 

417 if not isinstance(session_tags, dict): 

418 session_tags = {} 

419 

420 # System Under Test 

421 system_under_test = parse_dict_option( 

422 get_recap_option(config, "recap_system_under_test", "recap_system_under_test", "RECAP_SYSTEM_UNDER_TEST"), 

423 {"name": "pytest-recap"}, 

424 "system_under_test", 

425 terminalreporter, 

426 envvar="RECAP_SYSTEM_UNDER_TEST", 

427 ) 

428 if not isinstance(system_under_test, dict): 

429 system_under_test = {"name": "pytest-recap"} 

430 

431 # Testing System 

432 default_testing_system = { 

433 "hostname": socket.gethostname(), 

434 "platform": platform.platform(), 

435 "python_version": platform.python_version(), 

436 "pytest_version": pytest.__version__, 

437 "environment": os.environ.get("RECAP_ENV", "test"), 

438 } 

439 testing_system = parse_dict_option( 

440 get_recap_option(config, "recap_testing_system", "recap_testing_system", "RECAP_TESTING_SYSTEM"), 

441 default_testing_system, 

442 "testing_system", 

443 terminalreporter, 

444 envvar="RECAP_TESTING_SYSTEM", 

445 ) 

446 if not isinstance(testing_system, dict): 

447 testing_system = default_testing_system 

448 

449 # Session tags 

450 session_tags = parse_dict_option( 

451 get_recap_option(config, "recap_session_tags", "recap_session_tags", "RECAP_SESSION_TAGS"), 

452 {}, 

453 "session_tags", 

454 terminalreporter, 

455 ) 

456 if not isinstance(session_tags, dict): 

457 session_tags = {} 

458 

459 # Session stats 

460 test_result_objs: List[TestResult] = [TestResult.from_dict(test_result) for test_result in test_results] 

461 session_stats = TestSessionStats(test_result_objs, warnings_count=len(warnings)) 

462 

463 # Build and return session 

464 session = TestSession( 

465 session_id=session_id, 

466 session_tags=session_tags, 

467 system_under_test=system_under_test, 

468 testing_system=testing_system, 

469 session_start_time=session_start, 

470 session_stop_time=session_end, 

471 test_results=test_result_objs, 

472 rerun_test_groups=rerun_groups, 

473 errors=errors, 

474 warnings=warnings, 

475 session_stats=session_stats, 

476 ) 

477 return session 

478 

479 

480def write_recap_file(session: TestSession, destination: str, terminalreporter: TerminalReporter): 

481 """Write the recap session data to a file in JSON format. 

482 

483 Args: 

484 session (TestSession): The session recap object to write. 

485 destination (str): File or directory path for output. If None, a default location is used. 

486 terminalreporter: Pytest terminal reporter for output. 

487 

488 Raises: 

489 Exception: If writing the recap file fails. 

490 

491 """ 

492 recap_data: Dict = session.to_dict() 

493 now: datetime = datetime.now(timezone.utc) 

494 pretty: bool = getattr(getattr(terminalreporter, "config", None), "_recap_pretty", False) 

495 indent: Optional[int] = 2 if pretty else None 

496 json_bytes: bytes = json.dumps(recap_data, indent=indent).encode("utf-8") 

497 

498 # Cloud URI detection and dispatch 

499 if destination and ( 

500 destination.startswith("s3://") 

501 or destination.startswith("gs://") 

502 or destination.startswith("azure://") 

503 or destination.startswith("https://") 

504 ): 

505 try: 

506 upload_to_cloud(destination, json_bytes) 

507 filepath = destination 

508 except Exception as e: 

509 terminalreporter.write_line(f"RECAP PLUGIN ERROR (cloud upload): {e}") 

510 filepath = destination # Still print the path for test assertions 

511 else: 

512 # Determine the output file path (local) 

513 if destination: 

514 if os.path.isdir(destination) or destination.endswith("/"): 

515 os.makedirs(destination, exist_ok=True) 

516 filename = f"{now.strftime('%Y%m%d-%H%M%S')}_{getattr(session, 'system_under_test', {}).get('name', 'sut')}.json" 

517 filepath = os.path.join(destination, filename) 

518 else: 

519 filepath = destination 

520 parent_dir = os.path.dirname(filepath) 

521 if parent_dir: 

522 os.makedirs(parent_dir, exist_ok=True) 

523 else: 

524 base_dir = os.environ.get("SESSION_WRITE_BASE_DIR", os.path.expanduser("~/.pytest_recap_sessions")) 

525 base_dir = os.path.abspath(base_dir) 

526 date_dir = os.path.join(base_dir, now.strftime("%Y/%m")) 

527 os.makedirs(date_dir, exist_ok=True) 

528 filename = ( 

529 f"{now.strftime('%Y%m%d-%H%M%S')}_{getattr(session, 'system_under_test', {}).get('name', 'sut')}.json" 

530 ) 

531 filepath = os.path.join(date_dir, filename) 

532 filepath = os.path.abspath(filepath) 

533 try: 

534 storage = JSONStorage(filepath) 

535 # Pass indent to storage for pretty/minified output 

536 storage.save_single_session(recap_data, indent=indent) 

537 except Exception as e: 

538 terminalreporter.write_line(f"RECAP PLUGIN ERROR: {e}") 

539 raise 

540 

541 # Write recap file path/URI to terminal 

542 terminalreporter.write_sep("=", "pytest-recap") 

543 BLUE = "\033[34m" 

544 RESET = "\033[0m" 

545 

546 # Print cloud URI directly if applicable, else absolute file path 

547 def is_cloud_uri(uri): 

548 return isinstance(uri, str) and ( 

549 uri.startswith("s3://") or uri.startswith("gs://") or uri.startswith("azure://") 

550 ) 

551 

552 recap_uri = filepath if is_cloud_uri(filepath) else os.path.abspath(filepath) 

553 blue_path = f"Recap JSON written to: {BLUE}{recap_uri}{RESET}" 

554 terminalreporter.write_line(blue_path)