Coverage for pytest_recap/plugin.py: 71%
217 statements
« prev ^ index » next coverage.py v7.8.0, created at 2025-06-16 00:19 -0600
« prev ^ index » next coverage.py v7.8.0, created at 2025-06-16 00:19 -0600
1import ast
2import json
3import logging
4import os
5import platform
6import socket
7import uuid
8from datetime import datetime, timezone
9from typing import Dict, Generator, List, Optional, Tuple
10from warnings import WarningMessage
12import pytest
13from _pytest.config import Config
14from _pytest.config.argparsing import Parser
15from _pytest.nodes import Item
16from _pytest.reports import TestReport
17from _pytest.runner import CallInfo
18from _pytest.terminal import TerminalReporter
20from pytest_recap.cloud import upload_to_cloud
21from pytest_recap.models import RecapEvent, RerunTestGroup, TestResult, TestSession, TestSessionStats
22from pytest_recap.storage import JSONStorage
24# --- Global warning collection. This is required because Pytest hook pytest-warning-recorded
25# does not pass the Config object, so it cannot be used to store warnings.
26_collected_warnings = []
29# --- pytest hooks --- #
30def pytest_addoption(parser: Parser) -> None:
31 """Add command line options for pytest-recap, supporting environment variable defaults.
33 Args:
34 parser (Parser): The pytest parser object.
35 """
36 group = parser.getgroup("Pytest Recap")
37 recap_env = os.environ.get("RECAP_ENABLE", "0").lower()
38 recap_default = recap_env in ("1", "true", "yes", "y")
39 group.addoption(
40 "--recap",
41 action="store_true",
42 default=recap_default,
43 help="Enable pytest-recap plugin (or set environment variable RECAP_ENABLE)",
44 )
45 recap_dest_env = os.environ.get("RECAP_DESTINATION")
46 if recap_dest_env:
47 recap_dest_default = recap_dest_env
48 else:
49 timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
50 default_dir = os.path.expanduser("~/.pytest-recap-sessions")
51 os.makedirs(default_dir, exist_ok=True)
52 recap_dest_default = os.path.join(default_dir, f"{timestamp}-recap.json")
53 group.addoption(
54 "--recap-destination",
55 action="store",
56 default=recap_dest_default,
57 help="Specify pytest-recap storage destination (filepath) (or set environment variable RECAP_DESTINATION)",
58 )
59 group.addoption(
60 "--recap-system-under-test",
61 action="store",
62 default=None,
63 help="JSON or Python dict string for system under test metadata (or set RECAP_SYSTEM_UNDER_TEST)",
64 )
65 group.addoption(
66 "--recap-testing-system",
67 action="store",
68 default=None,
69 help="JSON or Python dict string for testing system metadata (or set RECAP_TESTING_SYSTEM)",
70 )
71 group.addoption(
72 "--recap-session-tags",
73 action="store",
74 default=None,
75 help="JSON or Python dict string for session tags (or set RECAP_SESSION_TAGS)",
76 )
77 group.addoption(
78 "--recap-pretty",
79 action="store_true",
80 default=None,
81 help="Pretty-print recap JSON output (or set RECAP_PRETTY=1, or ini: recap_pretty=1)",
82 )
84 parser.addini("recap_system_under_test", "System under test dict (JSON or Python dict string)", default="")
85 parser.addini("recap_testing_system", "Testing system dict (JSON or Python dict string)", default="")
86 parser.addini("recap_session_tags", "Session tags dict (JSON or Python dict string)", default="")
87 parser.addini("recap_pretty", "Pretty-print recap JSON output (1 for pretty, 0 for minified)", default="0")
90def pytest_configure(config: Config) -> None:
91 """Configure pytest-recap plugin.
93 Args:
94 config (Config): The pytest Config object.
95 """
96 config._recap_enabled: bool = config.getoption("--recap")
97 config._recap_destination: str = config.getoption("--recap-destination")
98 pretty = get_recap_option(config, "recap_pretty", "recap_pretty", "RECAP_PRETTY", default="0")
99 config._recap_pretty: bool = str(pretty).strip().lower() in {"1", "true", "yes", "y"}
102def pytest_sessionstart(session):
103 """Reset collected warnings at the start of each test session."""
104 global _collected_warnings
105 _collected_warnings = []
108@pytest.hookimpl(hookwrapper=True)
109def pytest_runtest_makereport(item: Item, call: CallInfo) -> Generator:
110 """Hook into pytest's test report generation to generate start and stop times, if not set already."""
111 outcome = yield
113 logger = logging.getLogger(__name__)
115 report: TestReport = outcome.get_result()
116 if report.when == "setup" and not hasattr(report, "start"):
117 logger.warning(f"Setting start time for {report.nodeid} since it was not set previously")
118 setattr(report, "start", datetime.now(timezone.utc).timestamp())
120 if report.when == "teardown" and not hasattr(report, "stop"):
121 logger.warning(f"Setting stop time for {report.nodeid} since it was not set previously")
122 setattr(report, "stop", datetime.now(timezone.utc).timestamp())
125def pytest_warning_recorded(warning_message: WarningMessage, when: str, nodeid: str, location: tuple):
126 """Collect warnings during pytest session for recap reporting.
128 Args:
129 warning_message (WarningMessage): The warning message object.
130 when (str): When the warning was recorded (e.g., 'call', 'setup', etc.).
131 nodeid (str): Node ID of the test (if any).
132 location (tuple): Location tuple (filename, lineno, function).
133 """
134 _collected_warnings.append(
135 RecapEvent(
136 nodeid=nodeid,
137 when=when,
138 message=str(warning_message.message),
139 category=getattr(warning_message.category, "__name__", str(warning_message.category)),
140 filename=warning_message.filename,
141 lineno=warning_message.lineno,
142 location=location,
143 )
144 )
147@pytest.hookimpl(hookwrapper=True)
148def pytest_terminal_summary(terminalreporter: TerminalReporter, exitstatus: int, config: Config) -> None:
149 """Hook into pytest's terminal summary to collect test results, errors, warnings, and write recap file.
151 Args:
152 terminalreporter (TerminalReporter): The pytest terminal reporter object.
153 exitstatus (int): Exit status of the pytest session.
154 config (Config): The pytest config object.
155 """
156 yield
158 if not getattr(config, "_recap_enabled", False):
159 return
161 test_results_tuple: Tuple[List[TestResult], datetime, datetime] = collect_test_results_and_session_times(
162 terminalreporter
163 )
164 test_results, session_start, session_end = test_results_tuple
165 rerun_groups: List[RerunTestGroup] = build_rerun_groups(test_results)
167 errors = [
168 RecapEvent(
169 nodeid=getattr(rep, "nodeid", None),
170 when=getattr(rep, "when", None),
171 outcome=getattr(rep, "outcome", None),
172 longrepr=str(getattr(rep, "longrepr", "")),
173 sections=list(getattr(rep, "sections", [])),
174 keywords=list(getattr(rep, "keywords", [])),
175 # message, category, filename, lineno, location use defaults
176 )
177 for rep in terminalreporter.stats.get("error", [])
178 ]
180 warnings = _collected_warnings.copy()
182 session: TestSession = build_recap_session(
183 test_results, session_start, session_end, rerun_groups, errors, warnings, terminalreporter, config
184 )
185 # Print summary of warnings and errors using RecapEvent helpers
186 warning_count = sum(bool(w.is_warning()) for w in warnings)
187 error_count = sum(bool(e.is_error()) for e in errors)
188 terminalreporter.write_sep("-", f"Recap: {warning_count} warnings, {error_count} errors collected")
190 # Optionally, print details
191 if warning_count > 0:
192 terminalreporter.write_line("\nWarnings:")
193 for w in warnings:
194 if w.is_warning():
195 terminalreporter.write_line(f" {w.filename}:{w.lineno} [{w.category}] {w.message}")
196 if error_count > 0:
197 terminalreporter.write_line("\nErrors:")
198 for e in errors:
199 if e.is_error():
200 terminalreporter.write_line(f" {e.nodeid} [{e.when}] {e.longrepr}")
202 write_recap_file(session, getattr(config, "_recap_destination", None), terminalreporter)
205# --- pytest-recap-specific functions, only used internally --- #
206def to_datetime(val: Optional[float]) -> Optional[datetime]:
207 """Convert a timestamp to a datetime object.
209 Args:
210 val (Optional[float]): The timestamp to convert.
212 Returns:
213 Optional[datetime]: The datetime object, or None if the timestamp is None.
214 """
215 return datetime.fromtimestamp(val, timezone.utc) if val is not None else None
218def collect_test_results_and_session_times(
219 terminalreporter: TerminalReporter,
220) -> Tuple[List[TestResult], datetime, datetime]:
221 """Collect test results and session times from the terminal reporter.
223 Args:
224 terminalreporter (TerminalReporter): The terminal reporter object.
226 Returns:
227 tuple: A tuple containing the list of test results, session start time, and session end time.
228 """
229 stats: Dict[str, List[TestReport]] = terminalreporter.stats
230 test_results: List[TestResult] = []
231 session_start: Optional[datetime] = None
232 session_end: Optional[datetime] = None
234 for outcome, report_list in stats.items():
235 # Skip orocessing '' outcomes (which are for setup phase), and 'warnings'
236 # (which we count warnings elsewhere, in pytest_warning_recorded)
237 if not outcome or outcome == "warnings":
238 continue
240 for report in report_list:
241 if not isinstance(report, TestReport):
242 continue
244 # Only process 'call' phase, and 'setup'/'teardown' phases for failed, error, or skipped tests
245 # TODO: why did i do this again?
246 if report.when == "call" or (
247 report.when in ("setup", "teardown") and report.outcome in ("failed", "error", "skipped")
248 ):
249 # Get start and end times from TestReport, and store them as datetime objects
250 report_time = to_datetime(getattr(report, "start", None) or getattr(report, "starttime", None))
251 report_end = to_datetime(getattr(report, "stop", None) or getattr(report, "stoptime", None))
253 # Update session start and end times if necessary
254 if session_start is None or (report_time and report_time < session_start):
255 session_start = report_time
256 if session_end is None or (report_end and report_end > session_end):
257 session_end = report_end
259 # longrepr can be a string, and object or None; whereas capstdout, capstderr, and caplog
260 # are always strings
261 longrepr = getattr(report, "longrepr", "")
262 longreprtext = str(longrepr) if longrepr is not None else ""
263 capstdout = getattr(report, "capstdout", "")
264 capstderr = getattr(report, "capstderr", "")
265 caplog = getattr(report, "caplog", "")
267 # Create TestResult object and append to list to return
268 test_results.append(
269 {
270 "nodeid": report.nodeid,
271 "outcome": outcome,
272 "start_time": report_time,
273 "stop_time": report_end,
274 "longreprtext": longreprtext,
275 "capstdout": capstdout,
276 "capstderr": capstderr,
277 "caplog": caplog,
278 }
279 )
281 # Set session start and end times to either the times found in the test results or the current time
282 session_start = session_start or datetime.now(timezone.utc)
283 session_end = session_end or datetime.now(timezone.utc)
285 return test_results, session_start, session_end
288def build_rerun_groups(test_results: List[TestResult]) -> List[RerunTestGroup]:
289 """Build a list of RerunTestGroup objects from a list of test results.
291 Args:
292 test_results (list): List of TestResult objects.
294 Returns:
295 list: List of RerunTestGroup objects, each containing reruns for a nodeid.
297 """
298 test_result_objs = [
299 TestResult(
300 nodeid=test_result["nodeid"],
301 outcome=test_result["outcome"],
302 longreprtext=test_result["longreprtext"],
303 start_time=test_result["start_time"],
304 stop_time=test_result["stop_time"],
305 )
306 for test_result in test_results
307 ]
308 rerun_test_groups: Dict[str, RerunTestGroup] = {}
309 for test_result in test_result_objs:
310 if test_result.nodeid not in rerun_test_groups:
311 rerun_test_groups[test_result.nodeid] = RerunTestGroup(nodeid=test_result.nodeid)
312 rerun_test_groups[test_result.nodeid].add_test(test_result)
313 return [group for group in rerun_test_groups.values() if len(group.tests) > 1]
316def parse_dict_option(
317 option_value: str,
318 default: dict,
319 option_name: str,
320 terminalreporter: TerminalReporter,
321 envvar: str = None,
322 source: str = None,
323) -> dict:
324 """Parse a recap option string value into a Python dict.
325 Supports both JSON and Python dict literal formats.
326 Returns the provided default if parsing fails.
327 """
328 if not option_value:
329 return default
330 try:
331 return json.loads(option_value)
332 except Exception:
333 pass
334 try:
335 return ast.literal_eval(option_value)
336 except Exception as e:
337 src = f" from {source}" if source else ""
338 env_info = f" (env var: {envvar})" if envvar else ""
339 msg = (
340 f"WARNING: Invalid RECAP_{option_name.upper()} value{src}{env_info}: {option_value!r}. "
341 f"Could not parse as dict: {e}. Using default."
342 )
343 if terminalreporter:
344 terminalreporter.write_line(msg)
345 else:
346 print(msg)
347 return default
350def get_recap_option(config: Config, opt: str, ini: str, envvar: str, default: str = "") -> str:
351 """Retrieve the raw option value for a recap option from CLI, environment variable, pytest.ini, or default.
352 This function is responsible for determining the source (precedence order: CLI > env > ini > default),
353 but does NOT parse the value into a dict—it always returns a string.
355 Args:
356 config (Config): The pytest Config object.
357 opt (str): The option name (the pytest cmd-line flag).
358 ini (str): The ini option name (the pytest ini file option).
359 envvar (str): The environment variable name.
360 default (str, optional): The default value. Defaults to "".
362 Returns:
363 str: The option value to use.
364 """
365 cli_val = getattr(config.option, opt, None)
366 if cli_val is not None and str(cli_val).strip() != "":
367 return cli_val
368 env_val = os.environ.get(envvar)
369 if env_val is not None and str(env_val).strip() != "":
370 return env_val
371 ini_val = config.getini(ini)
372 # If ini_val is a list (possible for ini options), join to string
373 if isinstance(ini_val, list):
374 ini_val = " ".join(str(x) for x in ini_val).strip()
375 if ini_val is not None and str(ini_val).strip() != "":
376 return ini_val.strip()
377 return default
380def build_recap_session(
381 test_results: List[TestResult],
382 session_start: datetime,
383 session_end: datetime,
384 rerun_groups: List[RerunTestGroup],
385 errors: List[Dict],
386 warnings: List[Dict],
387 terminalreporter: TerminalReporter,
388 config: Config,
389) -> TestSession:
390 """Build a TestSession object summarizing the test session.
392 Args:
393 test_results (list): List of test result dicts.
394 session_start (datetime): Session start time.
395 session_end (datetime): Session end time.
396 rerun_groups (list): List of RerunTestGroup objects.
397 terminalreporter: Pytest terminal reporter.
398 config: Pytest config object.
400 Returns:
401 TestSession: The constructed test session object.
403 Notes:
404 - session_tags, system_under_test, and testing_system can be set via CLI, env, or pytest.ini.
406 """
407 session_timestamp: str = session_start.strftime("%Y%m%d-%H%M%S")
408 session_id: str = f"{session_timestamp}-{str(uuid.uuid4())[:8]}".lower()
410 # Session tags
411 session_tags = parse_dict_option(
412 get_recap_option(config, "recap_session_tags", "recap_session_tags", "RECAP_SESSION_TAGS"),
413 {},
414 "session_tags",
415 terminalreporter,
416 )
417 if not isinstance(session_tags, dict):
418 session_tags = {}
420 # System Under Test
421 system_under_test = parse_dict_option(
422 get_recap_option(config, "recap_system_under_test", "recap_system_under_test", "RECAP_SYSTEM_UNDER_TEST"),
423 {"name": "pytest-recap"},
424 "system_under_test",
425 terminalreporter,
426 envvar="RECAP_SYSTEM_UNDER_TEST",
427 )
428 if not isinstance(system_under_test, dict):
429 system_under_test = {"name": "pytest-recap"}
431 # Testing System
432 default_testing_system = {
433 "hostname": socket.gethostname(),
434 "platform": platform.platform(),
435 "python_version": platform.python_version(),
436 "pytest_version": pytest.__version__,
437 "environment": os.environ.get("RECAP_ENV", "test"),
438 }
439 testing_system = parse_dict_option(
440 get_recap_option(config, "recap_testing_system", "recap_testing_system", "RECAP_TESTING_SYSTEM"),
441 default_testing_system,
442 "testing_system",
443 terminalreporter,
444 envvar="RECAP_TESTING_SYSTEM",
445 )
446 if not isinstance(testing_system, dict):
447 testing_system = default_testing_system
449 # Session tags
450 session_tags = parse_dict_option(
451 get_recap_option(config, "recap_session_tags", "recap_session_tags", "RECAP_SESSION_TAGS"),
452 {},
453 "session_tags",
454 terminalreporter,
455 )
456 if not isinstance(session_tags, dict):
457 session_tags = {}
459 # Session stats
460 test_result_objs: List[TestResult] = [TestResult.from_dict(test_result) for test_result in test_results]
461 session_stats = TestSessionStats(test_result_objs, warnings_count=len(warnings))
463 # Build and return session
464 session = TestSession(
465 session_id=session_id,
466 session_tags=session_tags,
467 system_under_test=system_under_test,
468 testing_system=testing_system,
469 session_start_time=session_start,
470 session_stop_time=session_end,
471 test_results=test_result_objs,
472 rerun_test_groups=rerun_groups,
473 errors=errors,
474 warnings=warnings,
475 session_stats=session_stats,
476 )
477 return session
480def write_recap_file(session: TestSession, destination: str, terminalreporter: TerminalReporter):
481 """Write the recap session data to a file in JSON format.
483 Args:
484 session (TestSession): The session recap object to write.
485 destination (str): File or directory path for output. If None, a default location is used.
486 terminalreporter: Pytest terminal reporter for output.
488 Raises:
489 Exception: If writing the recap file fails.
491 """
492 recap_data: Dict = session.to_dict()
493 now: datetime = datetime.now(timezone.utc)
494 pretty: bool = getattr(getattr(terminalreporter, "config", None), "_recap_pretty", False)
495 indent: Optional[int] = 2 if pretty else None
496 json_bytes: bytes = json.dumps(recap_data, indent=indent).encode("utf-8")
498 # Cloud URI detection and dispatch
499 if destination and (
500 destination.startswith("s3://")
501 or destination.startswith("gs://")
502 or destination.startswith("azure://")
503 or destination.startswith("https://")
504 ):
505 try:
506 upload_to_cloud(destination, json_bytes)
507 filepath = destination
508 except Exception as e:
509 terminalreporter.write_line(f"RECAP PLUGIN ERROR (cloud upload): {e}")
510 filepath = destination # Still print the path for test assertions
511 else:
512 # Determine the output file path (local)
513 if destination:
514 if os.path.isdir(destination) or destination.endswith("/"):
515 os.makedirs(destination, exist_ok=True)
516 filename = f"{now.strftime('%Y%m%d-%H%M%S')}_{getattr(session, 'system_under_test', {}).get('name', 'sut')}.json"
517 filepath = os.path.join(destination, filename)
518 else:
519 filepath = destination
520 parent_dir = os.path.dirname(filepath)
521 if parent_dir:
522 os.makedirs(parent_dir, exist_ok=True)
523 else:
524 base_dir = os.environ.get("SESSION_WRITE_BASE_DIR", os.path.expanduser("~/.pytest_recap_sessions"))
525 base_dir = os.path.abspath(base_dir)
526 date_dir = os.path.join(base_dir, now.strftime("%Y/%m"))
527 os.makedirs(date_dir, exist_ok=True)
528 filename = (
529 f"{now.strftime('%Y%m%d-%H%M%S')}_{getattr(session, 'system_under_test', {}).get('name', 'sut')}.json"
530 )
531 filepath = os.path.join(date_dir, filename)
532 filepath = os.path.abspath(filepath)
533 try:
534 storage = JSONStorage(filepath)
535 # Pass indent to storage for pretty/minified output
536 storage.save_single_session(recap_data, indent=indent)
537 except Exception as e:
538 terminalreporter.write_line(f"RECAP PLUGIN ERROR: {e}")
539 raise
541 # Write recap file path/URI to terminal
542 terminalreporter.write_sep("=", "pytest-recap")
543 BLUE = "\033[34m"
544 RESET = "\033[0m"
546 # Print cloud URI directly if applicable, else absolute file path
547 def is_cloud_uri(uri):
548 return isinstance(uri, str) and (
549 uri.startswith("s3://") or uri.startswith("gs://") or uri.startswith("azure://")
550 )
552 recap_uri = filepath if is_cloud_uri(filepath) else os.path.abspath(filepath)
553 blue_path = f"Recap JSON written to: {BLUE}{recap_uri}{RESET}"
554 terminalreporter.write_line(blue_path)