Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import atexit 

2import contextlib 

3import fnmatch 

4import importlib.util 

5import itertools 

6import os 

7import shutil 

8import sys 

9import uuid 

10import warnings 

11from enum import Enum 

12from functools import partial 

13from os.path import expanduser 

14from os.path import expandvars 

15from os.path import isabs 

16from os.path import sep 

17from posixpath import sep as posix_sep 

18from types import ModuleType 

19from typing import Iterable 

20from typing import Iterator 

21from typing import Optional 

22from typing import Set 

23from typing import TypeVar 

24from typing import Union 

25 

26import py 

27 

28from _pytest.compat import assert_never 

29from _pytest.outcomes import skip 

30from _pytest.warning_types import PytestWarning 

31 

32if sys.version_info[:2] >= (3, 6): 

33 from pathlib import Path, PurePath 

34else: 

35 from pathlib2 import Path, PurePath 

36 

37__all__ = ["Path", "PurePath"] 

38 

39 

40LOCK_TIMEOUT = 60 * 60 * 3 

41 

42 

43_AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath) 

44 

45 

46def get_lock_path(path: _AnyPurePath) -> _AnyPurePath: 

47 return path.joinpath(".lock") 

48 

49 

50def ensure_reset_dir(path: Path) -> None: 

51 """ 

52 ensures the given path is an empty directory 

53 """ 

54 if path.exists(): 

55 rm_rf(path) 

56 path.mkdir() 

57 

58 

59def on_rm_rf_error(func, path: str, exc, *, start_path: Path) -> bool: 

60 """Handles known read-only errors during rmtree. 

61 

62 The returned value is used only by our own tests. 

63 """ 

64 exctype, excvalue = exc[:2] 

65 

66 # another process removed the file in the middle of the "rm_rf" (xdist for example) 

67 # more context: https://github.com/pytest-dev/pytest/issues/5974#issuecomment-543799018 

68 if isinstance(excvalue, FileNotFoundError): 

69 return False 

70 

71 if not isinstance(excvalue, PermissionError): 

72 warnings.warn( 

73 PytestWarning( 

74 "(rm_rf) error removing {}\n{}: {}".format(path, exctype, excvalue) 

75 ) 

76 ) 

77 return False 

78 

79 if func not in (os.rmdir, os.remove, os.unlink): 

80 if func not in (os.open,): 

81 warnings.warn( 

82 PytestWarning( 

83 "(rm_rf) unknown function {} when removing {}:\n{}: {}".format( 

84 func, path, exctype, excvalue 

85 ) 

86 ) 

87 ) 

88 return False 

89 

90 # Chmod + retry. 

91 import stat 

92 

93 def chmod_rw(p: str) -> None: 

94 mode = os.stat(p).st_mode 

95 os.chmod(p, mode | stat.S_IRUSR | stat.S_IWUSR) 

96 

97 # For files, we need to recursively go upwards in the directories to 

98 # ensure they all are also writable. 

99 p = Path(path) 

100 if p.is_file(): 

101 for parent in p.parents: 

102 chmod_rw(str(parent)) 

103 # stop when we reach the original path passed to rm_rf 

104 if parent == start_path: 

105 break 

106 chmod_rw(str(path)) 

107 

108 func(path) 

109 return True 

110 

111 

112def ensure_extended_length_path(path: Path) -> Path: 

113 """Get the extended-length version of a path (Windows). 

114 

115 On Windows, by default, the maximum length of a path (MAX_PATH) is 260 

116 characters, and operations on paths longer than that fail. But it is possible 

117 to overcome this by converting the path to "extended-length" form before 

118 performing the operation: 

119 https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation 

120 

121 On Windows, this function returns the extended-length absolute version of path. 

122 On other platforms it returns path unchanged. 

123 """ 

124 if sys.platform.startswith("win32"): 

125 path = path.resolve() 

126 path = Path(get_extended_length_path_str(str(path))) 

127 return path 

128 

129 

130def get_extended_length_path_str(path: str) -> str: 

131 """Converts to extended length path as a str""" 

132 long_path_prefix = "\\\\?\\" 

133 unc_long_path_prefix = "\\\\?\\UNC\\" 

134 if path.startswith((long_path_prefix, unc_long_path_prefix)): 

135 return path 

136 # UNC 

137 if path.startswith("\\\\"): 

138 return unc_long_path_prefix + path[2:] 

139 return long_path_prefix + path 

140 

141 

142def rm_rf(path: Path) -> None: 

143 """Remove the path contents recursively, even if some elements 

144 are read-only. 

145 """ 

146 path = ensure_extended_length_path(path) 

147 onerror = partial(on_rm_rf_error, start_path=path) 

148 shutil.rmtree(str(path), onerror=onerror) 

149 

150 

151def find_prefixed(root: Path, prefix: str) -> Iterator[Path]: 

152 """finds all elements in root that begin with the prefix, case insensitive""" 

153 l_prefix = prefix.lower() 

154 for x in root.iterdir(): 

155 if x.name.lower().startswith(l_prefix): 

156 yield x 

157 

158 

159def extract_suffixes(iter: Iterable[PurePath], prefix: str) -> Iterator[str]: 

160 """ 

161 :param iter: iterator over path names 

162 :param prefix: expected prefix of the path names 

163 :returns: the parts of the paths following the prefix 

164 """ 

165 p_len = len(prefix) 

166 for p in iter: 

167 yield p.name[p_len:] 

168 

169 

170def find_suffixes(root: Path, prefix: str) -> Iterator[str]: 

171 """combines find_prefixes and extract_suffixes 

172 """ 

173 return extract_suffixes(find_prefixed(root, prefix), prefix) 

174 

175 

176def parse_num(maybe_num) -> int: 

177 """parses number path suffixes, returns -1 on error""" 

178 try: 

179 return int(maybe_num) 

180 except ValueError: 

181 return -1 

182 

183 

184def _force_symlink( 

185 root: Path, target: Union[str, PurePath], link_to: Union[str, Path] 

186) -> None: 

187 """helper to create the current symlink 

188 

189 it's full of race conditions that are reasonably ok to ignore 

190 for the context of best effort linking to the latest test run 

191 

192 the presumption being that in case of much parallelism 

193 the inaccuracy is going to be acceptable 

194 """ 

195 current_symlink = root.joinpath(target) 

196 try: 

197 current_symlink.unlink() 

198 except OSError: 

199 pass 

200 try: 

201 current_symlink.symlink_to(link_to) 

202 except Exception: 

203 pass 

204 

205 

206def make_numbered_dir(root: Path, prefix: str) -> Path: 

207 """create a directory with an increased number as suffix for the given prefix""" 

208 for i in range(10): 

209 # try up to 10 times to create the folder 

210 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1) 

211 new_number = max_existing + 1 

212 new_path = root.joinpath("{}{}".format(prefix, new_number)) 

213 try: 

214 new_path.mkdir() 

215 except Exception: 

216 pass 

217 else: 

218 _force_symlink(root, prefix + "current", new_path) 

219 return new_path 

220 else: 

221 raise OSError( 

222 "could not create numbered dir with prefix " 

223 "{prefix} in {root} after 10 tries".format(prefix=prefix, root=root) 

224 ) 

225 

226 

227def create_cleanup_lock(p: Path) -> Path: 

228 """crates a lock to prevent premature folder cleanup""" 

229 lock_path = get_lock_path(p) 

230 try: 

231 fd = os.open(str(lock_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) 

232 except FileExistsError as e: 

233 raise OSError("cannot create lockfile in {path}".format(path=p)) from e 

234 else: 

235 pid = os.getpid() 

236 spid = str(pid).encode() 

237 os.write(fd, spid) 

238 os.close(fd) 

239 if not lock_path.is_file(): 

240 raise OSError("lock path got renamed after successful creation") 

241 return lock_path 

242 

243 

244def register_cleanup_lock_removal(lock_path: Path, register=atexit.register): 

245 """registers a cleanup function for removing a lock, by default on atexit""" 

246 pid = os.getpid() 

247 

248 def cleanup_on_exit(lock_path: Path = lock_path, original_pid: int = pid) -> None: 

249 current_pid = os.getpid() 

250 if current_pid != original_pid: 

251 # fork 

252 return 

253 try: 

254 lock_path.unlink() 

255 except OSError: 

256 pass 

257 

258 return register(cleanup_on_exit) 

259 

260 

261def maybe_delete_a_numbered_dir(path: Path) -> None: 

262 """removes a numbered directory if its lock can be obtained and it does not seem to be in use""" 

263 path = ensure_extended_length_path(path) 

264 lock_path = None 

265 try: 

266 lock_path = create_cleanup_lock(path) 

267 parent = path.parent 

268 

269 garbage = parent.joinpath("garbage-{}".format(uuid.uuid4())) 

270 path.rename(garbage) 

271 rm_rf(garbage) 

272 except OSError: 

273 # known races: 

274 # * other process did a cleanup at the same time 

275 # * deletable folder was found 

276 # * process cwd (Windows) 

277 return 

278 finally: 

279 # if we created the lock, ensure we remove it even if we failed 

280 # to properly remove the numbered dir 

281 if lock_path is not None: 

282 try: 

283 lock_path.unlink() 

284 except OSError: 

285 pass 

286 

287 

288def ensure_deletable(path: Path, consider_lock_dead_if_created_before: float) -> bool: 

289 """checks if `path` is deletable based on whether the lock file is expired""" 

290 if path.is_symlink(): 

291 return False 

292 lock = get_lock_path(path) 

293 try: 

294 if not lock.is_file(): 

295 return True 

296 except OSError: 

297 # we might not have access to the lock file at all, in this case assume 

298 # we don't have access to the entire directory (#7491). 

299 return False 

300 try: 

301 lock_time = lock.stat().st_mtime 

302 except Exception: 

303 return False 

304 else: 

305 if lock_time < consider_lock_dead_if_created_before: 

306 # wa want to ignore any errors while trying to remove the lock such as: 

307 # - PermissionDenied, like the file permissions have changed since the lock creation 

308 # - FileNotFoundError, in case another pytest process got here first. 

309 # and any other cause of failure. 

310 with contextlib.suppress(OSError): 

311 lock.unlink() 

312 return True 

313 return False 

314 

315 

316def try_cleanup(path: Path, consider_lock_dead_if_created_before: float) -> None: 

317 """tries to cleanup a folder if we can ensure it's deletable""" 

318 if ensure_deletable(path, consider_lock_dead_if_created_before): 

319 maybe_delete_a_numbered_dir(path) 

320 

321 

322def cleanup_candidates(root: Path, prefix: str, keep: int) -> Iterator[Path]: 

323 """lists candidates for numbered directories to be removed - follows py.path""" 

324 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1) 

325 max_delete = max_existing - keep 

326 paths = find_prefixed(root, prefix) 

327 paths, paths2 = itertools.tee(paths) 

328 numbers = map(parse_num, extract_suffixes(paths2, prefix)) 

329 for path, number in zip(paths, numbers): 

330 if number <= max_delete: 

331 yield path 

332 

333 

334def cleanup_numbered_dir( 

335 root: Path, prefix: str, keep: int, consider_lock_dead_if_created_before: float 

336) -> None: 

337 """cleanup for lock driven numbered directories""" 

338 for path in cleanup_candidates(root, prefix, keep): 

339 try_cleanup(path, consider_lock_dead_if_created_before) 

340 for path in root.glob("garbage-*"): 

341 try_cleanup(path, consider_lock_dead_if_created_before) 

342 

343 

344def make_numbered_dir_with_cleanup( 

345 root: Path, prefix: str, keep: int, lock_timeout: float 

346) -> Path: 

347 """creates a numbered dir with a cleanup lock and removes old ones""" 

348 e = None 

349 for i in range(10): 

350 try: 

351 p = make_numbered_dir(root, prefix) 

352 lock_path = create_cleanup_lock(p) 

353 register_cleanup_lock_removal(lock_path) 

354 except Exception as exc: 

355 e = exc 

356 else: 

357 consider_lock_dead_if_created_before = p.stat().st_mtime - lock_timeout 

358 # Register a cleanup for program exit 

359 atexit.register( 

360 cleanup_numbered_dir, 

361 root, 

362 prefix, 

363 keep, 

364 consider_lock_dead_if_created_before, 

365 ) 

366 return p 

367 assert e is not None 

368 raise e 

369 

370 

371def resolve_from_str(input: str, root: py.path.local) -> Path: 

372 assert not isinstance(input, Path), "would break on py2" 

373 rootpath = Path(root) 

374 input = expanduser(input) 

375 input = expandvars(input) 

376 if isabs(input): 

377 return Path(input) 

378 else: 

379 return rootpath.joinpath(input) 

380 

381 

382def fnmatch_ex(pattern: str, path) -> bool: 

383 """FNMatcher port from py.path.common which works with PurePath() instances. 

384 

385 The difference between this algorithm and PurePath.match() is that the latter matches "**" glob expressions 

386 for each part of the path, while this algorithm uses the whole path instead. 

387 

388 For example: 

389 "tests/foo/bar/doc/test_foo.py" matches pattern "tests/**/doc/test*.py" with this algorithm, but not with 

390 PurePath.match(). 

391 

392 This algorithm was ported to keep backward-compatibility with existing settings which assume paths match according 

393 this logic. 

394 

395 References: 

396 * https://bugs.python.org/issue29249 

397 * https://bugs.python.org/issue34731 

398 """ 

399 path = PurePath(path) 

400 iswin32 = sys.platform.startswith("win") 

401 

402 if iswin32 and sep not in pattern and posix_sep in pattern: 

403 # Running on Windows, the pattern has no Windows path separators, 

404 # and the pattern has one or more Posix path separators. Replace 

405 # the Posix path separators with the Windows path separator. 

406 pattern = pattern.replace(posix_sep, sep) 

407 

408 if sep not in pattern: 

409 name = path.name 

410 else: 

411 name = str(path) 

412 if path.is_absolute() and not os.path.isabs(pattern): 

413 pattern = "*{}{}".format(os.sep, pattern) 

414 return fnmatch.fnmatch(name, pattern) 

415 

416 

417def parts(s: str) -> Set[str]: 

418 parts = s.split(sep) 

419 return {sep.join(parts[: i + 1]) or sep for i in range(len(parts))} 

420 

421 

422def symlink_or_skip(src, dst, **kwargs): 

423 """Makes a symlink or skips the test in case symlinks are not supported.""" 

424 try: 

425 os.symlink(str(src), str(dst), **kwargs) 

426 except OSError as e: 

427 skip("symlinks not supported: {}".format(e)) 

428 

429 

430class ImportMode(Enum): 

431 """Possible values for `mode` parameter of `import_path`""" 

432 

433 prepend = "prepend" 

434 append = "append" 

435 importlib = "importlib" 

436 

437 

438class ImportPathMismatchError(ImportError): 

439 """Raised on import_path() if there is a mismatch of __file__'s. 

440 

441 This can happen when `import_path` is called multiple times with different filenames that has 

442 the same basename but reside in packages 

443 (for example "/tests1/test_foo.py" and "/tests2/test_foo.py"). 

444 """ 

445 

446 

447def import_path( 

448 p: Union[str, py.path.local, Path], 

449 *, 

450 mode: Union[str, ImportMode] = ImportMode.prepend 

451) -> ModuleType: 

452 """ 

453 Imports and returns a module from the given path, which can be a file (a module) or 

454 a directory (a package). 

455 

456 The import mechanism used is controlled by the `mode` parameter: 

457 

458 * `mode == ImportMode.prepend`: the directory containing the module (or package, taking 

459 `__init__.py` files into account) will be put at the *start* of `sys.path` before 

460 being imported with `__import__. 

461 

462 * `mode == ImportMode.append`: same as `prepend`, but the directory will be appended 

463 to the end of `sys.path`, if not already in `sys.path`. 

464 

465 * `mode == ImportMode.importlib`: uses more fine control mechanisms provided by `importlib` 

466 to import the module, which avoids having to use `__import__` and muck with `sys.path` 

467 at all. It effectively allows having same-named test modules in different places. 

468 

469 :raise ImportPathMismatchError: if after importing the given `path` and the module `__file__` 

470 are different. Only raised in `prepend` and `append` modes. 

471 """ 

472 mode = ImportMode(mode) 

473 

474 path = Path(str(p)) 

475 

476 if not path.exists(): 

477 raise ImportError(path) 

478 

479 if mode is ImportMode.importlib: 

480 module_name = path.stem 

481 

482 for meta_importer in sys.meta_path: 

483 spec = meta_importer.find_spec(module_name, [str(path.parent)]) 

484 if spec is not None: 

485 break 

486 else: 

487 spec = importlib.util.spec_from_file_location(module_name, str(path)) 

488 

489 if spec is None: 

490 raise ImportError( 

491 "Can't find module {} at location {}".format(module_name, str(path)) 

492 ) 

493 mod = importlib.util.module_from_spec(spec) 

494 spec.loader.exec_module(mod) # type: ignore[union-attr] 

495 return mod 

496 

497 pkg_path = resolve_package_path(path) 

498 if pkg_path is not None: 

499 pkg_root = pkg_path.parent 

500 names = list(path.with_suffix("").relative_to(pkg_root).parts) 

501 if names[-1] == "__init__": 

502 names.pop() 

503 module_name = ".".join(names) 

504 else: 

505 pkg_root = path.parent 

506 module_name = path.stem 

507 

508 # change sys.path permanently: restoring it at the end of this function would cause surprising 

509 # problems because of delayed imports: for example, a conftest.py file imported by this function 

510 # might have local imports, which would fail at runtime if we restored sys.path. 

511 if mode is ImportMode.append: 

512 if str(pkg_root) not in sys.path: 

513 sys.path.append(str(pkg_root)) 

514 elif mode is ImportMode.prepend: 

515 if str(pkg_root) != sys.path[0]: 

516 sys.path.insert(0, str(pkg_root)) 

517 else: 

518 assert_never(mode) 

519 

520 importlib.import_module(module_name) 

521 

522 mod = sys.modules[module_name] 

523 if path.name == "__init__.py": 

524 return mod 

525 

526 ignore = os.environ.get("PY_IGNORE_IMPORTMISMATCH", "") 

527 if ignore != "1": 

528 module_file = mod.__file__ 

529 if module_file.endswith((".pyc", ".pyo")): 

530 module_file = module_file[:-1] 

531 if module_file.endswith(os.path.sep + "__init__.py"): 

532 module_file = module_file[: -(len(os.path.sep + "__init__.py"))] 

533 

534 try: 

535 is_same = os.path.samefile(str(path), module_file) 

536 except FileNotFoundError: 

537 is_same = False 

538 

539 if not is_same: 

540 raise ImportPathMismatchError(module_name, module_file, path) 

541 

542 return mod 

543 

544 

545def resolve_package_path(path: Path) -> Optional[Path]: 

546 """Return the Python package path by looking for the last 

547 directory upwards which still contains an __init__.py. 

548 Return None if it can not be determined. 

549 """ 

550 result = None 

551 for parent in itertools.chain((path,), path.parents): 

552 if parent.is_dir(): 

553 if not parent.joinpath("__init__.py").is_file(): 

554 break 

555 if not parent.name.isidentifier(): 

556 break 

557 result = parent 

558 return result