Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import absolute_import 

2 

3import errno 

4import logging 

5import socket 

6import sys 

7import warnings 

8from socket import error as SocketError 

9from socket import timeout as SocketTimeout 

10 

11from .connection import ( 

12 BaseSSLError, 

13 BrokenPipeError, 

14 DummyConnection, 

15 HTTPConnection, 

16 HTTPException, 

17 HTTPSConnection, 

18 VerifiedHTTPSConnection, 

19 port_by_scheme, 

20) 

21from .exceptions import ( 

22 ClosedPoolError, 

23 EmptyPoolError, 

24 HeaderParsingError, 

25 HostChangedError, 

26 InsecureRequestWarning, 

27 LocationValueError, 

28 MaxRetryError, 

29 NewConnectionError, 

30 ProtocolError, 

31 ProxyError, 

32 ReadTimeoutError, 

33 SSLError, 

34 TimeoutError, 

35) 

36from .packages import six 

37from .packages.six.moves import queue 

38from .packages.ssl_match_hostname import CertificateError 

39from .request import RequestMethods 

40from .response import HTTPResponse 

41from .util.connection import is_connection_dropped 

42from .util.proxy import connection_requires_http_tunnel 

43from .util.queue import LifoQueue 

44from .util.request import set_file_position 

45from .util.response import assert_header_parsing 

46from .util.retry import Retry 

47from .util.timeout import Timeout 

48from .util.url import Url, _encode_target 

49from .util.url import _normalize_host as normalize_host 

50from .util.url import get_host, parse_url 

51 

52xrange = six.moves.xrange 

53 

54log = logging.getLogger(__name__) 

55 

56_Default = object() 

57 

58 

59# Pool objects 

60class ConnectionPool(object): 

61 """ 

62 Base class for all connection pools, such as 

63 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. 

64 

65 .. note:: 

66 ConnectionPool.urlopen() does not normalize or percent-encode target URIs 

67 which is useful if your target server doesn't support percent-encoded 

68 target URIs. 

69 """ 

70 

71 scheme = None 

72 QueueCls = LifoQueue 

73 

74 def __init__(self, host, port=None): 

75 if not host: 

76 raise LocationValueError("No host specified.") 

77 

78 self.host = _normalize_host(host, scheme=self.scheme) 

79 self._proxy_host = host.lower() 

80 self.port = port 

81 

82 def __str__(self): 

83 return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) 

84 

85 def __enter__(self): 

86 return self 

87 

88 def __exit__(self, exc_type, exc_val, exc_tb): 

89 self.close() 

90 # Return False to re-raise any potential exceptions 

91 return False 

92 

93 def close(self): 

94 """ 

95 Close all pooled connections and disable the pool. 

96 """ 

97 pass 

98 

99 

100# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 

101_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} 

102 

103 

104class HTTPConnectionPool(ConnectionPool, RequestMethods): 

105 """ 

106 Thread-safe connection pool for one host. 

107 

108 :param host: 

109 Host used for this HTTP Connection (e.g. "localhost"), passed into 

110 :class:`http.client.HTTPConnection`. 

111 

112 :param port: 

113 Port used for this HTTP Connection (None is equivalent to 80), passed 

114 into :class:`http.client.HTTPConnection`. 

115 

116 :param strict: 

117 Causes BadStatusLine to be raised if the status line can't be parsed 

118 as a valid HTTP/1.0 or 1.1 status line, passed into 

119 :class:`http.client.HTTPConnection`. 

120 

121 .. note:: 

122 Only works in Python 2. This parameter is ignored in Python 3. 

123 

124 :param timeout: 

125 Socket timeout in seconds for each individual connection. This can 

126 be a float or integer, which sets the timeout for the HTTP request, 

127 or an instance of :class:`urllib3.util.Timeout` which gives you more 

128 fine-grained control over request timeouts. After the constructor has 

129 been parsed, this is always a `urllib3.util.Timeout` object. 

130 

131 :param maxsize: 

132 Number of connections to save that can be reused. More than 1 is useful 

133 in multithreaded situations. If ``block`` is set to False, more 

134 connections will be created but they will not be saved once they've 

135 been used. 

136 

137 :param block: 

138 If set to True, no more than ``maxsize`` connections will be used at 

139 a time. When no free connections are available, the call will block 

140 until a connection has been released. This is a useful side effect for 

141 particular multithreaded situations where one does not want to use more 

142 than maxsize connections per host to prevent flooding. 

143 

144 :param headers: 

145 Headers to include with all requests, unless other headers are given 

146 explicitly. 

147 

148 :param retries: 

149 Retry configuration to use by default with requests in this pool. 

150 

151 :param _proxy: 

152 Parsed proxy URL, should not be used directly, instead, see 

153 :class:`urllib3.ProxyManager` 

154 

155 :param _proxy_headers: 

156 A dictionary with proxy headers, should not be used directly, 

157 instead, see :class:`urllib3.ProxyManager` 

158 

159 :param \\**conn_kw: 

160 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, 

161 :class:`urllib3.connection.HTTPSConnection` instances. 

162 """ 

163 

164 scheme = "http" 

165 ConnectionCls = HTTPConnection 

166 ResponseCls = HTTPResponse 

167 

168 def __init__( 

169 self, 

170 host, 

171 port=None, 

172 strict=False, 

173 timeout=Timeout.DEFAULT_TIMEOUT, 

174 maxsize=1, 

175 block=False, 

176 headers=None, 

177 retries=None, 

178 _proxy=None, 

179 _proxy_headers=None, 

180 _proxy_config=None, 

181 **conn_kw 

182 ): 

183 ConnectionPool.__init__(self, host, port) 

184 RequestMethods.__init__(self, headers) 

185 

186 self.strict = strict 

187 

188 if not isinstance(timeout, Timeout): 

189 timeout = Timeout.from_float(timeout) 

190 

191 if retries is None: 

192 retries = Retry.DEFAULT 

193 

194 self.timeout = timeout 

195 self.retries = retries 

196 

197 self.pool = self.QueueCls(maxsize) 

198 self.block = block 

199 

200 self.proxy = _proxy 

201 self.proxy_headers = _proxy_headers or {} 

202 self.proxy_config = _proxy_config 

203 

204 # Fill the queue up so that doing get() on it will block properly 

205 for _ in xrange(maxsize): 

206 self.pool.put(None) 

207 

208 # These are mostly for testing and debugging purposes. 

209 self.num_connections = 0 

210 self.num_requests = 0 

211 self.conn_kw = conn_kw 

212 

213 if self.proxy: 

214 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. 

215 # We cannot know if the user has added default socket options, so we cannot replace the 

216 # list. 

217 self.conn_kw.setdefault("socket_options", []) 

218 

219 self.conn_kw["proxy"] = self.proxy 

220 self.conn_kw["proxy_config"] = self.proxy_config 

221 

222 def _new_conn(self): 

223 """ 

224 Return a fresh :class:`HTTPConnection`. 

225 """ 

226 self.num_connections += 1 

227 log.debug( 

228 "Starting new HTTP connection (%d): %s:%s", 

229 self.num_connections, 

230 self.host, 

231 self.port or "80", 

232 ) 

233 

234 conn = self.ConnectionCls( 

235 host=self.host, 

236 port=self.port, 

237 timeout=self.timeout.connect_timeout, 

238 strict=self.strict, 

239 **self.conn_kw 

240 ) 

241 return conn 

242 

243 def _get_conn(self, timeout=None): 

244 """ 

245 Get a connection. Will return a pooled connection if one is available. 

246 

247 If no connections are available and :prop:`.block` is ``False``, then a 

248 fresh connection is returned. 

249 

250 :param timeout: 

251 Seconds to wait before giving up and raising 

252 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and 

253 :prop:`.block` is ``True``. 

254 """ 

255 conn = None 

256 try: 

257 conn = self.pool.get(block=self.block, timeout=timeout) 

258 

259 except AttributeError: # self.pool is None 

260 raise ClosedPoolError(self, "Pool is closed.") 

261 

262 except queue.Empty: 

263 if self.block: 

264 raise EmptyPoolError( 

265 self, 

266 "Pool reached maximum size and no more connections are allowed.", 

267 ) 

268 pass # Oh well, we'll create a new connection then 

269 

270 # If this is a persistent connection, check if it got disconnected 

271 if conn and is_connection_dropped(conn): 

272 log.debug("Resetting dropped connection: %s", self.host) 

273 conn.close() 

274 if getattr(conn, "auto_open", 1) == 0: 

275 # This is a proxied connection that has been mutated by 

276 # http.client._tunnel() and cannot be reused (since it would 

277 # attempt to bypass the proxy) 

278 conn = None 

279 

280 return conn or self._new_conn() 

281 

282 def _put_conn(self, conn): 

283 """ 

284 Put a connection back into the pool. 

285 

286 :param conn: 

287 Connection object for the current host and port as returned by 

288 :meth:`._new_conn` or :meth:`._get_conn`. 

289 

290 If the pool is already full, the connection is closed and discarded 

291 because we exceeded maxsize. If connections are discarded frequently, 

292 then maxsize should be increased. 

293 

294 If the pool is closed, then the connection will be closed and discarded. 

295 """ 

296 try: 

297 self.pool.put(conn, block=False) 

298 return # Everything is dandy, done. 

299 except AttributeError: 

300 # self.pool is None. 

301 pass 

302 except queue.Full: 

303 # This should never happen if self.block == True 

304 log.warning("Connection pool is full, discarding connection: %s", self.host) 

305 

306 # Connection never got put back into the pool, close it. 

307 if conn: 

308 conn.close() 

309 

310 def _validate_conn(self, conn): 

311 """ 

312 Called right before a request is made, after the socket is created. 

313 """ 

314 pass 

315 

316 def _prepare_proxy(self, conn): 

317 # Nothing to do for HTTP connections. 

318 pass 

319 

320 def _get_timeout(self, timeout): 

321 """ Helper that always returns a :class:`urllib3.util.Timeout` """ 

322 if timeout is _Default: 

323 return self.timeout.clone() 

324 

325 if isinstance(timeout, Timeout): 

326 return timeout.clone() 

327 else: 

328 # User passed us an int/float. This is for backwards compatibility, 

329 # can be removed later 

330 return Timeout.from_float(timeout) 

331 

332 def _raise_timeout(self, err, url, timeout_value): 

333 """Is the error actually a timeout? Will raise a ReadTimeout or pass""" 

334 

335 if isinstance(err, SocketTimeout): 

336 raise ReadTimeoutError( 

337 self, url, "Read timed out. (read timeout=%s)" % timeout_value 

338 ) 

339 

340 # See the above comment about EAGAIN in Python 3. In Python 2 we have 

341 # to specifically catch it and throw the timeout error 

342 if hasattr(err, "errno") and err.errno in _blocking_errnos: 

343 raise ReadTimeoutError( 

344 self, url, "Read timed out. (read timeout=%s)" % timeout_value 

345 ) 

346 

347 # Catch possible read timeouts thrown as SSL errors. If not the 

348 # case, rethrow the original. We need to do this because of: 

349 # http://bugs.python.org/issue10272 

350 if "timed out" in str(err) or "did not complete (read)" in str( 

351 err 

352 ): # Python < 2.7.4 

353 raise ReadTimeoutError( 

354 self, url, "Read timed out. (read timeout=%s)" % timeout_value 

355 ) 

356 

357 def _make_request( 

358 self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw 

359 ): 

360 """ 

361 Perform a request on a given urllib connection object taken from our 

362 pool. 

363 

364 :param conn: 

365 a connection from one of our connection pools 

366 

367 :param timeout: 

368 Socket timeout in seconds for the request. This can be a 

369 float or integer, which will set the same timeout value for 

370 the socket connect and the socket read, or an instance of 

371 :class:`urllib3.util.Timeout`, which gives you more fine-grained 

372 control over your timeouts. 

373 """ 

374 self.num_requests += 1 

375 

376 timeout_obj = self._get_timeout(timeout) 

377 timeout_obj.start_connect() 

378 conn.timeout = timeout_obj.connect_timeout 

379 

380 # Trigger any extra validation we need to do. 

381 try: 

382 self._validate_conn(conn) 

383 except (SocketTimeout, BaseSSLError) as e: 

384 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. 

385 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) 

386 raise 

387 

388 # conn.request() calls http.client.*.request, not the method in 

389 # urllib3.request. It also calls makefile (recv) on the socket. 

390 try: 

391 if chunked: 

392 conn.request_chunked(method, url, **httplib_request_kw) 

393 else: 

394 conn.request(method, url, **httplib_request_kw) 

395 

396 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is 

397 # legitimately able to close the connection after sending a valid response. 

398 # With this behaviour, the received response is still readable. 

399 except BrokenPipeError: 

400 # Python 3 

401 pass 

402 except IOError as e: 

403 # Python 2 and macOS/Linux 

404 # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS 

405 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ 

406 if e.errno not in { 

407 errno.EPIPE, 

408 errno.ESHUTDOWN, 

409 errno.EPROTOTYPE, 

410 }: 

411 raise 

412 

413 # Reset the timeout for the recv() on the socket 

414 read_timeout = timeout_obj.read_timeout 

415 

416 # App Engine doesn't have a sock attr 

417 if getattr(conn, "sock", None): 

418 # In Python 3 socket.py will catch EAGAIN and return None when you 

419 # try and read into the file pointer created by http.client, which 

420 # instead raises a BadStatusLine exception. Instead of catching 

421 # the exception and assuming all BadStatusLine exceptions are read 

422 # timeouts, check for a zero timeout before making the request. 

423 if read_timeout == 0: 

424 raise ReadTimeoutError( 

425 self, url, "Read timed out. (read timeout=%s)" % read_timeout 

426 ) 

427 if read_timeout is Timeout.DEFAULT_TIMEOUT: 

428 conn.sock.settimeout(socket.getdefaulttimeout()) 

429 else: # None or a value 

430 conn.sock.settimeout(read_timeout) 

431 

432 # Receive the response from the server 

433 try: 

434 try: 

435 # Python 2.7, use buffering of HTTP responses 

436 httplib_response = conn.getresponse(buffering=True) 

437 except TypeError: 

438 # Python 3 

439 try: 

440 httplib_response = conn.getresponse() 

441 except BaseException as e: 

442 # Remove the TypeError from the exception chain in 

443 # Python 3 (including for exceptions like SystemExit). 

444 # Otherwise it looks like a bug in the code. 

445 six.raise_from(e, None) 

446 except (SocketTimeout, BaseSSLError, SocketError) as e: 

447 self._raise_timeout(err=e, url=url, timeout_value=read_timeout) 

448 raise 

449 

450 # AppEngine doesn't have a version attr. 

451 http_version = getattr(conn, "_http_vsn_str", "HTTP/?") 

452 log.debug( 

453 '%s://%s:%s "%s %s %s" %s %s', 

454 self.scheme, 

455 self.host, 

456 self.port, 

457 method, 

458 url, 

459 http_version, 

460 httplib_response.status, 

461 httplib_response.length, 

462 ) 

463 

464 try: 

465 assert_header_parsing(httplib_response.msg) 

466 except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 

467 log.warning( 

468 "Failed to parse headers (url=%s): %s", 

469 self._absolute_url(url), 

470 hpe, 

471 exc_info=True, 

472 ) 

473 

474 return httplib_response 

475 

476 def _absolute_url(self, path): 

477 return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url 

478 

479 def close(self): 

480 """ 

481 Close all pooled connections and disable the pool. 

482 """ 

483 if self.pool is None: 

484 return 

485 # Disable access to the pool 

486 old_pool, self.pool = self.pool, None 

487 

488 try: 

489 while True: 

490 conn = old_pool.get(block=False) 

491 if conn: 

492 conn.close() 

493 

494 except queue.Empty: 

495 pass # Done. 

496 

497 def is_same_host(self, url): 

498 """ 

499 Check if the given ``url`` is a member of the same host as this 

500 connection pool. 

501 """ 

502 if url.startswith("/"): 

503 return True 

504 

505 # TODO: Add optional support for socket.gethostbyname checking. 

506 scheme, host, port = get_host(url) 

507 if host is not None: 

508 host = _normalize_host(host, scheme=scheme) 

509 

510 # Use explicit default port for comparison when none is given 

511 if self.port and not port: 

512 port = port_by_scheme.get(scheme) 

513 elif not self.port and port == port_by_scheme.get(scheme): 

514 port = None 

515 

516 return (scheme, host, port) == (self.scheme, self.host, self.port) 

517 

518 def urlopen( 

519 self, 

520 method, 

521 url, 

522 body=None, 

523 headers=None, 

524 retries=None, 

525 redirect=True, 

526 assert_same_host=True, 

527 timeout=_Default, 

528 pool_timeout=None, 

529 release_conn=None, 

530 chunked=False, 

531 body_pos=None, 

532 **response_kw 

533 ): 

534 """ 

535 Get a connection from the pool and perform an HTTP request. This is the 

536 lowest level call for making a request, so you'll need to specify all 

537 the raw details. 

538 

539 .. note:: 

540 

541 More commonly, it's appropriate to use a convenience method provided 

542 by :class:`.RequestMethods`, such as :meth:`request`. 

543 

544 .. note:: 

545 

546 `release_conn` will only behave as expected if 

547 `preload_content=False` because we want to make 

548 `preload_content=False` the default behaviour someday soon without 

549 breaking backwards compatibility. 

550 

551 :param method: 

552 HTTP request method (such as GET, POST, PUT, etc.) 

553 

554 :param url: 

555 The URL to perform the request on. 

556 

557 :param body: 

558 Data to send in the request body, either :class:`str`, :class:`bytes`, 

559 an iterable of :class:`str`/:class:`bytes`, or a file-like object. 

560 

561 :param headers: 

562 Dictionary of custom headers to send, such as User-Agent, 

563 If-None-Match, etc. If None, pool headers are used. If provided, 

564 these headers completely replace any pool-specific headers. 

565 

566 :param retries: 

567 Configure the number of retries to allow before raising a 

568 :class:`~urllib3.exceptions.MaxRetryError` exception. 

569 

570 Pass ``None`` to retry until you receive a response. Pass a 

571 :class:`~urllib3.util.retry.Retry` object for fine-grained control 

572 over different types of retries. 

573 Pass an integer number to retry connection errors that many times, 

574 but no other types of errors. Pass zero to never retry. 

575 

576 If ``False``, then retries are disabled and any exception is raised 

577 immediately. Also, instead of raising a MaxRetryError on redirects, 

578 the redirect response will be returned. 

579 

580 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. 

581 

582 :param redirect: 

583 If True, automatically handle redirects (status codes 301, 302, 

584 303, 307, 308). Each redirect counts as a retry. Disabling retries 

585 will disable redirect, too. 

586 

587 :param assert_same_host: 

588 If ``True``, will make sure that the host of the pool requests is 

589 consistent else will raise HostChangedError. When ``False``, you can 

590 use the pool on an HTTP proxy and request foreign hosts. 

591 

592 :param timeout: 

593 If specified, overrides the default timeout for this one 

594 request. It may be a float (in seconds) or an instance of 

595 :class:`urllib3.util.Timeout`. 

596 

597 :param pool_timeout: 

598 If set and the pool is set to block=True, then this method will 

599 block for ``pool_timeout`` seconds and raise EmptyPoolError if no 

600 connection is available within the time period. 

601 

602 :param release_conn: 

603 If False, then the urlopen call will not release the connection 

604 back into the pool once a response is received (but will release if 

605 you read the entire contents of the response such as when 

606 `preload_content=True`). This is useful if you're not preloading 

607 the response's content immediately. You will need to call 

608 ``r.release_conn()`` on the response ``r`` to return the connection 

609 back into the pool. If None, it takes the value of 

610 ``response_kw.get('preload_content', True)``. 

611 

612 :param chunked: 

613 If True, urllib3 will send the body using chunked transfer 

614 encoding. Otherwise, urllib3 will send the body using the standard 

615 content-length form. Defaults to False. 

616 

617 :param int body_pos: 

618 Position to seek to in file-like body in the event of a retry or 

619 redirect. Typically this won't need to be set because urllib3 will 

620 auto-populate the value when needed. 

621 

622 :param \\**response_kw: 

623 Additional parameters are passed to 

624 :meth:`urllib3.response.HTTPResponse.from_httplib` 

625 """ 

626 

627 parsed_url = parse_url(url) 

628 destination_scheme = parsed_url.scheme 

629 

630 if headers is None: 

631 headers = self.headers 

632 

633 if not isinstance(retries, Retry): 

634 retries = Retry.from_int(retries, redirect=redirect, default=self.retries) 

635 

636 if release_conn is None: 

637 release_conn = response_kw.get("preload_content", True) 

638 

639 # Check host 

640 if assert_same_host and not self.is_same_host(url): 

641 raise HostChangedError(self, url, retries) 

642 

643 # Ensure that the URL we're connecting to is properly encoded 

644 if url.startswith("/"): 

645 url = six.ensure_str(_encode_target(url)) 

646 else: 

647 url = six.ensure_str(parsed_url.url) 

648 

649 conn = None 

650 

651 # Track whether `conn` needs to be released before 

652 # returning/raising/recursing. Update this variable if necessary, and 

653 # leave `release_conn` constant throughout the function. That way, if 

654 # the function recurses, the original value of `release_conn` will be 

655 # passed down into the recursive call, and its value will be respected. 

656 # 

657 # See issue #651 [1] for details. 

658 # 

659 # [1] <https://github.com/urllib3/urllib3/issues/651> 

660 release_this_conn = release_conn 

661 

662 http_tunnel_required = connection_requires_http_tunnel( 

663 self.proxy, self.proxy_config, destination_scheme 

664 ) 

665 

666 # Merge the proxy headers. Only done when not using HTTP CONNECT. We 

667 # have to copy the headers dict so we can safely change it without those 

668 # changes being reflected in anyone else's copy. 

669 if not http_tunnel_required: 

670 headers = headers.copy() 

671 headers.update(self.proxy_headers) 

672 

673 # Must keep the exception bound to a separate variable or else Python 3 

674 # complains about UnboundLocalError. 

675 err = None 

676 

677 # Keep track of whether we cleanly exited the except block. This 

678 # ensures we do proper cleanup in finally. 

679 clean_exit = False 

680 

681 # Rewind body position, if needed. Record current position 

682 # for future rewinds in the event of a redirect/retry. 

683 body_pos = set_file_position(body, body_pos) 

684 

685 try: 

686 # Request a connection from the queue. 

687 timeout_obj = self._get_timeout(timeout) 

688 conn = self._get_conn(timeout=pool_timeout) 

689 

690 conn.timeout = timeout_obj.connect_timeout 

691 

692 is_new_proxy_conn = self.proxy is not None and not getattr( 

693 conn, "sock", None 

694 ) 

695 if is_new_proxy_conn and http_tunnel_required: 

696 self._prepare_proxy(conn) 

697 

698 # Make the request on the httplib connection object. 

699 httplib_response = self._make_request( 

700 conn, 

701 method, 

702 url, 

703 timeout=timeout_obj, 

704 body=body, 

705 headers=headers, 

706 chunked=chunked, 

707 ) 

708 

709 # If we're going to release the connection in ``finally:``, then 

710 # the response doesn't need to know about the connection. Otherwise 

711 # it will also try to release it and we'll have a double-release 

712 # mess. 

713 response_conn = conn if not release_conn else None 

714 

715 # Pass method to Response for length checking 

716 response_kw["request_method"] = method 

717 

718 # Import httplib's response into our own wrapper object 

719 response = self.ResponseCls.from_httplib( 

720 httplib_response, 

721 pool=self, 

722 connection=response_conn, 

723 retries=retries, 

724 **response_kw 

725 ) 

726 

727 # Everything went great! 

728 clean_exit = True 

729 

730 except EmptyPoolError: 

731 # Didn't get a connection from the pool, no need to clean up 

732 clean_exit = True 

733 release_this_conn = False 

734 raise 

735 

736 except ( 

737 TimeoutError, 

738 HTTPException, 

739 SocketError, 

740 ProtocolError, 

741 BaseSSLError, 

742 SSLError, 

743 CertificateError, 

744 ) as e: 

745 # Discard the connection for these exceptions. It will be 

746 # replaced during the next _get_conn() call. 

747 clean_exit = False 

748 if isinstance(e, (BaseSSLError, CertificateError)): 

749 e = SSLError(e) 

750 elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: 

751 e = ProxyError("Cannot connect to proxy.", e) 

752 elif isinstance(e, (SocketError, HTTPException)): 

753 e = ProtocolError("Connection aborted.", e) 

754 

755 retries = retries.increment( 

756 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] 

757 ) 

758 retries.sleep() 

759 

760 # Keep track of the error for the retry warning. 

761 err = e 

762 

763 finally: 

764 if not clean_exit: 

765 # We hit some kind of exception, handled or otherwise. We need 

766 # to throw the connection away unless explicitly told not to. 

767 # Close the connection, set the variable to None, and make sure 

768 # we put the None back in the pool to avoid leaking it. 

769 conn = conn and conn.close() 

770 release_this_conn = True 

771 

772 if release_this_conn: 

773 # Put the connection back to be reused. If the connection is 

774 # expired then it will be None, which will get replaced with a 

775 # fresh connection during _get_conn. 

776 self._put_conn(conn) 

777 

778 if not conn: 

779 # Try again 

780 log.warning( 

781 "Retrying (%r) after connection broken by '%r': %s", retries, err, url 

782 ) 

783 return self.urlopen( 

784 method, 

785 url, 

786 body, 

787 headers, 

788 retries, 

789 redirect, 

790 assert_same_host, 

791 timeout=timeout, 

792 pool_timeout=pool_timeout, 

793 release_conn=release_conn, 

794 chunked=chunked, 

795 body_pos=body_pos, 

796 **response_kw 

797 ) 

798 

799 # Handle redirect? 

800 redirect_location = redirect and response.get_redirect_location() 

801 if redirect_location: 

802 if response.status == 303: 

803 method = "GET" 

804 

805 try: 

806 retries = retries.increment(method, url, response=response, _pool=self) 

807 except MaxRetryError: 

808 if retries.raise_on_redirect: 

809 response.drain_conn() 

810 raise 

811 return response 

812 

813 response.drain_conn() 

814 retries.sleep_for_retry(response) 

815 log.debug("Redirecting %s -> %s", url, redirect_location) 

816 return self.urlopen( 

817 method, 

818 redirect_location, 

819 body, 

820 headers, 

821 retries=retries, 

822 redirect=redirect, 

823 assert_same_host=assert_same_host, 

824 timeout=timeout, 

825 pool_timeout=pool_timeout, 

826 release_conn=release_conn, 

827 chunked=chunked, 

828 body_pos=body_pos, 

829 **response_kw 

830 ) 

831 

832 # Check if we should retry the HTTP response. 

833 has_retry_after = bool(response.getheader("Retry-After")) 

834 if retries.is_retry(method, response.status, has_retry_after): 

835 try: 

836 retries = retries.increment(method, url, response=response, _pool=self) 

837 except MaxRetryError: 

838 if retries.raise_on_status: 

839 response.drain_conn() 

840 raise 

841 return response 

842 

843 response.drain_conn() 

844 retries.sleep(response) 

845 log.debug("Retry: %s", url) 

846 return self.urlopen( 

847 method, 

848 url, 

849 body, 

850 headers, 

851 retries=retries, 

852 redirect=redirect, 

853 assert_same_host=assert_same_host, 

854 timeout=timeout, 

855 pool_timeout=pool_timeout, 

856 release_conn=release_conn, 

857 chunked=chunked, 

858 body_pos=body_pos, 

859 **response_kw 

860 ) 

861 

862 return response 

863 

864 

865class HTTPSConnectionPool(HTTPConnectionPool): 

866 """ 

867 Same as :class:`.HTTPConnectionPool`, but HTTPS. 

868 

869 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``, 

870 ``assert_hostname`` and ``host`` in this order to verify connections. 

871 If ``assert_hostname`` is False, no verification is done. 

872 

873 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, 

874 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` 

875 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade 

876 the connection socket into an SSL socket. 

877 """ 

878 

879 scheme = "https" 

880 ConnectionCls = HTTPSConnection 

881 

882 def __init__( 

883 self, 

884 host, 

885 port=None, 

886 strict=False, 

887 timeout=Timeout.DEFAULT_TIMEOUT, 

888 maxsize=1, 

889 block=False, 

890 headers=None, 

891 retries=None, 

892 _proxy=None, 

893 _proxy_headers=None, 

894 key_file=None, 

895 cert_file=None, 

896 cert_reqs=None, 

897 key_password=None, 

898 ca_certs=None, 

899 ssl_version=None, 

900 assert_hostname=None, 

901 assert_fingerprint=None, 

902 ca_cert_dir=None, 

903 **conn_kw 

904 ): 

905 

906 HTTPConnectionPool.__init__( 

907 self, 

908 host, 

909 port, 

910 strict, 

911 timeout, 

912 maxsize, 

913 block, 

914 headers, 

915 retries, 

916 _proxy, 

917 _proxy_headers, 

918 **conn_kw 

919 ) 

920 

921 self.key_file = key_file 

922 self.cert_file = cert_file 

923 self.cert_reqs = cert_reqs 

924 self.key_password = key_password 

925 self.ca_certs = ca_certs 

926 self.ca_cert_dir = ca_cert_dir 

927 self.ssl_version = ssl_version 

928 self.assert_hostname = assert_hostname 

929 self.assert_fingerprint = assert_fingerprint 

930 

931 def _prepare_conn(self, conn): 

932 """ 

933 Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` 

934 and establish the tunnel if proxy is used. 

935 """ 

936 

937 if isinstance(conn, VerifiedHTTPSConnection): 

938 conn.set_cert( 

939 key_file=self.key_file, 

940 key_password=self.key_password, 

941 cert_file=self.cert_file, 

942 cert_reqs=self.cert_reqs, 

943 ca_certs=self.ca_certs, 

944 ca_cert_dir=self.ca_cert_dir, 

945 assert_hostname=self.assert_hostname, 

946 assert_fingerprint=self.assert_fingerprint, 

947 ) 

948 conn.ssl_version = self.ssl_version 

949 return conn 

950 

951 def _prepare_proxy(self, conn): 

952 """ 

953 Establishes a tunnel connection through HTTP CONNECT. 

954 

955 Tunnel connection is established early because otherwise httplib would 

956 improperly set Host: header to proxy's IP:port. 

957 """ 

958 

959 conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) 

960 

961 if self.proxy.scheme == "https": 

962 conn.tls_in_tls_required = True 

963 

964 conn.connect() 

965 

966 def _new_conn(self): 

967 """ 

968 Return a fresh :class:`http.client.HTTPSConnection`. 

969 """ 

970 self.num_connections += 1 

971 log.debug( 

972 "Starting new HTTPS connection (%d): %s:%s", 

973 self.num_connections, 

974 self.host, 

975 self.port or "443", 

976 ) 

977 

978 if not self.ConnectionCls or self.ConnectionCls is DummyConnection: 

979 raise SSLError( 

980 "Can't connect to HTTPS URL because the SSL module is not available." 

981 ) 

982 

983 actual_host = self.host 

984 actual_port = self.port 

985 if self.proxy is not None: 

986 actual_host = self.proxy.host 

987 actual_port = self.proxy.port 

988 

989 conn = self.ConnectionCls( 

990 host=actual_host, 

991 port=actual_port, 

992 timeout=self.timeout.connect_timeout, 

993 strict=self.strict, 

994 cert_file=self.cert_file, 

995 key_file=self.key_file, 

996 key_password=self.key_password, 

997 **self.conn_kw 

998 ) 

999 

1000 return self._prepare_conn(conn) 

1001 

1002 def _validate_conn(self, conn): 

1003 """ 

1004 Called right before a request is made, after the socket is created. 

1005 """ 

1006 super(HTTPSConnectionPool, self)._validate_conn(conn) 

1007 

1008 # Force connect early to allow us to validate the connection. 

1009 if not getattr(conn, "sock", None): # AppEngine might not have `.sock` 

1010 conn.connect() 

1011 

1012 if not conn.is_verified: 

1013 warnings.warn( 

1014 ( 

1015 "Unverified HTTPS request is being made to host '%s'. " 

1016 "Adding certificate verification is strongly advised. See: " 

1017 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" 

1018 "#ssl-warnings" % conn.host 

1019 ), 

1020 InsecureRequestWarning, 

1021 ) 

1022 

1023 

1024def connection_from_url(url, **kw): 

1025 """ 

1026 Given a url, return an :class:`.ConnectionPool` instance of its host. 

1027 

1028 This is a shortcut for not having to parse out the scheme, host, and port 

1029 of the url before creating an :class:`.ConnectionPool` instance. 

1030 

1031 :param url: 

1032 Absolute URL string that must include the scheme. Port is optional. 

1033 

1034 :param \\**kw: 

1035 Passes additional parameters to the constructor of the appropriate 

1036 :class:`.ConnectionPool`. Useful for specifying things like 

1037 timeout, maxsize, headers, etc. 

1038 

1039 Example:: 

1040 

1041 >>> conn = connection_from_url('http://google.com/') 

1042 >>> r = conn.request('GET', '/') 

1043 """ 

1044 scheme, host, port = get_host(url) 

1045 port = port or port_by_scheme.get(scheme, 80) 

1046 if scheme == "https": 

1047 return HTTPSConnectionPool(host, port=port, **kw) 

1048 else: 

1049 return HTTPConnectionPool(host, port=port, **kw) 

1050 

1051 

1052def _normalize_host(host, scheme): 

1053 """ 

1054 Normalize hosts for comparisons and use with sockets. 

1055 """ 

1056 

1057 host = normalize_host(host, scheme) 

1058 

1059 # httplib doesn't like it when we include brackets in IPv6 addresses 

1060 # Specifically, if we include brackets but also pass the port then 

1061 # httplib crazily doubles up the square brackets on the Host header. 

1062 # Instead, we need to make sure we never pass ``None`` as the port. 

1063 # However, for backward compatibility reasons we can't actually 

1064 # *assert* that. See http://bugs.python.org/issue28539 

1065 if host.startswith("[") and host.endswith("]"): 

1066 host = host[1:-1] 

1067 return host