Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/urllib3/connectionpool.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import absolute_import
3import errno
4import logging
5import socket
6import sys
7import warnings
8from socket import error as SocketError
9from socket import timeout as SocketTimeout
11from .connection import (
12 BaseSSLError,
13 BrokenPipeError,
14 DummyConnection,
15 HTTPConnection,
16 HTTPException,
17 HTTPSConnection,
18 VerifiedHTTPSConnection,
19 port_by_scheme,
20)
21from .exceptions import (
22 ClosedPoolError,
23 EmptyPoolError,
24 HeaderParsingError,
25 HostChangedError,
26 InsecureRequestWarning,
27 LocationValueError,
28 MaxRetryError,
29 NewConnectionError,
30 ProtocolError,
31 ProxyError,
32 ReadTimeoutError,
33 SSLError,
34 TimeoutError,
35)
36from .packages import six
37from .packages.six.moves import queue
38from .packages.ssl_match_hostname import CertificateError
39from .request import RequestMethods
40from .response import HTTPResponse
41from .util.connection import is_connection_dropped
42from .util.proxy import connection_requires_http_tunnel
43from .util.queue import LifoQueue
44from .util.request import set_file_position
45from .util.response import assert_header_parsing
46from .util.retry import Retry
47from .util.timeout import Timeout
48from .util.url import Url, _encode_target
49from .util.url import _normalize_host as normalize_host
50from .util.url import get_host, parse_url
52xrange = six.moves.xrange
54log = logging.getLogger(__name__)
56_Default = object()
59# Pool objects
60class ConnectionPool(object):
61 """
62 Base class for all connection pools, such as
63 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
65 .. note::
66 ConnectionPool.urlopen() does not normalize or percent-encode target URIs
67 which is useful if your target server doesn't support percent-encoded
68 target URIs.
69 """
71 scheme = None
72 QueueCls = LifoQueue
74 def __init__(self, host, port=None):
75 if not host:
76 raise LocationValueError("No host specified.")
78 self.host = _normalize_host(host, scheme=self.scheme)
79 self._proxy_host = host.lower()
80 self.port = port
82 def __str__(self):
83 return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)
85 def __enter__(self):
86 return self
88 def __exit__(self, exc_type, exc_val, exc_tb):
89 self.close()
90 # Return False to re-raise any potential exceptions
91 return False
93 def close(self):
94 """
95 Close all pooled connections and disable the pool.
96 """
97 pass
100# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
101_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
104class HTTPConnectionPool(ConnectionPool, RequestMethods):
105 """
106 Thread-safe connection pool for one host.
108 :param host:
109 Host used for this HTTP Connection (e.g. "localhost"), passed into
110 :class:`http.client.HTTPConnection`.
112 :param port:
113 Port used for this HTTP Connection (None is equivalent to 80), passed
114 into :class:`http.client.HTTPConnection`.
116 :param strict:
117 Causes BadStatusLine to be raised if the status line can't be parsed
118 as a valid HTTP/1.0 or 1.1 status line, passed into
119 :class:`http.client.HTTPConnection`.
121 .. note::
122 Only works in Python 2. This parameter is ignored in Python 3.
124 :param timeout:
125 Socket timeout in seconds for each individual connection. This can
126 be a float or integer, which sets the timeout for the HTTP request,
127 or an instance of :class:`urllib3.util.Timeout` which gives you more
128 fine-grained control over request timeouts. After the constructor has
129 been parsed, this is always a `urllib3.util.Timeout` object.
131 :param maxsize:
132 Number of connections to save that can be reused. More than 1 is useful
133 in multithreaded situations. If ``block`` is set to False, more
134 connections will be created but they will not be saved once they've
135 been used.
137 :param block:
138 If set to True, no more than ``maxsize`` connections will be used at
139 a time. When no free connections are available, the call will block
140 until a connection has been released. This is a useful side effect for
141 particular multithreaded situations where one does not want to use more
142 than maxsize connections per host to prevent flooding.
144 :param headers:
145 Headers to include with all requests, unless other headers are given
146 explicitly.
148 :param retries:
149 Retry configuration to use by default with requests in this pool.
151 :param _proxy:
152 Parsed proxy URL, should not be used directly, instead, see
153 :class:`urllib3.ProxyManager`
155 :param _proxy_headers:
156 A dictionary with proxy headers, should not be used directly,
157 instead, see :class:`urllib3.ProxyManager`
159 :param \\**conn_kw:
160 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
161 :class:`urllib3.connection.HTTPSConnection` instances.
162 """
164 scheme = "http"
165 ConnectionCls = HTTPConnection
166 ResponseCls = HTTPResponse
168 def __init__(
169 self,
170 host,
171 port=None,
172 strict=False,
173 timeout=Timeout.DEFAULT_TIMEOUT,
174 maxsize=1,
175 block=False,
176 headers=None,
177 retries=None,
178 _proxy=None,
179 _proxy_headers=None,
180 _proxy_config=None,
181 **conn_kw
182 ):
183 ConnectionPool.__init__(self, host, port)
184 RequestMethods.__init__(self, headers)
186 self.strict = strict
188 if not isinstance(timeout, Timeout):
189 timeout = Timeout.from_float(timeout)
191 if retries is None:
192 retries = Retry.DEFAULT
194 self.timeout = timeout
195 self.retries = retries
197 self.pool = self.QueueCls(maxsize)
198 self.block = block
200 self.proxy = _proxy
201 self.proxy_headers = _proxy_headers or {}
202 self.proxy_config = _proxy_config
204 # Fill the queue up so that doing get() on it will block properly
205 for _ in xrange(maxsize):
206 self.pool.put(None)
208 # These are mostly for testing and debugging purposes.
209 self.num_connections = 0
210 self.num_requests = 0
211 self.conn_kw = conn_kw
213 if self.proxy:
214 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
215 # We cannot know if the user has added default socket options, so we cannot replace the
216 # list.
217 self.conn_kw.setdefault("socket_options", [])
219 self.conn_kw["proxy"] = self.proxy
220 self.conn_kw["proxy_config"] = self.proxy_config
222 def _new_conn(self):
223 """
224 Return a fresh :class:`HTTPConnection`.
225 """
226 self.num_connections += 1
227 log.debug(
228 "Starting new HTTP connection (%d): %s:%s",
229 self.num_connections,
230 self.host,
231 self.port or "80",
232 )
234 conn = self.ConnectionCls(
235 host=self.host,
236 port=self.port,
237 timeout=self.timeout.connect_timeout,
238 strict=self.strict,
239 **self.conn_kw
240 )
241 return conn
243 def _get_conn(self, timeout=None):
244 """
245 Get a connection. Will return a pooled connection if one is available.
247 If no connections are available and :prop:`.block` is ``False``, then a
248 fresh connection is returned.
250 :param timeout:
251 Seconds to wait before giving up and raising
252 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
253 :prop:`.block` is ``True``.
254 """
255 conn = None
256 try:
257 conn = self.pool.get(block=self.block, timeout=timeout)
259 except AttributeError: # self.pool is None
260 raise ClosedPoolError(self, "Pool is closed.")
262 except queue.Empty:
263 if self.block:
264 raise EmptyPoolError(
265 self,
266 "Pool reached maximum size and no more connections are allowed.",
267 )
268 pass # Oh well, we'll create a new connection then
270 # If this is a persistent connection, check if it got disconnected
271 if conn and is_connection_dropped(conn):
272 log.debug("Resetting dropped connection: %s", self.host)
273 conn.close()
274 if getattr(conn, "auto_open", 1) == 0:
275 # This is a proxied connection that has been mutated by
276 # http.client._tunnel() and cannot be reused (since it would
277 # attempt to bypass the proxy)
278 conn = None
280 return conn or self._new_conn()
282 def _put_conn(self, conn):
283 """
284 Put a connection back into the pool.
286 :param conn:
287 Connection object for the current host and port as returned by
288 :meth:`._new_conn` or :meth:`._get_conn`.
290 If the pool is already full, the connection is closed and discarded
291 because we exceeded maxsize. If connections are discarded frequently,
292 then maxsize should be increased.
294 If the pool is closed, then the connection will be closed and discarded.
295 """
296 try:
297 self.pool.put(conn, block=False)
298 return # Everything is dandy, done.
299 except AttributeError:
300 # self.pool is None.
301 pass
302 except queue.Full:
303 # This should never happen if self.block == True
304 log.warning("Connection pool is full, discarding connection: %s", self.host)
306 # Connection never got put back into the pool, close it.
307 if conn:
308 conn.close()
310 def _validate_conn(self, conn):
311 """
312 Called right before a request is made, after the socket is created.
313 """
314 pass
316 def _prepare_proxy(self, conn):
317 # Nothing to do for HTTP connections.
318 pass
320 def _get_timeout(self, timeout):
321 """ Helper that always returns a :class:`urllib3.util.Timeout` """
322 if timeout is _Default:
323 return self.timeout.clone()
325 if isinstance(timeout, Timeout):
326 return timeout.clone()
327 else:
328 # User passed us an int/float. This is for backwards compatibility,
329 # can be removed later
330 return Timeout.from_float(timeout)
332 def _raise_timeout(self, err, url, timeout_value):
333 """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
335 if isinstance(err, SocketTimeout):
336 raise ReadTimeoutError(
337 self, url, "Read timed out. (read timeout=%s)" % timeout_value
338 )
340 # See the above comment about EAGAIN in Python 3. In Python 2 we have
341 # to specifically catch it and throw the timeout error
342 if hasattr(err, "errno") and err.errno in _blocking_errnos:
343 raise ReadTimeoutError(
344 self, url, "Read timed out. (read timeout=%s)" % timeout_value
345 )
347 # Catch possible read timeouts thrown as SSL errors. If not the
348 # case, rethrow the original. We need to do this because of:
349 # http://bugs.python.org/issue10272
350 if "timed out" in str(err) or "did not complete (read)" in str(
351 err
352 ): # Python < 2.7.4
353 raise ReadTimeoutError(
354 self, url, "Read timed out. (read timeout=%s)" % timeout_value
355 )
357 def _make_request(
358 self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
359 ):
360 """
361 Perform a request on a given urllib connection object taken from our
362 pool.
364 :param conn:
365 a connection from one of our connection pools
367 :param timeout:
368 Socket timeout in seconds for the request. This can be a
369 float or integer, which will set the same timeout value for
370 the socket connect and the socket read, or an instance of
371 :class:`urllib3.util.Timeout`, which gives you more fine-grained
372 control over your timeouts.
373 """
374 self.num_requests += 1
376 timeout_obj = self._get_timeout(timeout)
377 timeout_obj.start_connect()
378 conn.timeout = timeout_obj.connect_timeout
380 # Trigger any extra validation we need to do.
381 try:
382 self._validate_conn(conn)
383 except (SocketTimeout, BaseSSLError) as e:
384 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
385 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
386 raise
388 # conn.request() calls http.client.*.request, not the method in
389 # urllib3.request. It also calls makefile (recv) on the socket.
390 try:
391 if chunked:
392 conn.request_chunked(method, url, **httplib_request_kw)
393 else:
394 conn.request(method, url, **httplib_request_kw)
396 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
397 # legitimately able to close the connection after sending a valid response.
398 # With this behaviour, the received response is still readable.
399 except BrokenPipeError:
400 # Python 3
401 pass
402 except IOError as e:
403 # Python 2 and macOS/Linux
404 # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS
405 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
406 if e.errno not in {
407 errno.EPIPE,
408 errno.ESHUTDOWN,
409 errno.EPROTOTYPE,
410 }:
411 raise
413 # Reset the timeout for the recv() on the socket
414 read_timeout = timeout_obj.read_timeout
416 # App Engine doesn't have a sock attr
417 if getattr(conn, "sock", None):
418 # In Python 3 socket.py will catch EAGAIN and return None when you
419 # try and read into the file pointer created by http.client, which
420 # instead raises a BadStatusLine exception. Instead of catching
421 # the exception and assuming all BadStatusLine exceptions are read
422 # timeouts, check for a zero timeout before making the request.
423 if read_timeout == 0:
424 raise ReadTimeoutError(
425 self, url, "Read timed out. (read timeout=%s)" % read_timeout
426 )
427 if read_timeout is Timeout.DEFAULT_TIMEOUT:
428 conn.sock.settimeout(socket.getdefaulttimeout())
429 else: # None or a value
430 conn.sock.settimeout(read_timeout)
432 # Receive the response from the server
433 try:
434 try:
435 # Python 2.7, use buffering of HTTP responses
436 httplib_response = conn.getresponse(buffering=True)
437 except TypeError:
438 # Python 3
439 try:
440 httplib_response = conn.getresponse()
441 except BaseException as e:
442 # Remove the TypeError from the exception chain in
443 # Python 3 (including for exceptions like SystemExit).
444 # Otherwise it looks like a bug in the code.
445 six.raise_from(e, None)
446 except (SocketTimeout, BaseSSLError, SocketError) as e:
447 self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
448 raise
450 # AppEngine doesn't have a version attr.
451 http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
452 log.debug(
453 '%s://%s:%s "%s %s %s" %s %s',
454 self.scheme,
455 self.host,
456 self.port,
457 method,
458 url,
459 http_version,
460 httplib_response.status,
461 httplib_response.length,
462 )
464 try:
465 assert_header_parsing(httplib_response.msg)
466 except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3
467 log.warning(
468 "Failed to parse headers (url=%s): %s",
469 self._absolute_url(url),
470 hpe,
471 exc_info=True,
472 )
474 return httplib_response
476 def _absolute_url(self, path):
477 return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url
479 def close(self):
480 """
481 Close all pooled connections and disable the pool.
482 """
483 if self.pool is None:
484 return
485 # Disable access to the pool
486 old_pool, self.pool = self.pool, None
488 try:
489 while True:
490 conn = old_pool.get(block=False)
491 if conn:
492 conn.close()
494 except queue.Empty:
495 pass # Done.
497 def is_same_host(self, url):
498 """
499 Check if the given ``url`` is a member of the same host as this
500 connection pool.
501 """
502 if url.startswith("/"):
503 return True
505 # TODO: Add optional support for socket.gethostbyname checking.
506 scheme, host, port = get_host(url)
507 if host is not None:
508 host = _normalize_host(host, scheme=scheme)
510 # Use explicit default port for comparison when none is given
511 if self.port and not port:
512 port = port_by_scheme.get(scheme)
513 elif not self.port and port == port_by_scheme.get(scheme):
514 port = None
516 return (scheme, host, port) == (self.scheme, self.host, self.port)
518 def urlopen(
519 self,
520 method,
521 url,
522 body=None,
523 headers=None,
524 retries=None,
525 redirect=True,
526 assert_same_host=True,
527 timeout=_Default,
528 pool_timeout=None,
529 release_conn=None,
530 chunked=False,
531 body_pos=None,
532 **response_kw
533 ):
534 """
535 Get a connection from the pool and perform an HTTP request. This is the
536 lowest level call for making a request, so you'll need to specify all
537 the raw details.
539 .. note::
541 More commonly, it's appropriate to use a convenience method provided
542 by :class:`.RequestMethods`, such as :meth:`request`.
544 .. note::
546 `release_conn` will only behave as expected if
547 `preload_content=False` because we want to make
548 `preload_content=False` the default behaviour someday soon without
549 breaking backwards compatibility.
551 :param method:
552 HTTP request method (such as GET, POST, PUT, etc.)
554 :param url:
555 The URL to perform the request on.
557 :param body:
558 Data to send in the request body, either :class:`str`, :class:`bytes`,
559 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
561 :param headers:
562 Dictionary of custom headers to send, such as User-Agent,
563 If-None-Match, etc. If None, pool headers are used. If provided,
564 these headers completely replace any pool-specific headers.
566 :param retries:
567 Configure the number of retries to allow before raising a
568 :class:`~urllib3.exceptions.MaxRetryError` exception.
570 Pass ``None`` to retry until you receive a response. Pass a
571 :class:`~urllib3.util.retry.Retry` object for fine-grained control
572 over different types of retries.
573 Pass an integer number to retry connection errors that many times,
574 but no other types of errors. Pass zero to never retry.
576 If ``False``, then retries are disabled and any exception is raised
577 immediately. Also, instead of raising a MaxRetryError on redirects,
578 the redirect response will be returned.
580 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
582 :param redirect:
583 If True, automatically handle redirects (status codes 301, 302,
584 303, 307, 308). Each redirect counts as a retry. Disabling retries
585 will disable redirect, too.
587 :param assert_same_host:
588 If ``True``, will make sure that the host of the pool requests is
589 consistent else will raise HostChangedError. When ``False``, you can
590 use the pool on an HTTP proxy and request foreign hosts.
592 :param timeout:
593 If specified, overrides the default timeout for this one
594 request. It may be a float (in seconds) or an instance of
595 :class:`urllib3.util.Timeout`.
597 :param pool_timeout:
598 If set and the pool is set to block=True, then this method will
599 block for ``pool_timeout`` seconds and raise EmptyPoolError if no
600 connection is available within the time period.
602 :param release_conn:
603 If False, then the urlopen call will not release the connection
604 back into the pool once a response is received (but will release if
605 you read the entire contents of the response such as when
606 `preload_content=True`). This is useful if you're not preloading
607 the response's content immediately. You will need to call
608 ``r.release_conn()`` on the response ``r`` to return the connection
609 back into the pool. If None, it takes the value of
610 ``response_kw.get('preload_content', True)``.
612 :param chunked:
613 If True, urllib3 will send the body using chunked transfer
614 encoding. Otherwise, urllib3 will send the body using the standard
615 content-length form. Defaults to False.
617 :param int body_pos:
618 Position to seek to in file-like body in the event of a retry or
619 redirect. Typically this won't need to be set because urllib3 will
620 auto-populate the value when needed.
622 :param \\**response_kw:
623 Additional parameters are passed to
624 :meth:`urllib3.response.HTTPResponse.from_httplib`
625 """
627 parsed_url = parse_url(url)
628 destination_scheme = parsed_url.scheme
630 if headers is None:
631 headers = self.headers
633 if not isinstance(retries, Retry):
634 retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
636 if release_conn is None:
637 release_conn = response_kw.get("preload_content", True)
639 # Check host
640 if assert_same_host and not self.is_same_host(url):
641 raise HostChangedError(self, url, retries)
643 # Ensure that the URL we're connecting to is properly encoded
644 if url.startswith("/"):
645 url = six.ensure_str(_encode_target(url))
646 else:
647 url = six.ensure_str(parsed_url.url)
649 conn = None
651 # Track whether `conn` needs to be released before
652 # returning/raising/recursing. Update this variable if necessary, and
653 # leave `release_conn` constant throughout the function. That way, if
654 # the function recurses, the original value of `release_conn` will be
655 # passed down into the recursive call, and its value will be respected.
656 #
657 # See issue #651 [1] for details.
658 #
659 # [1] <https://github.com/urllib3/urllib3/issues/651>
660 release_this_conn = release_conn
662 http_tunnel_required = connection_requires_http_tunnel(
663 self.proxy, self.proxy_config, destination_scheme
664 )
666 # Merge the proxy headers. Only done when not using HTTP CONNECT. We
667 # have to copy the headers dict so we can safely change it without those
668 # changes being reflected in anyone else's copy.
669 if not http_tunnel_required:
670 headers = headers.copy()
671 headers.update(self.proxy_headers)
673 # Must keep the exception bound to a separate variable or else Python 3
674 # complains about UnboundLocalError.
675 err = None
677 # Keep track of whether we cleanly exited the except block. This
678 # ensures we do proper cleanup in finally.
679 clean_exit = False
681 # Rewind body position, if needed. Record current position
682 # for future rewinds in the event of a redirect/retry.
683 body_pos = set_file_position(body, body_pos)
685 try:
686 # Request a connection from the queue.
687 timeout_obj = self._get_timeout(timeout)
688 conn = self._get_conn(timeout=pool_timeout)
690 conn.timeout = timeout_obj.connect_timeout
692 is_new_proxy_conn = self.proxy is not None and not getattr(
693 conn, "sock", None
694 )
695 if is_new_proxy_conn and http_tunnel_required:
696 self._prepare_proxy(conn)
698 # Make the request on the httplib connection object.
699 httplib_response = self._make_request(
700 conn,
701 method,
702 url,
703 timeout=timeout_obj,
704 body=body,
705 headers=headers,
706 chunked=chunked,
707 )
709 # If we're going to release the connection in ``finally:``, then
710 # the response doesn't need to know about the connection. Otherwise
711 # it will also try to release it and we'll have a double-release
712 # mess.
713 response_conn = conn if not release_conn else None
715 # Pass method to Response for length checking
716 response_kw["request_method"] = method
718 # Import httplib's response into our own wrapper object
719 response = self.ResponseCls.from_httplib(
720 httplib_response,
721 pool=self,
722 connection=response_conn,
723 retries=retries,
724 **response_kw
725 )
727 # Everything went great!
728 clean_exit = True
730 except EmptyPoolError:
731 # Didn't get a connection from the pool, no need to clean up
732 clean_exit = True
733 release_this_conn = False
734 raise
736 except (
737 TimeoutError,
738 HTTPException,
739 SocketError,
740 ProtocolError,
741 BaseSSLError,
742 SSLError,
743 CertificateError,
744 ) as e:
745 # Discard the connection for these exceptions. It will be
746 # replaced during the next _get_conn() call.
747 clean_exit = False
748 if isinstance(e, (BaseSSLError, CertificateError)):
749 e = SSLError(e)
750 elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
751 e = ProxyError("Cannot connect to proxy.", e)
752 elif isinstance(e, (SocketError, HTTPException)):
753 e = ProtocolError("Connection aborted.", e)
755 retries = retries.increment(
756 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
757 )
758 retries.sleep()
760 # Keep track of the error for the retry warning.
761 err = e
763 finally:
764 if not clean_exit:
765 # We hit some kind of exception, handled or otherwise. We need
766 # to throw the connection away unless explicitly told not to.
767 # Close the connection, set the variable to None, and make sure
768 # we put the None back in the pool to avoid leaking it.
769 conn = conn and conn.close()
770 release_this_conn = True
772 if release_this_conn:
773 # Put the connection back to be reused. If the connection is
774 # expired then it will be None, which will get replaced with a
775 # fresh connection during _get_conn.
776 self._put_conn(conn)
778 if not conn:
779 # Try again
780 log.warning(
781 "Retrying (%r) after connection broken by '%r': %s", retries, err, url
782 )
783 return self.urlopen(
784 method,
785 url,
786 body,
787 headers,
788 retries,
789 redirect,
790 assert_same_host,
791 timeout=timeout,
792 pool_timeout=pool_timeout,
793 release_conn=release_conn,
794 chunked=chunked,
795 body_pos=body_pos,
796 **response_kw
797 )
799 # Handle redirect?
800 redirect_location = redirect and response.get_redirect_location()
801 if redirect_location:
802 if response.status == 303:
803 method = "GET"
805 try:
806 retries = retries.increment(method, url, response=response, _pool=self)
807 except MaxRetryError:
808 if retries.raise_on_redirect:
809 response.drain_conn()
810 raise
811 return response
813 response.drain_conn()
814 retries.sleep_for_retry(response)
815 log.debug("Redirecting %s -> %s", url, redirect_location)
816 return self.urlopen(
817 method,
818 redirect_location,
819 body,
820 headers,
821 retries=retries,
822 redirect=redirect,
823 assert_same_host=assert_same_host,
824 timeout=timeout,
825 pool_timeout=pool_timeout,
826 release_conn=release_conn,
827 chunked=chunked,
828 body_pos=body_pos,
829 **response_kw
830 )
832 # Check if we should retry the HTTP response.
833 has_retry_after = bool(response.getheader("Retry-After"))
834 if retries.is_retry(method, response.status, has_retry_after):
835 try:
836 retries = retries.increment(method, url, response=response, _pool=self)
837 except MaxRetryError:
838 if retries.raise_on_status:
839 response.drain_conn()
840 raise
841 return response
843 response.drain_conn()
844 retries.sleep(response)
845 log.debug("Retry: %s", url)
846 return self.urlopen(
847 method,
848 url,
849 body,
850 headers,
851 retries=retries,
852 redirect=redirect,
853 assert_same_host=assert_same_host,
854 timeout=timeout,
855 pool_timeout=pool_timeout,
856 release_conn=release_conn,
857 chunked=chunked,
858 body_pos=body_pos,
859 **response_kw
860 )
862 return response
865class HTTPSConnectionPool(HTTPConnectionPool):
866 """
867 Same as :class:`.HTTPConnectionPool`, but HTTPS.
869 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
870 ``assert_hostname`` and ``host`` in this order to verify connections.
871 If ``assert_hostname`` is False, no verification is done.
873 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
874 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
875 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
876 the connection socket into an SSL socket.
877 """
879 scheme = "https"
880 ConnectionCls = HTTPSConnection
882 def __init__(
883 self,
884 host,
885 port=None,
886 strict=False,
887 timeout=Timeout.DEFAULT_TIMEOUT,
888 maxsize=1,
889 block=False,
890 headers=None,
891 retries=None,
892 _proxy=None,
893 _proxy_headers=None,
894 key_file=None,
895 cert_file=None,
896 cert_reqs=None,
897 key_password=None,
898 ca_certs=None,
899 ssl_version=None,
900 assert_hostname=None,
901 assert_fingerprint=None,
902 ca_cert_dir=None,
903 **conn_kw
904 ):
906 HTTPConnectionPool.__init__(
907 self,
908 host,
909 port,
910 strict,
911 timeout,
912 maxsize,
913 block,
914 headers,
915 retries,
916 _proxy,
917 _proxy_headers,
918 **conn_kw
919 )
921 self.key_file = key_file
922 self.cert_file = cert_file
923 self.cert_reqs = cert_reqs
924 self.key_password = key_password
925 self.ca_certs = ca_certs
926 self.ca_cert_dir = ca_cert_dir
927 self.ssl_version = ssl_version
928 self.assert_hostname = assert_hostname
929 self.assert_fingerprint = assert_fingerprint
931 def _prepare_conn(self, conn):
932 """
933 Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
934 and establish the tunnel if proxy is used.
935 """
937 if isinstance(conn, VerifiedHTTPSConnection):
938 conn.set_cert(
939 key_file=self.key_file,
940 key_password=self.key_password,
941 cert_file=self.cert_file,
942 cert_reqs=self.cert_reqs,
943 ca_certs=self.ca_certs,
944 ca_cert_dir=self.ca_cert_dir,
945 assert_hostname=self.assert_hostname,
946 assert_fingerprint=self.assert_fingerprint,
947 )
948 conn.ssl_version = self.ssl_version
949 return conn
951 def _prepare_proxy(self, conn):
952 """
953 Establishes a tunnel connection through HTTP CONNECT.
955 Tunnel connection is established early because otherwise httplib would
956 improperly set Host: header to proxy's IP:port.
957 """
959 conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
961 if self.proxy.scheme == "https":
962 conn.tls_in_tls_required = True
964 conn.connect()
966 def _new_conn(self):
967 """
968 Return a fresh :class:`http.client.HTTPSConnection`.
969 """
970 self.num_connections += 1
971 log.debug(
972 "Starting new HTTPS connection (%d): %s:%s",
973 self.num_connections,
974 self.host,
975 self.port or "443",
976 )
978 if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
979 raise SSLError(
980 "Can't connect to HTTPS URL because the SSL module is not available."
981 )
983 actual_host = self.host
984 actual_port = self.port
985 if self.proxy is not None:
986 actual_host = self.proxy.host
987 actual_port = self.proxy.port
989 conn = self.ConnectionCls(
990 host=actual_host,
991 port=actual_port,
992 timeout=self.timeout.connect_timeout,
993 strict=self.strict,
994 cert_file=self.cert_file,
995 key_file=self.key_file,
996 key_password=self.key_password,
997 **self.conn_kw
998 )
1000 return self._prepare_conn(conn)
1002 def _validate_conn(self, conn):
1003 """
1004 Called right before a request is made, after the socket is created.
1005 """
1006 super(HTTPSConnectionPool, self)._validate_conn(conn)
1008 # Force connect early to allow us to validate the connection.
1009 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
1010 conn.connect()
1012 if not conn.is_verified:
1013 warnings.warn(
1014 (
1015 "Unverified HTTPS request is being made to host '%s'. "
1016 "Adding certificate verification is strongly advised. See: "
1017 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1018 "#ssl-warnings" % conn.host
1019 ),
1020 InsecureRequestWarning,
1021 )
1024def connection_from_url(url, **kw):
1025 """
1026 Given a url, return an :class:`.ConnectionPool` instance of its host.
1028 This is a shortcut for not having to parse out the scheme, host, and port
1029 of the url before creating an :class:`.ConnectionPool` instance.
1031 :param url:
1032 Absolute URL string that must include the scheme. Port is optional.
1034 :param \\**kw:
1035 Passes additional parameters to the constructor of the appropriate
1036 :class:`.ConnectionPool`. Useful for specifying things like
1037 timeout, maxsize, headers, etc.
1039 Example::
1041 >>> conn = connection_from_url('http://google.com/')
1042 >>> r = conn.request('GET', '/')
1043 """
1044 scheme, host, port = get_host(url)
1045 port = port or port_by_scheme.get(scheme, 80)
1046 if scheme == "https":
1047 return HTTPSConnectionPool(host, port=port, **kw)
1048 else:
1049 return HTTPConnectionPool(host, port=port, **kw)
1052def _normalize_host(host, scheme):
1053 """
1054 Normalize hosts for comparisons and use with sockets.
1055 """
1057 host = normalize_host(host, scheme)
1059 # httplib doesn't like it when we include brackets in IPv6 addresses
1060 # Specifically, if we include brackets but also pass the port then
1061 # httplib crazily doubles up the square brackets on the Host header.
1062 # Instead, we need to make sure we never pass ``None`` as the port.
1063 # However, for backward compatibility reasons we can't actually
1064 # *assert* that. See http://bugs.python.org/issue28539
1065 if host.startswith("[") and host.endswith("]"):
1066 host = host[1:-1]
1067 return host