Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

# urllib3/response.py 

# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 

# 

# This module is part of urllib3 and is released under 

# the MIT License: http://www.opensource.org/licenses/mit-license.php 

 

 

import logging 

import zlib 

import io 

 

from ._collections import HTTPHeaderDict 

from .exceptions import DecodeError 

from .packages.six import string_types as basestring, binary_type 

from .util import is_fp_closed 

 

 

log = logging.getLogger(__name__) 

 

 

class DeflateDecoder(object): 

 

    def __init__(self): 

        self._first_try = True 

        self._data = binary_type() 

        self._obj = zlib.decompressobj() 

 

    def __getattr__(self, name): 

        return getattr(self._obj, name) 

 

    def decompress(self, data): 

        if not self._first_try: 

            return self._obj.decompress(data) 

 

        self._data += data 

        try: 

            return self._obj.decompress(data) 

        except zlib.error: 

            self._first_try = False 

            self._obj = zlib.decompressobj(-zlib.MAX_WBITS) 

            try: 

                return self.decompress(self._data) 

            finally: 

                self._data = None 

 

 

def _get_decoder(mode): 

    if mode == 'gzip': 

        return zlib.decompressobj(16 + zlib.MAX_WBITS) 

 

    return DeflateDecoder() 

 

 

class HTTPResponse(io.IOBase): 

    """ 

    HTTP Response container. 

 

    Backwards-compatible to httplib's HTTPResponse but the response ``body`` is 

    loaded and decoded on-demand when the ``data`` property is accessed. 

 

    Extra parameters for behaviour not present in httplib.HTTPResponse: 

 

    :param preload_content: 

        If True, the response's body will be preloaded during construction. 

 

    :param decode_content: 

        If True, attempts to decode specific content-encoding's based on headers 

        (like 'gzip' and 'deflate') will be skipped and raw data will be used 

        instead. 

 

    :param original_response: 

        When this HTTPResponse wrapper is generated from an httplib.HTTPResponse 

        object, it's convenient to include the original for debug purposes. It's 

        otherwise unused. 

    """ 

 

    CONTENT_DECODERS = ['gzip', 'deflate'] 

    REDIRECT_STATUSES = [301, 302, 303, 307, 308] 

 

    def __init__(self, body='', headers=None, status=0, version=0, reason=None, 

                 strict=0, preload_content=True, decode_content=True, 

                 original_response=None, pool=None, connection=None): 

 

        self.headers = HTTPHeaderDict() 

        if headers: 

            self.headers.update(headers) 

        self.status = status 

        self.version = version 

        self.reason = reason 

        self.strict = strict 

        self.decode_content = decode_content 

 

        self._decoder = None 

        self._body = body if body and isinstance(body, basestring) else None 

        self._fp = None 

        self._original_response = original_response 

        self._fp_bytes_read = 0 

 

        self._pool = pool 

        self._connection = connection 

 

        if hasattr(body, 'read'): 

            self._fp = body 

 

        if preload_content and not self._body: 

            self._body = self.read(decode_content=decode_content) 

 

    def get_redirect_location(self): 

        """ 

        Should we redirect and where to? 

 

        :returns: Truthy redirect location string if we got a redirect status 

            code and valid location. ``None`` if redirect status and no 

            location. ``False`` if not a redirect status code. 

        """ 

        if self.status in self.REDIRECT_STATUSES: 

            return self.headers.get('location') 

 

        return False 

 

    def release_conn(self): 

        if not self._pool or not self._connection: 

            return 

 

        self._pool._put_conn(self._connection) 

        self._connection = None 

 

    @property 

    def data(self): 

        # For backwords-compat with earlier urllib3 0.4 and earlier. 

        if self._body: 

            return self._body 

 

        if self._fp: 

            return self.read(cache_content=True) 

 

    def tell(self): 

        """ 

        Obtain the number of bytes pulled over the wire so far. May differ from 

        the amount of content returned by :meth:``HTTPResponse.read`` if bytes 

        are encoded on the wire (e.g, compressed). 

        """ 

        return self._fp_bytes_read 

 

    def read(self, amt=None, decode_content=None, cache_content=False): 

        """ 

        Similar to :meth:`httplib.HTTPResponse.read`, but with two additional 

        parameters: ``decode_content`` and ``cache_content``. 

 

        :param amt: 

            How much of the content to read. If specified, caching is skipped 

            because it doesn't make sense to cache partial content as the full 

            response. 

 

        :param decode_content: 

            If True, will attempt to decode the body based on the 

            'content-encoding' header. 

 

        :param cache_content: 

            If True, will save the returned data such that the same result is 

            returned despite of the state of the underlying file object. This 

            is useful if you want the ``.data`` property to continue working 

            after having ``.read()`` the file object. (Overridden if ``amt`` is 

            set.) 

        """ 

        # Note: content-encoding value should be case-insensitive, per RFC 2616 

        # Section 3.5 

        content_encoding = self.headers.get('content-encoding', '').lower() 

        if self._decoder is None: 

            if content_encoding in self.CONTENT_DECODERS: 

                self._decoder = _get_decoder(content_encoding) 

        if decode_content is None: 

            decode_content = self.decode_content 

 

        if self._fp is None: 

            return 

 

        flush_decoder = False 

 

        try: 

            if amt is None: 

                # cStringIO doesn't like amt=None 

                data = self._fp.read() 

                flush_decoder = True 

            else: 

                cache_content = False 

                data = self._fp.read(amt) 

                if amt != 0 and not data:  # Platform-specific: Buggy versions of Python. 

                    # Close the connection when no data is returned 

                    # 

                    # This is redundant to what httplib/http.client _should_ 

                    # already do.  However, versions of python released before 

                    # December 15, 2012 (http://bugs.python.org/issue16298) do not 

                    # properly close the connection in all cases. There is no harm 

                    # in redundantly calling close. 

                    self._fp.close() 

                    flush_decoder = True 

 

            self._fp_bytes_read += len(data) 

 

            try: 

                if decode_content and self._decoder: 

                    data = self._decoder.decompress(data) 

            except (IOError, zlib.error) as e: 

                raise DecodeError( 

                    "Received response with content-encoding: %s, but " 

                    "failed to decode it." % content_encoding, 

                    e) 

 

            if flush_decoder and decode_content and self._decoder: 

                buf = self._decoder.decompress(binary_type()) 

                data += buf + self._decoder.flush() 

 

            if cache_content: 

                self._body = data 

 

            return data 

 

        finally: 

            if self._original_response and self._original_response.isclosed(): 

                self.release_conn() 

 

    def stream(self, amt=2**16, decode_content=None): 

        """ 

        A generator wrapper for the read() method. A call will block until 

        ``amt`` bytes have been read from the connection or until the 

        connection is closed. 

 

        :param amt: 

            How much of the content to read. The generator will return up to 

            much data per iteration, but may return less. This is particularly 

            likely when using compressed data. However, the empty string will 

            never be returned. 

 

        :param decode_content: 

            If True, will attempt to decode the body based on the 

            'content-encoding' header. 

        """ 

        while not is_fp_closed(self._fp): 

            data = self.read(amt=amt, decode_content=decode_content) 

 

            if data: 

                yield data 

 

 

    @classmethod 

    def from_httplib(ResponseCls, r, **response_kw): 

        """ 

        Given an :class:`httplib.HTTPResponse` instance ``r``, return a 

        corresponding :class:`urllib3.response.HTTPResponse` object. 

 

        Remaining parameters are passed to the HTTPResponse constructor, along 

        with ``original_response=r``. 

        """ 

 

        headers = HTTPHeaderDict() 

        for k, v in r.getheaders(): 

            headers.add(k, v) 

 

        # HTTPResponse objects in Python 3 don't have a .strict attribute 

        strict = getattr(r, 'strict', 0) 

        return ResponseCls(body=r, 

                           headers=headers, 

                           status=r.status, 

                           version=r.version, 

                           reason=r.reason, 

                           strict=strict, 

                           original_response=r, 

                           **response_kw) 

 

    # Backwards-compatibility methods for httplib.HTTPResponse 

    def getheaders(self): 

        return self.headers 

 

    def getheader(self, name, default=None): 

        return self.headers.get(name, default) 

 

    # Overrides from io.IOBase 

    def close(self): 

        if not self.closed: 

            self._fp.close() 

 

    @property 

    def closed(self): 

        if self._fp is None: 

            return True 

        elif hasattr(self._fp, 'closed'): 

            return self._fp.closed 

        elif hasattr(self._fp, 'isclosed'):  # Python 2 

            return self._fp.isclosed() 

        else: 

            return True 

 

    def fileno(self): 

        if self._fp is None: 

            raise IOError("HTTPResponse has no file to get a fileno from") 

        elif hasattr(self._fp, "fileno"): 

            return self._fp.fileno() 

        else: 

            raise IOError("The file-like object  this HTTPResponse is wrapped " 

                          "around has no file descriptor") 

 

    def flush(self): 

        if self._fp is not None and hasattr(self._fp, 'flush'): 

            return self._fp.flush() 

 

    def readable(self): 

        return True