1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """
20 BufferedFile.
21 """
22
23 from cStringIO import StringIO
27 """
28 Reusable base class to implement python-style file buffering around a
29 simpler stream.
30 """
31
32 _DEFAULT_BUFSIZE = 8192
33
34 SEEK_SET = 0
35 SEEK_CUR = 1
36 SEEK_END = 2
37
38 FLAG_READ = 0x1
39 FLAG_WRITE = 0x2
40 FLAG_APPEND = 0x4
41 FLAG_BINARY = 0x10
42 FLAG_BUFFERED = 0x20
43 FLAG_LINE_BUFFERED = 0x40
44 FLAG_UNIVERSAL_NEWLINE = 0x80
45
47 self.newlines = None
48 self._flags = 0
49 self._bufsize = self._DEFAULT_BUFSIZE
50 self._wbuffer = StringIO()
51 self._rbuffer = ''
52 self._at_trailing_cr = False
53 self._closed = False
54
55
56
57 self._pos = self._realpos = 0
58
59 self._size = 0
60
63
65 """
66 Returns an iterator that can be used to iterate over the lines in this
67 file. This iterator happens to return the file itself, since a file is
68 its own iterator.
69
70 @raise ValueError: if the file is closed.
71
72 @return: an interator.
73 @rtype: iterator
74 """
75 if self._closed:
76 raise ValueError('I/O operation on closed file')
77 return self
78
80 """
81 Close the file. Future read and write operations will fail.
82 """
83 self.flush()
84 self._closed = True
85
87 """
88 Write out any data in the write buffer. This may do nothing if write
89 buffering is not turned on.
90 """
91 self._write_all(self._wbuffer.getvalue())
92 self._wbuffer = StringIO()
93 return
94
96 """
97 Returns the next line from the input, or raises L{StopIteration} when
98 EOF is hit. Unlike python file objects, it's okay to mix calls to
99 C{next} and L{readline}.
100
101 @raise StopIteration: when the end of the file is reached.
102
103 @return: a line read from the file.
104 @rtype: str
105 """
106 line = self.readline()
107 if not line:
108 raise StopIteration
109 return line
110
111 - def read(self, size=None):
112 """
113 Read at most C{size} bytes from the file (less if we hit the end of the
114 file first). If the C{size} argument is negative or omitted, read all
115 the remaining data in the file.
116
117 @param size: maximum number of bytes to read
118 @type size: int
119 @return: data read from the file, or an empty string if EOF was
120 encountered immediately
121 @rtype: str
122 """
123 if self._closed:
124 raise IOError('File is closed')
125 if not (self._flags & self.FLAG_READ):
126 raise IOError('File is not open for reading')
127 if (size is None) or (size < 0):
128
129 result = self._rbuffer
130 self._rbuffer = ''
131 self._pos += len(result)
132 while True:
133 try:
134 new_data = self._read(self._DEFAULT_BUFSIZE)
135 except EOFError:
136 new_data = None
137 if (new_data is None) or (len(new_data) == 0):
138 break
139 result += new_data
140 self._realpos += len(new_data)
141 self._pos += len(new_data)
142 return result
143 if size <= len(self._rbuffer):
144 result = self._rbuffer[:size]
145 self._rbuffer = self._rbuffer[size:]
146 self._pos += len(result)
147 return result
148 while len(self._rbuffer) < size:
149 read_size = size - len(self._rbuffer)
150 if self._flags & self.FLAG_BUFFERED:
151 read_size = max(self._bufsize, read_size)
152 try:
153 new_data = self._read(read_size)
154 except EOFError:
155 new_data = None
156 if (new_data is None) or (len(new_data) == 0):
157 break
158 self._rbuffer += new_data
159 self._realpos += len(new_data)
160 result = self._rbuffer[:size]
161 self._rbuffer = self._rbuffer[size:]
162 self._pos += len(result)
163 return result
164
166 """
167 Read one entire line from the file. A trailing newline character is
168 kept in the string (but may be absent when a file ends with an
169 incomplete line). If the size argument is present and non-negative, it
170 is a maximum byte count (including the trailing newline) and an
171 incomplete line may be returned. An empty string is returned only when
172 EOF is encountered immediately.
173
174 @note: Unlike stdio's C{fgets()}, the returned string contains null
175 characters (C{'\\0'}) if they occurred in the input.
176
177 @param size: maximum length of returned string.
178 @type size: int
179 @return: next line of the file, or an empty string if the end of the
180 file has been reached.
181 @rtype: str
182 """
183
184 if self._closed:
185 raise IOError('File is closed')
186 if not (self._flags & self.FLAG_READ):
187 raise IOError('File not open for reading')
188 line = self._rbuffer
189 while True:
190 if self._at_trailing_cr and (self._flags & self.FLAG_UNIVERSAL_NEWLINE) and (len(line) > 0):
191
192
193 if line[0] == '\n':
194 line = line[1:]
195 self._record_newline('\r\n')
196 else:
197 self._record_newline('\r')
198 self._at_trailing_cr = False
199
200
201 if (size is not None) and (size >= 0):
202 if len(line) >= size:
203
204 self._rbuffer = line[size:]
205 line = line[:size]
206 self._pos += len(line)
207 return line
208 n = size - len(line)
209 else:
210 n = self._bufsize
211 if ('\n' in line) or ((self._flags & self.FLAG_UNIVERSAL_NEWLINE) and ('\r' in line)):
212 break
213 try:
214 new_data = self._read(n)
215 except EOFError:
216 new_data = None
217 if (new_data is None) or (len(new_data) == 0):
218 self._rbuffer = ''
219 self._pos += len(line)
220 return line
221 line += new_data
222 self._realpos += len(new_data)
223
224 pos = line.find('\n')
225 if self._flags & self.FLAG_UNIVERSAL_NEWLINE:
226 rpos = line.find('\r')
227 if (rpos >= 0) and ((rpos < pos) or (pos < 0)):
228 pos = rpos
229 xpos = pos + 1
230 if (line[pos] == '\r') and (xpos < len(line)) and (line[xpos] == '\n'):
231 xpos += 1
232 self._rbuffer = line[xpos:]
233 lf = line[pos:xpos]
234 line = line[:pos] + '\n'
235 if (len(self._rbuffer) == 0) and (lf == '\r'):
236
237
238 self._at_trailing_cr = True
239 else:
240 self._record_newline(lf)
241 self._pos += len(line)
242 return line
243
245 """
246 Read all remaining lines using L{readline} and return them as a list.
247 If the optional C{sizehint} argument is present, instead of reading up
248 to EOF, whole lines totalling approximately sizehint bytes (possibly
249 after rounding up to an internal buffer size) are read.
250
251 @param sizehint: desired maximum number of bytes to read.
252 @type sizehint: int
253 @return: list of lines read from the file.
254 @rtype: list
255 """
256 lines = []
257 bytes = 0
258 while True:
259 line = self.readline()
260 if len(line) == 0:
261 break
262 lines.append(line)
263 bytes += len(line)
264 if (sizehint is not None) and (bytes >= sizehint):
265 break
266 return lines
267
268 - def seek(self, offset, whence=0):
269 """
270 Set the file's current position, like stdio's C{fseek}. Not all file
271 objects support seeking.
272
273 @note: If a file is opened in append mode (C{'a'} or C{'a+'}), any seek
274 operations will be undone at the next write (as the file position
275 will move back to the end of the file).
276
277 @param offset: position to move to within the file, relative to
278 C{whence}.
279 @type offset: int
280 @param whence: type of movement: 0 = absolute; 1 = relative to the
281 current position; 2 = relative to the end of the file.
282 @type whence: int
283
284 @raise IOError: if the file doesn't support random access.
285 """
286 raise IOError('File does not support seeking.')
287
289 """
290 Return the file's current position. This may not be accurate or
291 useful if the underlying file doesn't support random access, or was
292 opened in append mode.
293
294 @return: file position (in bytes).
295 @rtype: int
296 """
297 return self._pos
298
300 """
301 Write data to the file. If write buffering is on (C{bufsize} was
302 specified and non-zero), some or all of the data may not actually be
303 written yet. (Use L{flush} or L{close} to force buffered data to be
304 written out.)
305
306 @param data: data to write.
307 @type data: str
308 """
309 if self._closed:
310 raise IOError('File is closed')
311 if not (self._flags & self.FLAG_WRITE):
312 raise IOError('File not open for writing')
313 if not (self._flags & self.FLAG_BUFFERED):
314 self._write_all(data)
315 return
316 self._wbuffer.write(data)
317 if self._flags & self.FLAG_LINE_BUFFERED:
318
319 last_newline_pos = data.rfind('\n')
320 if last_newline_pos >= 0:
321 wbuf = self._wbuffer.getvalue()
322 last_newline_pos += len(wbuf) - len(data)
323 self._write_all(wbuf[:last_newline_pos + 1])
324 self._wbuffer = StringIO()
325 self._wbuffer.write(wbuf[last_newline_pos + 1:])
326 return
327
328
329 if self._wbuffer.tell() >= self._bufsize:
330 self.flush()
331 return
332
334 """
335 Write a sequence of strings to the file. The sequence can be any
336 iterable object producing strings, typically a list of strings. (The
337 name is intended to match L{readlines}; C{writelines} does not add line
338 separators.)
339
340 @param sequence: an iterable sequence of strings.
341 @type sequence: sequence
342 """
343 for line in sequence:
344 self.write(line)
345 return
346
348 """
349 Identical to C{iter(f)}. This is a deprecated file interface that
350 predates python iterator support.
351
352 @return: an iterator.
353 @rtype: iterator
354 """
355 return self
356
357 @property
360
361
362
363
364
366 """
367 I{(subclass override)}
368 Read data from the stream. Return C{None} or raise C{EOFError} to
369 indicate EOF.
370 """
371 raise EOFError()
372
374 """
375 I{(subclass override)}
376 Write data into the stream.
377 """
378 raise IOError('write not implemented')
379
381 """
382 I{(subclass override)}
383 Return the size of the file. This is called from within L{_set_mode}
384 if the file is opened in append mode, so the file position can be
385 tracked and L{seek} and L{tell} will work correctly. If the file is
386 a stream that can't be randomly accessed, you don't need to override
387 this method,
388 """
389 return 0
390
391
392
393
394
396 """
397 Subclasses call this method to initialize the BufferedFile.
398 """
399
400 self._bufsize = self._DEFAULT_BUFSIZE
401 if bufsize < 0:
402
403
404 bufsize = 0
405 if bufsize == 1:
406
407
408
409 self._flags |= self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED
410 elif bufsize > 1:
411 self._bufsize = bufsize
412 self._flags |= self.FLAG_BUFFERED
413 self._flags &= ~self.FLAG_LINE_BUFFERED
414 elif bufsize == 0:
415
416 self._flags &= ~(self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED)
417
418 if ('r' in mode) or ('+' in mode):
419 self._flags |= self.FLAG_READ
420 if ('w' in mode) or ('+' in mode):
421 self._flags |= self.FLAG_WRITE
422 if ('a' in mode):
423 self._flags |= self.FLAG_WRITE | self.FLAG_APPEND
424 self._size = self._get_size()
425 self._pos = self._realpos = self._size
426 if ('b' in mode):
427 self._flags |= self.FLAG_BINARY
428 if ('U' in mode):
429 self._flags |= self.FLAG_UNIVERSAL_NEWLINE
430
431
432
433 self.newlines = None
434
436
437
438 while len(data) > 0:
439 count = self._write(data)
440 data = data[count:]
441 if self._flags & self.FLAG_APPEND:
442 self._size += count
443 self._pos = self._realpos = self._size
444 else:
445 self._pos += count
446 self._realpos += count
447 return None
448
450
451
452
453 if not (self._flags & self.FLAG_UNIVERSAL_NEWLINE):
454 return
455 if self.newlines is None:
456 self.newlines = newline
457 elif (type(self.newlines) is str) and (self.newlines != newline):
458 self.newlines = (self.newlines, newline)
459 elif newline not in self.newlines:
460 self.newlines += (newline,)
461