1
2
3 """
4 Classes for throttling web-queries, so as to stay within limits.
5 """
6
7
8 __docformat__ = 'restructuredtext en'
9
10
11
12
13 import time
14 from exceptions import ValueError, RuntimeError
15
16 import impl
17 import errors
18
19
20
21
22 FAIL_AND_RAISE = 'RAISE'
23 FAIL_AND_WAIT = 'WAIT'
24
25
26
27
29 """
30 A limit upon query usage.
31
32 Often webservices will request that users restrict themselves to a request
33 every second, or no more than 1000 a day, etc. This is a base class for
34 implementing those limits. Different restrictions can be implemented in
35 derived classes.
36
37 Limits are constructed with set behaviour
38
39 """
40
41
42 _repr_fields = [
43 'fail_action',
44 'wait_duration',
45 ]
46
47 - def __init__ (self, fail_action=None, wait_duration=1.0):
48 """
49 Ctor, allowing the polling period and failure behaviour to be set.
50
51 """
52 self.fail_action = fail_action or FAIL_AND_RAISE
53 self.wait_duration = wait_duration
54
56 """
57 Has the query exceeded its limit?
58
59 :Parameters:
60 wquery
61 The object or service to be throttled. This allows the same limit
62 to service several objects in different ways (e.g. by having them
63 share the same limit, or be handled independently).
64
65 :Returns:
66 A boolean, giving whether the query is within limit or not.
67
68 This is a primarily internal method for testing whether a limit has been
69 reached. Handling that circumstance is left to the calling method
70 `check_limit`. This should be overridden in derived class to implement
71 different throttling methods.
72
73 """
74 if (self.within_limit (wquery)):
75 self.log_success (wquery)
76 else:
77 if (self.fail_action == FAIL_AND_RAISE):
78 raise errors.QueryThrottleError()
79 elif (self.fail_action == FAIL_AND_WAIT):
80 time.sleep (self.wait_duration)
81 while (not self.within_limit (wquery)):
82 time.sleep (self.wait_duration)
83 self.log_success (wquery)
84 else:
85 raise ValueError ("unrecognised fail action '%s'" %
86 self.fail_action)
87
89 """
90 Has the query exceeded its limit?
91
92 :Parameters:
93 wquery
94 See `check_limit`
95
96 This should be called by services to test whether a limit has been
97 reached. Handling that circumstance is left to the calling method
98 `check_limit`. This should be overridden in derived class to implement
99 different throttling methods.
100
101 """
102 return True
103
104
106 """
107 Sucessful queries will probably effect the success of future ones, so
108 here is a place to log them.
109 """
110 pass
111
112
114 """
115 Limit a query to every N seconds at most.
116
117 By default this throttle holds the query until the wait period is over.
118 Note that this throttle can be used across a set of queries, so that the
119 limit applies for the set. In this case the waiting behaviour could be
120 undesirable, with a large population of queries on hold.
121
122 """
124 """
125 C'tor, allowing the wait period and failure behaviour to be set.
126
127 :Parameters:
128 wait : int or float
129 The period to enforce between queries.
130 fail_action
131 See `BaseQueryThrottle`.
132
133 """
134 BaseQueryThrottle.__init__ (self, fail_action)
135 self.wait = wait
136 self._prev_time = 0
137
139 """
140 Has it been longer than the wait period since the last query?
141
142 """
143 return (self.wait < (time.time() - self._prev_time))
144
146 self._prev_time = time.time()
147
148
150 """
151 Limit a query to once every second at most.
152
153 This is a common limit, and so is provided as a convenience.
154
155 """
158
159
161 """
162 Limit a query to a maximum number.
163
164 Many web-services have a per-day query limit (e.g. 500 per day for ISBNdb).
165 It is difficult to implement this across multiple invocations of the
166 query objects and Python interpreter, but this can serve as a crude
167 implementation. By default, it raises an exception if the limit is
168 reached.
169
170 """
172 """
173 C'tor, allowing the maximum queries and failure behaviour to be set.
174
175 :Parameters:
176 max : int
177 The total number of queries allowed.
178 fail_action
179 See `BaseQueryThrottle`.
180
181 """
182 BaseQueryThrottle.__init__ (self, fail_action)
183 self.max = max
184 self._query_count = 0
185
187 """
188 Have fewer queries been posted than the limit?
189
190 Note that if multiple queries simulatanoeusly test via this function,
191 exceeding the limit is possible.
192
193 """
194 return (self._query_count < self.max)
195
197 self._query_count += 1
198
199
201 """
202 Limit a query to 500 attempts at most.
203
204 This is a common limit, and so is provided as a convenience.
205
206 """
209
210
211
212
213
215 import doctest
216 doctest.testmod()
217
218
219
220
221 if __name__ == '__main__':
222 _doctest()
223
224
225
226