Package biblio :: Package webquery :: Module querythrottle
[hide private]
[frames] | no frames]

Source Code for Module biblio.webquery.querythrottle

  1  #! /usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  """ 
  4  Classes for throttling web-queries, so as to stay within limits. 
  5  """ 
  6  # TODO: error-handling logic is correct? 
  7   
  8  __docformat__ = 'restructuredtext en' 
  9   
 10   
 11  ### IMPORTS ### 
 12   
 13  import time 
 14  from exceptions import ValueError, RuntimeError 
 15   
 16  import impl 
 17  import errors 
 18   
 19   
 20  ### CONSTANTS & DEFINES ### 
 21   
 22  FAIL_AND_RAISE = 'RAISE' 
 23  FAIL_AND_WAIT = 'WAIT' 
 24   
 25   
 26  ### IMPLEMENTATION ### 
 27   
28 -class BaseQueryThrottle (impl.ReprObj):
29 """ 30 A limit upon query usage. 31 32 Often webservices will request that users restrict themselves to a request 33 every second, or no more than 1000 a day, etc. This is a base class for 34 implementing those limits. Different restrictions can be implemented in 35 derived classes. 36 37 Limits are constructed with set behaviour 38 39 """ 40 # TODO: introduce special handlers for various failure actions? 41 42 _repr_fields = [ 43 'fail_action', 44 'wait_duration', 45 ] 46
47 - def __init__ (self, fail_action=None, wait_duration=1.0):
48 """ 49 Ctor, allowing the polling period and failure behaviour to be set. 50 51 """ 52 self.fail_action = fail_action or FAIL_AND_RAISE 53 self.wait_duration = wait_duration
54
55 - def check_limit (self, wquery):
56 """ 57 Has the query exceeded its limit? 58 59 :Parameters: 60 wquery 61 The object or service to be throttled. This allows the same limit 62 to service several objects in different ways (e.g. by having them 63 share the same limit, or be handled independently). 64 65 :Returns: 66 A boolean, giving whether the query is within limit or not. 67 68 This is a primarily internal method for testing whether a limit has been 69 reached. Handling that circumstance is left to the calling method 70 `check_limit`. This should be overridden in derived class to implement 71 different throttling methods. 72 73 """ 74 if (self.within_limit (wquery)): 75 self.log_success (wquery) 76 else: 77 if (self.fail_action == FAIL_AND_RAISE): 78 raise errors.QueryThrottleError() 79 elif (self.fail_action == FAIL_AND_WAIT): 80 time.sleep (self.wait_duration) 81 while (not self.within_limit (wquery)): 82 time.sleep (self.wait_duration) 83 self.log_success (wquery) 84 else: 85 raise ValueError ("unrecognised fail action '%s'" % 86 self.fail_action)
87
88 - def within_limit (self, wquery):
89 """ 90 Has the query exceeded its limit? 91 92 :Parameters: 93 wquery 94 See `check_limit` 95 96 This should be called by services to test whether a limit has been 97 reached. Handling that circumstance is left to the calling method 98 `check_limit`. This should be overridden in derived class to implement 99 different throttling methods. 100 101 """ 102 return True
103 104
105 - def log_success (self, wquery):
106 """ 107 Sucessful queries will probably effect the success of future ones, so 108 here is a place to log them. 109 """ 110 pass
111 112
113 -class WaitNSecondsThrottle (BaseQueryThrottle):
114 """ 115 Limit a query to every N seconds at most. 116 117 By default this throttle holds the query until the wait period is over. 118 Note that this throttle can be used across a set of queries, so that the 119 limit applies for the set. In this case the waiting behaviour could be 120 undesirable, with a large population of queries on hold. 121 122 """
123 - def __init__ (self, wait, fail_action=FAIL_AND_WAIT):
124 """ 125 C'tor, allowing the wait period and failure behaviour to be set. 126 127 :Parameters: 128 wait : int or float 129 The period to enforce between queries. 130 fail_action 131 See `BaseQueryThrottle`. 132 133 """ 134 BaseQueryThrottle.__init__ (self, fail_action) 135 self.wait = wait 136 self._prev_time = 0
137
138 - def within_limit (self, wquery):
139 """ 140 Has it been longer than the wait period since the last query? 141 142 """ 143 return (self.wait < (time.time() - self._prev_time))
144
145 - def log_success (self, wquery):
146 self._prev_time = time.time()
147 148
149 -class WaitOneSecondThrottle (WaitNSecondsThrottle):
150 """ 151 Limit a query to once every second at most. 152 153 This is a common limit, and so is provided as a convenience. 154 155 """
156 - def __init__ (self, fail_action=FAIL_AND_WAIT):
157 WaitNSecondsThrottle.__init__ (self, wait=1.0, fail_action=fail_action)
158 159
160 -class AbsoluteNumberThrottle (BaseQueryThrottle):
161 """ 162 Limit a query to a maximum number. 163 164 Many web-services have a per-day query limit (e.g. 500 per day for ISBNdb). 165 It is difficult to implement this across multiple invocations of the 166 query objects and Python interpreter, but this can serve as a crude 167 implementation. By default, it raises an exception if the limit is 168 reached. 169 170 """
171 - def __init__ (self, max, fail_action=FAIL_AND_RAISE):
172 """ 173 C'tor, allowing the maximum queries and failure behaviour to be set. 174 175 :Parameters: 176 max : int 177 The total number of queries allowed. 178 fail_action 179 See `BaseQueryThrottle`. 180 181 """ 182 BaseQueryThrottle.__init__ (self, fail_action) 183 self.max = max 184 self._query_count = 0
185
186 - def within_limit (self, wquery):
187 """ 188 Have fewer queries been posted than the limit? 189 190 Note that if multiple queries simulatanoeusly test via this function, 191 exceeding the limit is possible. 192 193 """ 194 return (self._query_count < self.max)
195
196 - def log_success (self, wquery):
197 self._query_count += 1
198 199
200 -class Max500Throttle (object):
201 """ 202 Limit a query to 500 attempts at most. 203 204 This is a common limit, and so is provided as a convenience. 205 206 """
207 - def __init__ (self):
209 210 211 212 ### TEST & DEBUG ### 213
214 -def _doctest ():
215 import doctest 216 doctest.testmod()
217 218 219 ### MAIN ### 220 221 if __name__ == '__main__': 222 _doctest() 223 224 225 ### END ###################################################################### 226