Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import with_statement 

2 

3import contextlib 

4import datetime 

5from functools import partial 

6from functools import wraps 

7import logging 

8from numbers import Number 

9import threading 

10import time 

11 

12from decorator import decorate 

13 

14from . import exception 

15from .api import CachedValue 

16from .api import NO_VALUE 

17from .backends import _backend_loader 

18from .backends import register_backend # noqa 

19from .proxy import ProxyBackend 

20from .util import function_key_generator 

21from .util import function_multi_key_generator 

22from .util import repr_obj 

23from .. import Lock 

24from .. import NeedRegenerationException 

25from ..util import coerce_string_conf 

26from ..util import compat 

27from ..util import memoized_property 

28from ..util import NameRegistry 

29from ..util import PluginLoader 

30 

31value_version = 1 

32"""An integer placed in the :class:`.CachedValue` 

33so that new versions of dogpile.cache can detect cached 

34values from a previous, backwards-incompatible version. 

35 

36""" 

37 

38log = logging.getLogger(__name__) 

39 

40 

41class RegionInvalidationStrategy(object): 

42 """Region invalidation strategy interface 

43 

44 Implement this interface and pass implementation instance 

45 to :meth:`.CacheRegion.configure` to override default region invalidation. 

46 

47 Example:: 

48 

49 class CustomInvalidationStrategy(RegionInvalidationStrategy): 

50 

51 def __init__(self): 

52 self._soft_invalidated = None 

53 self._hard_invalidated = None 

54 

55 def invalidate(self, hard=None): 

56 if hard: 

57 self._soft_invalidated = None 

58 self._hard_invalidated = time.time() 

59 else: 

60 self._soft_invalidated = time.time() 

61 self._hard_invalidated = None 

62 

63 def is_invalidated(self, timestamp): 

64 return ((self._soft_invalidated and 

65 timestamp < self._soft_invalidated) or 

66 (self._hard_invalidated and 

67 timestamp < self._hard_invalidated)) 

68 

69 def was_hard_invalidated(self): 

70 return bool(self._hard_invalidated) 

71 

72 def is_hard_invalidated(self, timestamp): 

73 return (self._hard_invalidated and 

74 timestamp < self._hard_invalidated) 

75 

76 def was_soft_invalidated(self): 

77 return bool(self._soft_invalidated) 

78 

79 def is_soft_invalidated(self, timestamp): 

80 return (self._soft_invalidated and 

81 timestamp < self._soft_invalidated) 

82 

83 The custom implementation is injected into a :class:`.CacheRegion` 

84 at configure time using the 

85 :paramref:`.CacheRegion.configure.region_invalidator` parameter:: 

86 

87 region = CacheRegion() 

88 

89 region = region.configure(region_invalidator=CustomInvalidationStrategy()) # noqa 

90 

91 Invalidation strategies that wish to have access to the 

92 :class:`.CacheRegion` itself should construct the invalidator given the 

93 region as an argument:: 

94 

95 class MyInvalidator(RegionInvalidationStrategy): 

96 def __init__(self, region): 

97 self.region = region 

98 # ... 

99 

100 # ... 

101 

102 region = CacheRegion() 

103 region = region.configure(region_invalidator=MyInvalidator(region)) 

104 

105 .. versionadded:: 0.6.2 

106 

107 .. seealso:: 

108 

109 :paramref:`.CacheRegion.configure.region_invalidator` 

110 

111 """ 

112 

113 def invalidate(self, hard=True): 

114 """Region invalidation. 

115 

116 :class:`.CacheRegion` propagated call. 

117 The default invalidation system works by setting 

118 a current timestamp (using ``time.time()``) to consider all older 

119 timestamps effectively invalidated. 

120 

121 """ 

122 

123 raise NotImplementedError() 

124 

125 def is_hard_invalidated(self, timestamp): 

126 """Check timestamp to determine if it was hard invalidated. 

127 

128 :return: Boolean. True if ``timestamp`` is older than 

129 the last region invalidation time and region is invalidated 

130 in hard mode. 

131 

132 """ 

133 

134 raise NotImplementedError() 

135 

136 def is_soft_invalidated(self, timestamp): 

137 """Check timestamp to determine if it was soft invalidated. 

138 

139 :return: Boolean. True if ``timestamp`` is older than 

140 the last region invalidation time and region is invalidated 

141 in soft mode. 

142 

143 """ 

144 

145 raise NotImplementedError() 

146 

147 def is_invalidated(self, timestamp): 

148 """Check timestamp to determine if it was invalidated. 

149 

150 :return: Boolean. True if ``timestamp`` is older than 

151 the last region invalidation time. 

152 

153 """ 

154 

155 raise NotImplementedError() 

156 

157 def was_soft_invalidated(self): 

158 """Indicate the region was invalidated in soft mode. 

159 

160 :return: Boolean. True if region was invalidated in soft mode. 

161 

162 """ 

163 

164 raise NotImplementedError() 

165 

166 def was_hard_invalidated(self): 

167 """Indicate the region was invalidated in hard mode. 

168 

169 :return: Boolean. True if region was invalidated in hard mode. 

170 

171 """ 

172 

173 raise NotImplementedError() 

174 

175 

176class DefaultInvalidationStrategy(RegionInvalidationStrategy): 

177 def __init__(self): 

178 self._is_hard_invalidated = None 

179 self._invalidated = None 

180 

181 def invalidate(self, hard=True): 

182 self._is_hard_invalidated = bool(hard) 

183 self._invalidated = time.time() 

184 

185 def is_invalidated(self, timestamp): 

186 return self._invalidated is not None and timestamp < self._invalidated 

187 

188 def was_hard_invalidated(self): 

189 return self._is_hard_invalidated is True 

190 

191 def is_hard_invalidated(self, timestamp): 

192 return self.was_hard_invalidated() and self.is_invalidated(timestamp) 

193 

194 def was_soft_invalidated(self): 

195 return self._is_hard_invalidated is False 

196 

197 def is_soft_invalidated(self, timestamp): 

198 return self.was_soft_invalidated() and self.is_invalidated(timestamp) 

199 

200 

201class CacheRegion(object): 

202 r"""A front end to a particular cache backend. 

203 

204 :param name: Optional, a string name for the region. 

205 This isn't used internally 

206 but can be accessed via the ``.name`` parameter, helpful 

207 for configuring a region from a config file. 

208 :param function_key_generator: Optional. A 

209 function that will produce a "cache key" given 

210 a data creation function and arguments, when using 

211 the :meth:`.CacheRegion.cache_on_arguments` method. 

212 The structure of this function 

213 should be two levels: given the data creation function, 

214 return a new function that generates the key based on 

215 the given arguments. Such as:: 

216 

217 def my_key_generator(namespace, fn, **kw): 

218 fname = fn.__name__ 

219 def generate_key(*arg): 

220 return namespace + "_" + fname + "_".join(str(s) for s in arg) 

221 return generate_key 

222 

223 

224 region = make_region( 

225 function_key_generator = my_key_generator 

226 ).configure( 

227 "dogpile.cache.dbm", 

228 expiration_time=300, 

229 arguments={ 

230 "filename":"file.dbm" 

231 } 

232 ) 

233 

234 The ``namespace`` is that passed to 

235 :meth:`.CacheRegion.cache_on_arguments`. It's not consulted 

236 outside this function, so in fact can be of any form. 

237 For example, it can be passed as a tuple, used to specify 

238 arguments to pluck from \**kw:: 

239 

240 def my_key_generator(namespace, fn): 

241 def generate_key(*arg, **kw): 

242 return ":".join( 

243 [kw[k] for k in namespace] + 

244 [str(x) for x in arg] 

245 ) 

246 return generate_key 

247 

248 

249 Where the decorator might be used as:: 

250 

251 @my_region.cache_on_arguments(namespace=('x', 'y')) 

252 def my_function(a, b, **kw): 

253 return my_data() 

254 

255 .. seealso:: 

256 

257 :func:`.function_key_generator` - default key generator 

258 

259 :func:`.kwarg_function_key_generator` - optional gen that also 

260 uses keyword arguments 

261 

262 :param function_multi_key_generator: Optional. 

263 Similar to ``function_key_generator`` parameter, but it's used in 

264 :meth:`.CacheRegion.cache_multi_on_arguments`. Generated function 

265 should return list of keys. For example:: 

266 

267 def my_multi_key_generator(namespace, fn, **kw): 

268 namespace = fn.__name__ + (namespace or '') 

269 

270 def generate_keys(*args): 

271 return [namespace + ':' + str(a) for a in args] 

272 

273 return generate_keys 

274 

275 :param key_mangler: Function which will be used on all incoming 

276 keys before passing to the backend. Defaults to ``None``, 

277 in which case the key mangling function recommended by 

278 the cache backend will be used. A typical mangler 

279 is the SHA1 mangler found at :func:`.sha1_mangle_key` 

280 which coerces keys into a SHA1 

281 hash, so that the string length is fixed. To 

282 disable all key mangling, set to ``False``. Another typical 

283 mangler is the built-in Python function ``str``, which can be used 

284 to convert non-string or Unicode keys to bytestrings, which is 

285 needed when using a backend such as bsddb or dbm under Python 2.x 

286 in conjunction with Unicode keys. 

287 :param async_creation_runner: A callable that, when specified, 

288 will be passed to and called by dogpile.lock when 

289 there is a stale value present in the cache. It will be passed the 

290 mutex and is responsible releasing that mutex when finished. 

291 This can be used to defer the computation of expensive creator 

292 functions to later points in the future by way of, for example, a 

293 background thread, a long-running queue, or a task manager system 

294 like Celery. 

295 

296 For a specific example using async_creation_runner, new values can 

297 be created in a background thread like so:: 

298 

299 import threading 

300 

301 def async_creation_runner(cache, somekey, creator, mutex): 

302 ''' Used by dogpile.core:Lock when appropriate ''' 

303 def runner(): 

304 try: 

305 value = creator() 

306 cache.set(somekey, value) 

307 finally: 

308 mutex.release() 

309 

310 thread = threading.Thread(target=runner) 

311 thread.start() 

312 

313 

314 region = make_region( 

315 async_creation_runner=async_creation_runner, 

316 ).configure( 

317 'dogpile.cache.memcached', 

318 expiration_time=5, 

319 arguments={ 

320 'url': '127.0.0.1:11211', 

321 'distributed_lock': True, 

322 } 

323 ) 

324 

325 Remember that the first request for a key with no associated 

326 value will always block; async_creator will not be invoked. 

327 However, subsequent requests for cached-but-expired values will 

328 still return promptly. They will be refreshed by whatever 

329 asynchronous means the provided async_creation_runner callable 

330 implements. 

331 

332 By default the async_creation_runner is disabled and is set 

333 to ``None``. 

334 

335 .. versionadded:: 0.4.2 added the async_creation_runner 

336 feature. 

337 

338 """ 

339 

340 def __init__( 

341 self, 

342 name=None, 

343 function_key_generator=function_key_generator, 

344 function_multi_key_generator=function_multi_key_generator, 

345 key_mangler=None, 

346 async_creation_runner=None, 

347 ): 

348 """Construct a new :class:`.CacheRegion`.""" 

349 self.name = name 

350 self.function_key_generator = function_key_generator 

351 self.function_multi_key_generator = function_multi_key_generator 

352 self.key_mangler = self._user_defined_key_mangler = key_mangler 

353 self.async_creation_runner = async_creation_runner 

354 self.region_invalidator = DefaultInvalidationStrategy() 

355 

356 def configure( 

357 self, 

358 backend, 

359 expiration_time=None, 

360 arguments=None, 

361 _config_argument_dict=None, 

362 _config_prefix=None, 

363 wrap=None, 

364 replace_existing_backend=False, 

365 region_invalidator=None, 

366 ): 

367 """Configure a :class:`.CacheRegion`. 

368 

369 The :class:`.CacheRegion` itself 

370 is returned. 

371 

372 :param backend: Required. This is the name of the 

373 :class:`.CacheBackend` to use, and is resolved by loading 

374 the class from the ``dogpile.cache`` entrypoint. 

375 

376 :param expiration_time: Optional. The expiration time passed 

377 to the dogpile system. May be passed as an integer number 

378 of seconds, or as a ``datetime.timedelta`` value. 

379 

380 .. versionadded 0.5.0 

381 ``expiration_time`` may be optionally passed as a 

382 ``datetime.timedelta`` value. 

383 

384 The :meth:`.CacheRegion.get_or_create` 

385 method as well as the :meth:`.CacheRegion.cache_on_arguments` 

386 decorator (though note: **not** the :meth:`.CacheRegion.get` 

387 method) will call upon the value creation function after this 

388 time period has passed since the last generation. 

389 

390 :param arguments: Optional. The structure here is passed 

391 directly to the constructor of the :class:`.CacheBackend` 

392 in use, though is typically a dictionary. 

393 

394 :param wrap: Optional. A list of :class:`.ProxyBackend` 

395 classes and/or instances, each of which will be applied 

396 in a chain to ultimately wrap the original backend, 

397 so that custom functionality augmentation can be applied. 

398 

399 .. versionadded:: 0.5.0 

400 

401 .. seealso:: 

402 

403 :ref:`changing_backend_behavior` 

404 

405 :param replace_existing_backend: if True, the existing cache backend 

406 will be replaced. Without this flag, an exception is raised if 

407 a backend is already configured. 

408 

409 .. versionadded:: 0.5.7 

410 

411 :param region_invalidator: Optional. Override default invalidation 

412 strategy with custom implementation of 

413 :class:`.RegionInvalidationStrategy`. 

414 

415 .. versionadded:: 0.6.2 

416 

417 """ 

418 

419 if "backend" in self.__dict__ and not replace_existing_backend: 

420 raise exception.RegionAlreadyConfigured( 

421 "This region is already " 

422 "configured with backend: %s. " 

423 "Specify replace_existing_backend=True to replace." 

424 % self.backend 

425 ) 

426 

427 try: 

428 backend_cls = _backend_loader.load(backend) 

429 except PluginLoader.NotFound: 

430 raise exception.PluginNotFound( 

431 "Couldn't find cache plugin to load: %s" % backend 

432 ) 

433 

434 if _config_argument_dict: 

435 self.backend = backend_cls.from_config_dict( 

436 _config_argument_dict, _config_prefix 

437 ) 

438 else: 

439 self.backend = backend_cls(arguments or {}) 

440 

441 if not expiration_time or isinstance(expiration_time, Number): 

442 self.expiration_time = expiration_time 

443 elif isinstance(expiration_time, datetime.timedelta): 

444 self.expiration_time = int( 

445 compat.timedelta_total_seconds(expiration_time) 

446 ) 

447 else: 

448 raise exception.ValidationError( 

449 "expiration_time is not a number or timedelta." 

450 ) 

451 

452 if not self._user_defined_key_mangler: 

453 self.key_mangler = self.backend.key_mangler 

454 

455 self._lock_registry = NameRegistry(self._create_mutex) 

456 

457 if getattr(wrap, "__iter__", False): 

458 for wrapper in reversed(wrap): 

459 self.wrap(wrapper) 

460 

461 if region_invalidator: 

462 self.region_invalidator = region_invalidator 

463 

464 return self 

465 

466 def wrap(self, proxy): 

467 """ Takes a ProxyBackend instance or class and wraps the 

468 attached backend. """ 

469 

470 # if we were passed a type rather than an instance then 

471 # initialize it. 

472 if type(proxy) == type: 

473 proxy = proxy() 

474 

475 if not issubclass(type(proxy), ProxyBackend): 

476 raise TypeError( 

477 "Type %s is not a valid ProxyBackend" % type(proxy) 

478 ) 

479 

480 self.backend = proxy.wrap(self.backend) 

481 

482 def _mutex(self, key): 

483 return self._lock_registry.get(key) 

484 

485 class _LockWrapper(object): 

486 """weakref-capable wrapper for threading.Lock""" 

487 

488 def __init__(self): 

489 self.lock = threading.Lock() 

490 

491 def acquire(self, wait=True): 

492 return self.lock.acquire(wait) 

493 

494 def release(self): 

495 self.lock.release() 

496 

497 def _create_mutex(self, key): 

498 mutex = self.backend.get_mutex(key) 

499 if mutex is not None: 

500 return mutex 

501 else: 

502 return self._LockWrapper() 

503 

504 # cached value 

505 _actual_backend = None 

506 

507 @property 

508 def actual_backend(self): 

509 """Return the ultimate backend underneath any proxies. 

510 

511 The backend might be the result of one or more ``proxy.wrap`` 

512 applications. If so, derive the actual underlying backend. 

513 

514 .. versionadded:: 0.6.6 

515 

516 """ 

517 if self._actual_backend is None: 

518 _backend = self.backend 

519 while hasattr(_backend, "proxied"): 

520 _backend = _backend.proxied 

521 self._actual_backend = _backend 

522 return self._actual_backend 

523 

524 def invalidate(self, hard=True): 

525 """Invalidate this :class:`.CacheRegion`. 

526 

527 The default invalidation system works by setting 

528 a current timestamp (using ``time.time()``) 

529 representing the "minimum creation time" for 

530 a value. Any retrieved value whose creation 

531 time is prior to this timestamp 

532 is considered to be stale. It does not 

533 affect the data in the cache in any way, and is 

534 **local to this instance of :class:`.CacheRegion`.** 

535 

536 .. warning:: 

537 

538 The :meth:`.CacheRegion.invalidate` method's default mode of 

539 operation is to set a timestamp **local to this CacheRegion 

540 in this Python process only**. It does not impact other Python 

541 processes or regions as the timestamp is **only stored locally in 

542 memory**. To implement invalidation where the 

543 timestamp is stored in the cache or similar so that all Python 

544 processes can be affected by an invalidation timestamp, implement a 

545 custom :class:`.RegionInvalidationStrategy`. 

546 

547 Once set, the invalidation time is honored by 

548 the :meth:`.CacheRegion.get_or_create`, 

549 :meth:`.CacheRegion.get_or_create_multi` and 

550 :meth:`.CacheRegion.get` methods. 

551 

552 The method supports both "hard" and "soft" invalidation 

553 options. With "hard" invalidation, 

554 :meth:`.CacheRegion.get_or_create` will force an immediate 

555 regeneration of the value which all getters will wait for. 

556 With "soft" invalidation, subsequent getters will return the 

557 "old" value until the new one is available. 

558 

559 Usage of "soft" invalidation requires that the region or the method 

560 is given a non-None expiration time. 

561 

562 .. versionadded:: 0.3.0 

563 

564 :param hard: if True, cache values will all require immediate 

565 regeneration; dogpile logic won't be used. If False, the 

566 creation time of existing values will be pushed back before 

567 the expiration time so that a return+regen will be invoked. 

568 

569 .. versionadded:: 0.5.1 

570 

571 """ 

572 self.region_invalidator.invalidate(hard) 

573 

574 def configure_from_config(self, config_dict, prefix): 

575 """Configure from a configuration dictionary 

576 and a prefix. 

577 

578 Example:: 

579 

580 local_region = make_region() 

581 memcached_region = make_region() 

582 

583 # regions are ready to use for function 

584 # decorators, but not yet for actual caching 

585 

586 # later, when config is available 

587 myconfig = { 

588 "cache.local.backend":"dogpile.cache.dbm", 

589 "cache.local.arguments.filename":"/path/to/dbmfile.dbm", 

590 "cache.memcached.backend":"dogpile.cache.pylibmc", 

591 "cache.memcached.arguments.url":"127.0.0.1, 10.0.0.1", 

592 } 

593 local_region.configure_from_config(myconfig, "cache.local.") 

594 memcached_region.configure_from_config(myconfig, 

595 "cache.memcached.") 

596 

597 """ 

598 config_dict = coerce_string_conf(config_dict) 

599 return self.configure( 

600 config_dict["%sbackend" % prefix], 

601 expiration_time=config_dict.get( 

602 "%sexpiration_time" % prefix, None 

603 ), 

604 _config_argument_dict=config_dict, 

605 _config_prefix="%sarguments." % prefix, 

606 wrap=config_dict.get("%swrap" % prefix, None), 

607 replace_existing_backend=config_dict.get( 

608 "%sreplace_existing_backend" % prefix, False 

609 ), 

610 ) 

611 

612 @memoized_property 

613 def backend(self): 

614 raise exception.RegionNotConfigured( 

615 "No backend is configured on this region." 

616 ) 

617 

618 @property 

619 def is_configured(self): 

620 """Return True if the backend has been configured via the 

621 :meth:`.CacheRegion.configure` method already. 

622 

623 .. versionadded:: 0.5.1 

624 

625 """ 

626 return "backend" in self.__dict__ 

627 

628 def get(self, key, expiration_time=None, ignore_expiration=False): 

629 """Return a value from the cache, based on the given key. 

630 

631 If the value is not present, the method returns the token 

632 ``NO_VALUE``. ``NO_VALUE`` evaluates to False, but is separate from 

633 ``None`` to distinguish between a cached value of ``None``. 

634 

635 By default, the configured expiration time of the 

636 :class:`.CacheRegion`, or alternatively the expiration 

637 time supplied by the ``expiration_time`` argument, 

638 is tested against the creation time of the retrieved 

639 value versus the current time (as reported by ``time.time()``). 

640 If stale, the cached value is ignored and the ``NO_VALUE`` 

641 token is returned. Passing the flag ``ignore_expiration=True`` 

642 bypasses the expiration time check. 

643 

644 .. versionchanged:: 0.3.0 

645 :meth:`.CacheRegion.get` now checks the value's creation time 

646 against the expiration time, rather than returning 

647 the value unconditionally. 

648 

649 The method also interprets the cached value in terms 

650 of the current "invalidation" time as set by 

651 the :meth:`.invalidate` method. If a value is present, 

652 but its creation time is older than the current 

653 invalidation time, the ``NO_VALUE`` token is returned. 

654 Passing the flag ``ignore_expiration=True`` bypasses 

655 the invalidation time check. 

656 

657 .. versionadded:: 0.3.0 

658 Support for the :meth:`.CacheRegion.invalidate` 

659 method. 

660 

661 :param key: Key to be retrieved. While it's typical for a key to be a 

662 string, it is ultimately passed directly down to the cache backend, 

663 before being optionally processed by the key_mangler function, so can 

664 be of any type recognized by the backend or by the key_mangler 

665 function, if present. 

666 

667 :param expiration_time: Optional expiration time value 

668 which will supersede that configured on the :class:`.CacheRegion` 

669 itself. 

670 

671 .. note:: The :paramref:`.CacheRegion.get.expiration_time` 

672 argument is **not persisted in the cache** and is relevant 

673 only to **this specific cache retrieval operation**, relative to 

674 the creation time stored with the existing cached value. 

675 Subsequent calls to :meth:`.CacheRegion.get` are **not** affected 

676 by this value. 

677 

678 .. versionadded:: 0.3.0 

679 

680 :param ignore_expiration: if ``True``, the value is returned 

681 from the cache if present, regardless of configured 

682 expiration times or whether or not :meth:`.invalidate` 

683 was called. 

684 

685 .. versionadded:: 0.3.0 

686 

687 .. seealso:: 

688 

689 :meth:`.CacheRegion.get_multi` 

690 

691 :meth:`.CacheRegion.get_or_create` 

692 

693 :meth:`.CacheRegion.set` 

694 

695 :meth:`.CacheRegion.delete` 

696 

697 

698 """ 

699 

700 if self.key_mangler: 

701 key = self.key_mangler(key) 

702 value = self.backend.get(key) 

703 value = self._unexpired_value_fn(expiration_time, ignore_expiration)( 

704 value 

705 ) 

706 

707 return value.payload 

708 

709 def _unexpired_value_fn(self, expiration_time, ignore_expiration): 

710 if ignore_expiration: 

711 return lambda value: value 

712 else: 

713 if expiration_time is None: 

714 expiration_time = self.expiration_time 

715 

716 current_time = time.time() 

717 

718 def value_fn(value): 

719 if value is NO_VALUE: 

720 return value 

721 elif ( 

722 expiration_time is not None 

723 and current_time - value.metadata["ct"] > expiration_time 

724 ): 

725 return NO_VALUE 

726 elif self.region_invalidator.is_invalidated( 

727 value.metadata["ct"] 

728 ): 

729 return NO_VALUE 

730 else: 

731 return value 

732 

733 return value_fn 

734 

735 def get_multi(self, keys, expiration_time=None, ignore_expiration=False): 

736 """Return multiple values from the cache, based on the given keys. 

737 

738 Returns values as a list matching the keys given. 

739 

740 E.g.:: 

741 

742 values = region.get_multi(["one", "two", "three"]) 

743 

744 To convert values to a dictionary, use ``zip()``:: 

745 

746 keys = ["one", "two", "three"] 

747 values = region.get_multi(keys) 

748 dictionary = dict(zip(keys, values)) 

749 

750 Keys which aren't present in the list are returned as 

751 the ``NO_VALUE`` token. ``NO_VALUE`` evaluates to False, 

752 but is separate from 

753 ``None`` to distinguish between a cached value of ``None``. 

754 

755 By default, the configured expiration time of the 

756 :class:`.CacheRegion`, or alternatively the expiration 

757 time supplied by the ``expiration_time`` argument, 

758 is tested against the creation time of the retrieved 

759 value versus the current time (as reported by ``time.time()``). 

760 If stale, the cached value is ignored and the ``NO_VALUE`` 

761 token is returned. Passing the flag ``ignore_expiration=True`` 

762 bypasses the expiration time check. 

763 

764 .. versionadded:: 0.5.0 

765 

766 """ 

767 if not keys: 

768 return [] 

769 

770 if self.key_mangler: 

771 keys = list(map(lambda key: self.key_mangler(key), keys)) 

772 

773 backend_values = self.backend.get_multi(keys) 

774 

775 _unexpired_value_fn = self._unexpired_value_fn( 

776 expiration_time, ignore_expiration 

777 ) 

778 return [ 

779 value.payload if value is not NO_VALUE else value 

780 for value in ( 

781 _unexpired_value_fn(value) for value in backend_values 

782 ) 

783 ] 

784 

785 @contextlib.contextmanager 

786 def _log_time(self, keys): 

787 start_time = time.time() 

788 yield 

789 seconds = time.time() - start_time 

790 log.debug( 

791 "Cache value generated in %(seconds).3f seconds for key(s): " 

792 "%(keys)r", 

793 {"seconds": seconds, "keys": repr_obj(keys)}, 

794 ) 

795 

796 def _is_cache_miss(self, value, orig_key): 

797 if value is NO_VALUE: 

798 log.debug("No value present for key: %r", orig_key) 

799 elif value.metadata["v"] != value_version: 

800 log.debug("Dogpile version update for key: %r", orig_key) 

801 elif self.region_invalidator.is_hard_invalidated(value.metadata["ct"]): 

802 log.debug("Hard invalidation detected for key: %r", orig_key) 

803 else: 

804 return False 

805 

806 return True 

807 

808 def get_or_create( 

809 self, 

810 key, 

811 creator, 

812 expiration_time=None, 

813 should_cache_fn=None, 

814 creator_args=None, 

815 ): 

816 """Return a cached value based on the given key. 

817 

818 If the value does not exist or is considered to be expired 

819 based on its creation time, the given 

820 creation function may or may not be used to recreate the value 

821 and persist the newly generated value in the cache. 

822 

823 Whether or not the function is used depends on if the 

824 *dogpile lock* can be acquired or not. If it can't, it means 

825 a different thread or process is already running a creation 

826 function for this key against the cache. When the dogpile 

827 lock cannot be acquired, the method will block if no 

828 previous value is available, until the lock is released and 

829 a new value available. If a previous value 

830 is available, that value is returned immediately without blocking. 

831 

832 If the :meth:`.invalidate` method has been called, and 

833 the retrieved value's timestamp is older than the invalidation 

834 timestamp, the value is unconditionally prevented from 

835 being returned. The method will attempt to acquire the dogpile 

836 lock to generate a new value, or will wait 

837 until the lock is released to return the new value. 

838 

839 .. versionchanged:: 0.3.0 

840 The value is unconditionally regenerated if the creation 

841 time is older than the last call to :meth:`.invalidate`. 

842 

843 :param key: Key to be retrieved. While it's typical for a key to be a 

844 string, it is ultimately passed directly down to the cache backend, 

845 before being optionally processed by the key_mangler function, so can 

846 be of any type recognized by the backend or by the key_mangler 

847 function, if present. 

848 

849 :param creator: function which creates a new value. 

850 

851 :param creator_args: optional tuple of (args, kwargs) that will be 

852 passed to the creator function if present. 

853 

854 .. versionadded:: 0.7.0 

855 

856 :param expiration_time: optional expiration time which will override 

857 the expiration time already configured on this :class:`.CacheRegion` 

858 if not None. To set no expiration, use the value -1. 

859 

860 .. note:: The :paramref:`.CacheRegion.get_or_create.expiration_time` 

861 argument is **not persisted in the cache** and is relevant 

862 only to **this specific cache retrieval operation**, relative to 

863 the creation time stored with the existing cached value. 

864 Subsequent calls to :meth:`.CacheRegion.get_or_create` are **not** 

865 affected by this value. 

866 

867 :param should_cache_fn: optional callable function which will receive 

868 the value returned by the "creator", and will then return True or 

869 False, indicating if the value should actually be cached or not. If 

870 it returns False, the value is still returned, but isn't cached. 

871 E.g.:: 

872 

873 def dont_cache_none(value): 

874 return value is not None 

875 

876 value = region.get_or_create("some key", 

877 create_value, 

878 should_cache_fn=dont_cache_none) 

879 

880 Above, the function returns the value of create_value() if 

881 the cache is invalid, however if the return value is None, 

882 it won't be cached. 

883 

884 .. versionadded:: 0.4.3 

885 

886 .. seealso:: 

887 

888 :meth:`.CacheRegion.get` 

889 

890 :meth:`.CacheRegion.cache_on_arguments` - applies 

891 :meth:`.get_or_create` to any function using a decorator. 

892 

893 :meth:`.CacheRegion.get_or_create_multi` - multiple key/value 

894 version 

895 

896 """ 

897 orig_key = key 

898 if self.key_mangler: 

899 key = self.key_mangler(key) 

900 

901 def get_value(): 

902 value = self.backend.get(key) 

903 if self._is_cache_miss(value, orig_key): 

904 raise NeedRegenerationException() 

905 

906 ct = value.metadata["ct"] 

907 if self.region_invalidator.is_soft_invalidated(ct): 

908 ct = time.time() - expiration_time - 0.0001 

909 

910 return value.payload, ct 

911 

912 def gen_value(): 

913 with self._log_time(orig_key): 

914 if creator_args: 

915 created_value = creator( 

916 *creator_args[0], **creator_args[1] 

917 ) 

918 else: 

919 created_value = creator() 

920 value = self._value(created_value) 

921 

922 if not should_cache_fn or should_cache_fn(created_value): 

923 self.backend.set(key, value) 

924 

925 return value.payload, value.metadata["ct"] 

926 

927 if expiration_time is None: 

928 expiration_time = self.expiration_time 

929 

930 if ( 

931 expiration_time is None 

932 and self.region_invalidator.was_soft_invalidated() 

933 ): 

934 raise exception.DogpileCacheException( 

935 "Non-None expiration time required " "for soft invalidation" 

936 ) 

937 

938 if expiration_time == -1: 

939 expiration_time = None 

940 

941 if self.async_creation_runner: 

942 

943 def async_creator(mutex): 

944 if creator_args: 

945 

946 @wraps(creator) 

947 def go(): 

948 return creator(*creator_args[0], **creator_args[1]) 

949 

950 else: 

951 go = creator 

952 return self.async_creation_runner(self, orig_key, go, mutex) 

953 

954 else: 

955 async_creator = None 

956 

957 with Lock( 

958 self._mutex(key), 

959 gen_value, 

960 get_value, 

961 expiration_time, 

962 async_creator, 

963 ) as value: 

964 return value 

965 

966 def get_or_create_multi( 

967 self, keys, creator, expiration_time=None, should_cache_fn=None 

968 ): 

969 """Return a sequence of cached values based on a sequence of keys. 

970 

971 The behavior for generation of values based on keys corresponds 

972 to that of :meth:`.Region.get_or_create`, with the exception that 

973 the ``creator()`` function may be asked to generate any subset of 

974 the given keys. The list of keys to be generated is passed to 

975 ``creator()``, and ``creator()`` should return the generated values 

976 as a sequence corresponding to the order of the keys. 

977 

978 The method uses the same approach as :meth:`.Region.get_multi` 

979 and :meth:`.Region.set_multi` to get and set values from the 

980 backend. 

981 

982 If you are using a :class:`.CacheBackend` or :class:`.ProxyBackend` 

983 that modifies values, take note this function invokes 

984 ``.set_multi()`` for newly generated values using the same values it 

985 returns to the calling function. A correct implementation of 

986 ``.set_multi()`` will not modify values in-place on the submitted 

987 ``mapping`` dict. 

988 

989 :param keys: Sequence of keys to be retrieved. 

990 

991 :param creator: function which accepts a sequence of keys and 

992 returns a sequence of new values. 

993 

994 :param expiration_time: optional expiration time which will override 

995 the expiration time already configured on this :class:`.CacheRegion` 

996 if not None. To set no expiration, use the value -1. 

997 

998 :param should_cache_fn: optional callable function which will receive 

999 each value returned by the "creator", and will then return True or 

1000 False, indicating if the value should actually be cached or not. If 

1001 it returns False, the value is still returned, but isn't cached. 

1002 

1003 .. versionadded:: 0.5.0 

1004 

1005 .. seealso:: 

1006 

1007 

1008 :meth:`.CacheRegion.cache_multi_on_arguments` 

1009 

1010 :meth:`.CacheRegion.get_or_create` 

1011 

1012 """ 

1013 

1014 def get_value(key): 

1015 value = values.get(key, NO_VALUE) 

1016 

1017 if self._is_cache_miss(value, orig_key): 

1018 # dogpile.core understands a 0 here as 

1019 # "the value is not available", e.g. 

1020 # _has_value() will return False. 

1021 return value.payload, 0 

1022 else: 

1023 ct = value.metadata["ct"] 

1024 if self.region_invalidator.is_soft_invalidated(ct): 

1025 ct = time.time() - expiration_time - 0.0001 

1026 

1027 return value.payload, ct 

1028 

1029 def gen_value(): 

1030 raise NotImplementedError() 

1031 

1032 def async_creator(key, mutex): 

1033 mutexes[key] = mutex 

1034 

1035 if expiration_time is None: 

1036 expiration_time = self.expiration_time 

1037 

1038 if ( 

1039 expiration_time is None 

1040 and self.region_invalidator.was_soft_invalidated() 

1041 ): 

1042 raise exception.DogpileCacheException( 

1043 "Non-None expiration time required " "for soft invalidation" 

1044 ) 

1045 

1046 if expiration_time == -1: 

1047 expiration_time = None 

1048 

1049 mutexes = {} 

1050 

1051 sorted_unique_keys = sorted(set(keys)) 

1052 

1053 if self.key_mangler: 

1054 mangled_keys = [self.key_mangler(k) for k in sorted_unique_keys] 

1055 else: 

1056 mangled_keys = sorted_unique_keys 

1057 

1058 orig_to_mangled = dict(zip(sorted_unique_keys, mangled_keys)) 

1059 

1060 values = dict(zip(mangled_keys, self.backend.get_multi(mangled_keys))) 

1061 

1062 for orig_key, mangled_key in orig_to_mangled.items(): 

1063 with Lock( 

1064 self._mutex(mangled_key), 

1065 gen_value, 

1066 lambda: get_value(mangled_key), 

1067 expiration_time, 

1068 async_creator=lambda mutex: async_creator(orig_key, mutex), 

1069 ): 

1070 pass 

1071 try: 

1072 if mutexes: 

1073 # sort the keys, the idea is to prevent deadlocks. 

1074 # though haven't been able to simulate one anyway. 

1075 keys_to_get = sorted(mutexes) 

1076 

1077 with self._log_time(keys_to_get): 

1078 new_values = creator(*keys_to_get) 

1079 

1080 values_w_created = dict( 

1081 (orig_to_mangled[k], self._value(v)) 

1082 for k, v in zip(keys_to_get, new_values) 

1083 ) 

1084 

1085 if not should_cache_fn: 

1086 self.backend.set_multi(values_w_created) 

1087 else: 

1088 values_to_cache = dict( 

1089 (k, v) 

1090 for k, v in values_w_created.items() 

1091 if should_cache_fn(v[0]) 

1092 ) 

1093 

1094 if values_to_cache: 

1095 self.backend.set_multi(values_to_cache) 

1096 

1097 values.update(values_w_created) 

1098 return [values[orig_to_mangled[k]].payload for k in keys] 

1099 finally: 

1100 for mutex in mutexes.values(): 

1101 mutex.release() 

1102 

1103 def _value(self, value): 

1104 """Return a :class:`.CachedValue` given a value.""" 

1105 return CachedValue(value, {"ct": time.time(), "v": value_version}) 

1106 

1107 def set(self, key, value): 

1108 """Place a new value in the cache under the given key.""" 

1109 

1110 if self.key_mangler: 

1111 key = self.key_mangler(key) 

1112 self.backend.set(key, self._value(value)) 

1113 

1114 def set_multi(self, mapping): 

1115 """Place new values in the cache under the given keys. 

1116 

1117 .. versionadded:: 0.5.0 

1118 

1119 """ 

1120 if not mapping: 

1121 return 

1122 

1123 if self.key_mangler: 

1124 mapping = dict( 

1125 (self.key_mangler(k), self._value(v)) 

1126 for k, v in mapping.items() 

1127 ) 

1128 else: 

1129 mapping = dict((k, self._value(v)) for k, v in mapping.items()) 

1130 self.backend.set_multi(mapping) 

1131 

1132 def delete(self, key): 

1133 """Remove a value from the cache. 

1134 

1135 This operation is idempotent (can be called multiple times, or on a 

1136 non-existent key, safely) 

1137 """ 

1138 

1139 if self.key_mangler: 

1140 key = self.key_mangler(key) 

1141 

1142 self.backend.delete(key) 

1143 

1144 def delete_multi(self, keys): 

1145 """Remove multiple values from the cache. 

1146 

1147 This operation is idempotent (can be called multiple times, or on a 

1148 non-existent key, safely) 

1149 

1150 .. versionadded:: 0.5.0 

1151 

1152 """ 

1153 

1154 if self.key_mangler: 

1155 keys = list(map(lambda key: self.key_mangler(key), keys)) 

1156 

1157 self.backend.delete_multi(keys) 

1158 

1159 def cache_on_arguments( 

1160 self, 

1161 namespace=None, 

1162 expiration_time=None, 

1163 should_cache_fn=None, 

1164 to_str=compat.string_type, 

1165 function_key_generator=None, 

1166 ): 

1167 """A function decorator that will cache the return 

1168 value of the function using a key derived from the 

1169 function itself and its arguments. 

1170 

1171 The decorator internally makes use of the 

1172 :meth:`.CacheRegion.get_or_create` method to access the 

1173 cache and conditionally call the function. See that 

1174 method for additional behavioral details. 

1175 

1176 E.g.:: 

1177 

1178 @someregion.cache_on_arguments() 

1179 def generate_something(x, y): 

1180 return somedatabase.query(x, y) 

1181 

1182 The decorated function can then be called normally, where 

1183 data will be pulled from the cache region unless a new 

1184 value is needed:: 

1185 

1186 result = generate_something(5, 6) 

1187 

1188 The function is also given an attribute ``invalidate()``, which 

1189 provides for invalidation of the value. Pass to ``invalidate()`` 

1190 the same arguments you'd pass to the function itself to represent 

1191 a particular value:: 

1192 

1193 generate_something.invalidate(5, 6) 

1194 

1195 Another attribute ``set()`` is added to provide extra caching 

1196 possibilities relative to the function. This is a convenience 

1197 method for :meth:`.CacheRegion.set` which will store a given 

1198 value directly without calling the decorated function. 

1199 The value to be cached is passed as the first argument, and the 

1200 arguments which would normally be passed to the function 

1201 should follow:: 

1202 

1203 generate_something.set(3, 5, 6) 

1204 

1205 The above example is equivalent to calling 

1206 ``generate_something(5, 6)``, if the function were to produce 

1207 the value ``3`` as the value to be cached. 

1208 

1209 .. versionadded:: 0.4.1 Added ``set()`` method to decorated function. 

1210 

1211 Similar to ``set()`` is ``refresh()``. This attribute will 

1212 invoke the decorated function and populate a new value into 

1213 the cache with the new value, as well as returning that value:: 

1214 

1215 newvalue = generate_something.refresh(5, 6) 

1216 

1217 .. versionadded:: 0.5.0 Added ``refresh()`` method to decorated 

1218 function. 

1219 

1220 ``original()`` on other hand will invoke the decorated function 

1221 without any caching:: 

1222 

1223 newvalue = generate_something.original(5, 6) 

1224 

1225 .. versionadded:: 0.6.0 Added ``original()`` method to decorated 

1226 function. 

1227 

1228 Lastly, the ``get()`` method returns either the value cached 

1229 for the given key, or the token ``NO_VALUE`` if no such key 

1230 exists:: 

1231 

1232 value = generate_something.get(5, 6) 

1233 

1234 .. versionadded:: 0.5.3 Added ``get()`` method to decorated 

1235 function. 

1236 

1237 The default key generation will use the name 

1238 of the function, the module name for the function, 

1239 the arguments passed, as well as an optional "namespace" 

1240 parameter in order to generate a cache key. 

1241 

1242 Given a function ``one`` inside the module 

1243 ``myapp.tools``:: 

1244 

1245 @region.cache_on_arguments(namespace="foo") 

1246 def one(a, b): 

1247 return a + b 

1248 

1249 Above, calling ``one(3, 4)`` will produce a 

1250 cache key as follows:: 

1251 

1252 myapp.tools:one|foo|3 4 

1253 

1254 The key generator will ignore an initial argument 

1255 of ``self`` or ``cls``, making the decorator suitable 

1256 (with caveats) for use with instance or class methods. 

1257 Given the example:: 

1258 

1259 class MyClass(object): 

1260 @region.cache_on_arguments(namespace="foo") 

1261 def one(self, a, b): 

1262 return a + b 

1263 

1264 The cache key above for ``MyClass().one(3, 4)`` will 

1265 again produce the same cache key of ``myapp.tools:one|foo|3 4`` - 

1266 the name ``self`` is skipped. 

1267 

1268 The ``namespace`` parameter is optional, and is used 

1269 normally to disambiguate two functions of the same 

1270 name within the same module, as can occur when decorating 

1271 instance or class methods as below:: 

1272 

1273 class MyClass(object): 

1274 @region.cache_on_arguments(namespace='MC') 

1275 def somemethod(self, x, y): 

1276 "" 

1277 

1278 class MyOtherClass(object): 

1279 @region.cache_on_arguments(namespace='MOC') 

1280 def somemethod(self, x, y): 

1281 "" 

1282 

1283 Above, the ``namespace`` parameter disambiguates 

1284 between ``somemethod`` on ``MyClass`` and ``MyOtherClass``. 

1285 Python class declaration mechanics otherwise prevent 

1286 the decorator from having awareness of the ``MyClass`` 

1287 and ``MyOtherClass`` names, as the function is received 

1288 by the decorator before it becomes an instance method. 

1289 

1290 The function key generation can be entirely replaced 

1291 on a per-region basis using the ``function_key_generator`` 

1292 argument present on :func:`.make_region` and 

1293 :class:`.CacheRegion`. If defaults to 

1294 :func:`.function_key_generator`. 

1295 

1296 :param namespace: optional string argument which will be 

1297 established as part of the cache key. This may be needed 

1298 to disambiguate functions of the same name within the same 

1299 source file, such as those 

1300 associated with classes - note that the decorator itself 

1301 can't see the parent class on a function as the class is 

1302 being declared. 

1303 

1304 :param expiration_time: if not None, will override the normal 

1305 expiration time. 

1306 

1307 May be specified as a callable, taking no arguments, that 

1308 returns a value to be used as the ``expiration_time``. This callable 

1309 will be called whenever the decorated function itself is called, in 

1310 caching or retrieving. Thus, this can be used to 

1311 determine a *dynamic* expiration time for the cached function 

1312 result. Example use cases include "cache the result until the 

1313 end of the day, week or time period" and "cache until a certain date 

1314 or time passes". 

1315 

1316 .. versionchanged:: 0.5.0 

1317 ``expiration_time`` may be passed as a callable to 

1318 :meth:`.CacheRegion.cache_on_arguments`. 

1319 

1320 :param should_cache_fn: passed to :meth:`.CacheRegion.get_or_create`. 

1321 

1322 .. versionadded:: 0.4.3 

1323 

1324 :param to_str: callable, will be called on each function argument 

1325 in order to convert to a string. Defaults to ``str()``. If the 

1326 function accepts non-ascii unicode arguments on Python 2.x, the 

1327 ``unicode()`` builtin can be substituted, but note this will 

1328 produce unicode cache keys which may require key mangling before 

1329 reaching the cache. 

1330 

1331 .. versionadded:: 0.5.0 

1332 

1333 :param function_key_generator: a function that will produce a 

1334 "cache key". This function will supersede the one configured on the 

1335 :class:`.CacheRegion` itself. 

1336 

1337 .. versionadded:: 0.5.5 

1338 

1339 .. seealso:: 

1340 

1341 :meth:`.CacheRegion.cache_multi_on_arguments` 

1342 

1343 :meth:`.CacheRegion.get_or_create` 

1344 

1345 """ 

1346 expiration_time_is_callable = compat.callable(expiration_time) 

1347 

1348 if function_key_generator is None: 

1349 function_key_generator = self.function_key_generator 

1350 

1351 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw): 

1352 key = key_generator(*arg, **kw) 

1353 

1354 timeout = ( 

1355 expiration_time() 

1356 if expiration_time_is_callable 

1357 else expiration_time 

1358 ) 

1359 return self.get_or_create( 

1360 key, user_func, timeout, should_cache_fn, (arg, kw) 

1361 ) 

1362 

1363 def cache_decorator(user_func): 

1364 if to_str is compat.string_type: 

1365 # backwards compatible 

1366 key_generator = function_key_generator(namespace, user_func) 

1367 else: 

1368 key_generator = function_key_generator( 

1369 namespace, user_func, to_str=to_str 

1370 ) 

1371 

1372 def refresh(*arg, **kw): 

1373 """ 

1374 Like invalidate, but regenerates the value instead 

1375 """ 

1376 key = key_generator(*arg, **kw) 

1377 value = user_func(*arg, **kw) 

1378 self.set(key, value) 

1379 return value 

1380 

1381 def invalidate(*arg, **kw): 

1382 key = key_generator(*arg, **kw) 

1383 self.delete(key) 

1384 

1385 def set_(value, *arg, **kw): 

1386 key = key_generator(*arg, **kw) 

1387 self.set(key, value) 

1388 

1389 def get(*arg, **kw): 

1390 key = key_generator(*arg, **kw) 

1391 return self.get(key) 

1392 

1393 user_func.set = set_ 

1394 user_func.invalidate = invalidate 

1395 user_func.get = get 

1396 user_func.refresh = refresh 

1397 user_func.original = user_func 

1398 

1399 # Use `decorate` to preserve the signature of :param:`user_func`. 

1400 

1401 return decorate( 

1402 user_func, partial(get_or_create_for_user_func, key_generator) 

1403 ) 

1404 

1405 return cache_decorator 

1406 

1407 def cache_multi_on_arguments( 

1408 self, 

1409 namespace=None, 

1410 expiration_time=None, 

1411 should_cache_fn=None, 

1412 asdict=False, 

1413 to_str=compat.string_type, 

1414 function_multi_key_generator=None, 

1415 ): 

1416 """A function decorator that will cache multiple return 

1417 values from the function using a sequence of keys derived from the 

1418 function itself and the arguments passed to it. 

1419 

1420 This method is the "multiple key" analogue to the 

1421 :meth:`.CacheRegion.cache_on_arguments` method. 

1422 

1423 Example:: 

1424 

1425 @someregion.cache_multi_on_arguments() 

1426 def generate_something(*keys): 

1427 return [ 

1428 somedatabase.query(key) 

1429 for key in keys 

1430 ] 

1431 

1432 The decorated function can be called normally. The decorator 

1433 will produce a list of cache keys using a mechanism similar to 

1434 that of :meth:`.CacheRegion.cache_on_arguments`, combining the 

1435 name of the function with the optional namespace and with the 

1436 string form of each key. It will then consult the cache using 

1437 the same mechanism as that of :meth:`.CacheRegion.get_multi` 

1438 to retrieve all current values; the originally passed keys 

1439 corresponding to those values which aren't generated or need 

1440 regeneration will be assembled into a new argument list, and 

1441 the decorated function is then called with that subset of 

1442 arguments. 

1443 

1444 The returned result is a list:: 

1445 

1446 result = generate_something("key1", "key2", "key3") 

1447 

1448 The decorator internally makes use of the 

1449 :meth:`.CacheRegion.get_or_create_multi` method to access the 

1450 cache and conditionally call the function. See that 

1451 method for additional behavioral details. 

1452 

1453 Unlike the :meth:`.CacheRegion.cache_on_arguments` method, 

1454 :meth:`.CacheRegion.cache_multi_on_arguments` works only with 

1455 a single function signature, one which takes a simple list of 

1456 keys as arguments. 

1457 

1458 Like :meth:`.CacheRegion.cache_on_arguments`, the decorated function 

1459 is also provided with a ``set()`` method, which here accepts a 

1460 mapping of keys and values to set in the cache:: 

1461 

1462 generate_something.set({"k1": "value1", 

1463 "k2": "value2", "k3": "value3"}) 

1464 

1465 ...an ``invalidate()`` method, which has the effect of deleting 

1466 the given sequence of keys using the same mechanism as that of 

1467 :meth:`.CacheRegion.delete_multi`:: 

1468 

1469 generate_something.invalidate("k1", "k2", "k3") 

1470 

1471 ...a ``refresh()`` method, which will call the creation 

1472 function, cache the new values, and return them:: 

1473 

1474 values = generate_something.refresh("k1", "k2", "k3") 

1475 

1476 ...and a ``get()`` method, which will return values 

1477 based on the given arguments:: 

1478 

1479 values = generate_something.get("k1", "k2", "k3") 

1480 

1481 .. versionadded:: 0.5.3 Added ``get()`` method to decorated 

1482 function. 

1483 

1484 Parameters passed to :meth:`.CacheRegion.cache_multi_on_arguments` 

1485 have the same meaning as those passed to 

1486 :meth:`.CacheRegion.cache_on_arguments`. 

1487 

1488 :param namespace: optional string argument which will be 

1489 established as part of each cache key. 

1490 

1491 :param expiration_time: if not None, will override the normal 

1492 expiration time. May be passed as an integer or a 

1493 callable. 

1494 

1495 :param should_cache_fn: passed to 

1496 :meth:`.CacheRegion.get_or_create_multi`. This function is given a 

1497 value as returned by the creator, and only if it returns True will 

1498 that value be placed in the cache. 

1499 

1500 :param asdict: if ``True``, the decorated function should return 

1501 its result as a dictionary of keys->values, and the final result 

1502 of calling the decorated function will also be a dictionary. 

1503 If left at its default value of ``False``, the decorated function 

1504 should return its result as a list of values, and the final 

1505 result of calling the decorated function will also be a list. 

1506 

1507 When ``asdict==True`` if the dictionary returned by the decorated 

1508 function is missing keys, those keys will not be cached. 

1509 

1510 :param to_str: callable, will be called on each function argument 

1511 in order to convert to a string. Defaults to ``str()``. If the 

1512 function accepts non-ascii unicode arguments on Python 2.x, the 

1513 ``unicode()`` builtin can be substituted, but note this will 

1514 produce unicode cache keys which may require key mangling before 

1515 reaching the cache. 

1516 

1517 .. versionadded:: 0.5.0 

1518 

1519 :param function_multi_key_generator: a function that will produce a 

1520 list of keys. This function will supersede the one configured on the 

1521 :class:`.CacheRegion` itself. 

1522 

1523 .. versionadded:: 0.5.5 

1524 

1525 .. seealso:: 

1526 

1527 :meth:`.CacheRegion.cache_on_arguments` 

1528 

1529 :meth:`.CacheRegion.get_or_create_multi` 

1530 

1531 """ 

1532 expiration_time_is_callable = compat.callable(expiration_time) 

1533 

1534 if function_multi_key_generator is None: 

1535 function_multi_key_generator = self.function_multi_key_generator 

1536 

1537 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw): 

1538 cache_keys = arg 

1539 keys = key_generator(*arg, **kw) 

1540 key_lookup = dict(zip(keys, cache_keys)) 

1541 

1542 @wraps(user_func) 

1543 def creator(*keys_to_create): 

1544 return user_func(*[key_lookup[k] for k in keys_to_create]) 

1545 

1546 timeout = ( 

1547 expiration_time() 

1548 if expiration_time_is_callable 

1549 else expiration_time 

1550 ) 

1551 

1552 if asdict: 

1553 

1554 def dict_create(*keys): 

1555 d_values = creator(*keys) 

1556 return [ 

1557 d_values.get(key_lookup[k], NO_VALUE) for k in keys 

1558 ] 

1559 

1560 def wrap_cache_fn(value): 

1561 if value is NO_VALUE: 

1562 return False 

1563 elif not should_cache_fn: 

1564 return True 

1565 else: 

1566 return should_cache_fn(value) 

1567 

1568 result = self.get_or_create_multi( 

1569 keys, dict_create, timeout, wrap_cache_fn 

1570 ) 

1571 result = dict( 

1572 (k, v) 

1573 for k, v in zip(cache_keys, result) 

1574 if v is not NO_VALUE 

1575 ) 

1576 else: 

1577 result = self.get_or_create_multi( 

1578 keys, creator, timeout, should_cache_fn 

1579 ) 

1580 

1581 return result 

1582 

1583 def cache_decorator(user_func): 

1584 key_generator = function_multi_key_generator( 

1585 namespace, user_func, to_str=to_str 

1586 ) 

1587 

1588 def invalidate(*arg): 

1589 keys = key_generator(*arg) 

1590 self.delete_multi(keys) 

1591 

1592 def set_(mapping): 

1593 keys = list(mapping) 

1594 gen_keys = key_generator(*keys) 

1595 self.set_multi( 

1596 dict( 

1597 (gen_key, mapping[key]) 

1598 for gen_key, key in zip(gen_keys, keys) 

1599 ) 

1600 ) 

1601 

1602 def get(*arg): 

1603 keys = key_generator(*arg) 

1604 return self.get_multi(keys) 

1605 

1606 def refresh(*arg): 

1607 keys = key_generator(*arg) 

1608 values = user_func(*arg) 

1609 if asdict: 

1610 self.set_multi(dict(zip(keys, [values[a] for a in arg]))) 

1611 return values 

1612 else: 

1613 self.set_multi(dict(zip(keys, values))) 

1614 return values 

1615 

1616 user_func.set = set_ 

1617 user_func.invalidate = invalidate 

1618 user_func.refresh = refresh 

1619 user_func.get = get 

1620 

1621 # Use `decorate` to preserve the signature of :param:`user_func`. 

1622 

1623 return decorate( 

1624 user_func, partial(get_or_create_for_user_func, key_generator) 

1625 ) 

1626 

1627 return cache_decorator 

1628 

1629 

1630def make_region(*arg, **kw): 

1631 """Instantiate a new :class:`.CacheRegion`. 

1632 

1633 Currently, :func:`.make_region` is a passthrough 

1634 to :class:`.CacheRegion`. See that class for 

1635 constructor arguments. 

1636 

1637 """ 

1638 return CacheRegion(*arg, **kw)