From: Aaron Schulz Date: Tue, 21 Nov 2017 22:11:01 +0000 (-0800) Subject: objectcache: add "staleTTL" into WANObjectCache::getWithSetCallback() X-Git-Tag: 1.31.0-rc.0~1407^2 X-Git-Url: http://git.cyclocoop.org///%22%40url%40//%22?a=commitdiff_plain;h=fce6862e46be7056d723e0b30283dc8d07168bbf;p=lhc%2Fweb%2Fwiklou.git objectcache: add "staleTTL" into WANObjectCache::getWithSetCallback() This simply involves passing it through to the set() call Also added some related commons to adaptiveTTL() involving usage of this option. Change-Id: Id5833a5d4efb6cad2eb646832e5b0188e86e12fc --- diff --git a/includes/libs/objectcache/HashBagOStuff.php b/includes/libs/objectcache/HashBagOStuff.php index 6d583da07c..f8e3b17a8c 100644 --- a/includes/libs/objectcache/HashBagOStuff.php +++ b/includes/libs/objectcache/HashBagOStuff.php @@ -52,7 +52,7 @@ class HashBagOStuff extends BagOStuff { protected function expire( $key ) { $et = $this->bag[$key][self::KEY_EXP]; - if ( $et == self::TTL_INDEFINITE || $et > time() ) { + if ( $et == self::TTL_INDEFINITE || $et > $this->getCurrentTime() ) { return false; } @@ -115,4 +115,8 @@ class HashBagOStuff extends BagOStuff { public function clear() { $this->bag = []; } + + protected function getCurrentTime() { + return time(); + } } diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index e63b32ed0f..723ccc0eb5 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -135,7 +135,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { const TTL_LAGGED = 30; /** Idiom for delete() for "no hold-off" */ const HOLDOFF_NONE = 0; - /** Idiom for set() for "do not augment the storage medium TTL" */ + /** Idiom for set()/getWithSetCallback() for "do not augment the storage medium TTL" */ const STALE_TTL_NONE = 0; /** Idiom for getWithSetCallback() for "no minimum required as-of timestamp" */ @@ -868,6 +868,11 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * Default: WANObjectCache::LOW_TTL. * - ageNew: Consider popularity refreshes only once a key reaches this age in seconds. * Default: WANObjectCache::AGE_NEW. + * - staleTTL: Seconds to keep the key around if it is stale. This means that on cache + * miss the callback may get $oldValue/$oldAsOf values for keys that have already been + * expired for this specified time. This is useful if adaptiveTTL() is used on the old + * value's as-of time when it is verified as still being correct. + * Default: WANObjectCache::STALE_TTL_NONE * @return mixed Value found or written to the key * @note Options added in 1.28: version, busyValue, hotTTR, ageNew, pcGroup, minAsOf * @note Callable type hints are not used to avoid class-autoloading @@ -957,6 +962,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { protected function doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf = null ) { $lowTTL = isset( $opts['lowTTL'] ) ? $opts['lowTTL'] : min( self::LOW_TTL, $ttl ); $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE; + $staleTTL = isset( $opts['staleTTL'] ) ? $opts['staleTTL'] : self::STALE_TTL_NONE; $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : []; $busyValue = isset( $opts['busyValue'] ) ? $opts['busyValue'] : null; $popWindow = isset( $opts['hotTTR'] ) ? $opts['hotTTR'] : self::HOT_TTR; @@ -1056,6 +1062,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { if ( $valueIsCacheable ) { $setOpts['lockTSE'] = $lockTSE; + $setOpts['staleTTL'] = $staleTTL; // Use best known "since" timestamp if not provided $setOpts += [ 'since' => $preCallbackTime ]; // Update the cache; this will fail if the key is tombstoned @@ -1496,6 +1503,46 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * $ttl = $cache->adaptiveTTL( $mtime, $cache::TTL_DAY ); * @endcode * + * Another use case is when there are no applicable "last modified" fields in the DB, + * and there are too many dependencies for explicit purges to be viable, and the rate of + * change to relevant content is unstable, and it is highly valued to have the cached value + * be as up-to-date as possible. + * + * Example usage: + * @code + * $query = ""; + * $idListFromComplexQuery = $cache->getWithSetCallback( + * $cache->makeKey( 'complex-graph-query', $hashOfQuery ), + * GraphQueryClass::STARTING_TTL, + * function ( $oldValue, &$ttl, array &$setOpts, $oldAsOf ) use ( $query, $cache ) { + * $gdb = $this->getReplicaGraphDbConnection(); + * // Account for any snapshot/replica DB lag + * $setOpts += GraphDatabase::getCacheSetOptions( $gdb ); + * + * $newList = iterator_to_array( $gdb->query( $query ) ); + * sort( $newList, SORT_NUMERIC ); // normalize + * + * $minTTL = GraphQueryClass::MIN_TTL; + * $maxTTL = GraphQueryClass::MAX_TTL; + * if ( $oldValue !== false ) { + * // Note that $oldAsOf is the last time this callback ran + * $ttl = ( $newList === $oldValue ) + * // No change: cache for 150% of the age of $oldValue + * ? $cache->adaptiveTTL( $oldAsOf, $maxTTL, $minTTL, 1.5 ) + * // Changed: cache for %50 of the age of $oldValue + * : $cache->adaptiveTTL( $oldAsOf, $maxTTL, $minTTL, .5 ); + * } + * + * return $newList; + * }, + * [ + * // Keep stale values around for doing comparisons for TTL calculations. + * // High values improve long-tail keys hit-rates, though might waste space. + * 'staleTTL' => GraphQueryClass::GRACE_TTL + * ] + * ); + * @endcode + * * @param int|float $mtime UNIX timestamp * @param int $maxTTL Maximum TTL (seconds) * @param int $minTTL Minimum TTL (seconds); Default: 30 diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php index 592f72a09e..b779231d9f 100644 --- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php +++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php @@ -266,6 +266,46 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase { $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts ); $this->assertEquals( $value, $v, "Value still returned after deleted" ); $this->assertEquals( 1, $wasSet, "Value process cached while deleted" ); + + $backToTheFutureCache = new TimeAdjustableWANObjectCache( [ + 'cache' => new TimeAdjustableHashBagOStuff(), + 'pool' => 'empty' + ] ); + + $oldValReceived = -1; + $oldAsOfReceived = -1; + $checkFunc = function ( $oldVal, &$ttl, array $setOpts, $oldAsOf ) + use ( &$oldValReceived, &$oldAsOfReceived, &$wasSet ) { + ++$wasSet; + $oldValReceived = $oldVal; + $oldAsOfReceived = $oldAsOf; + + return 'xxx' . $wasSet; + }; + + $wasSet = 0; + $key = wfRandomString(); + $v = $backToTheFutureCache->getWithSetCallback( + $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); + $this->assertEquals( 'xxx1', $v, "Value returned" ); + $this->assertEquals( false, $oldValReceived, "Callback got no stale value" ); + $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" ); + + $backToTheFutureCache->setTime( microtime( true ) + 40 ); + $v = $backToTheFutureCache->getWithSetCallback( + $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); + $this->assertEquals( 'xxx2', $v, "Value still returned after expired" ); + $this->assertEquals( 2, $wasSet, "Value recalculated while expired" ); + $this->assertEquals( 'xxx1', $oldValReceived, "Callback got stale value" ); + $this->assertNotEquals( null, $oldAsOfReceived, "Callback got stale value" ); + + $backToTheFutureCache->setTime( microtime( true ) + 300 ); + $v = $backToTheFutureCache->getWithSetCallback( + $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); + $this->assertEquals( 'xxx3', $v, "Value still returned after expired" ); + $this->assertEquals( 3, $wasSet, "Value recalculated while expired" ); + $this->assertEquals( false, $oldValReceived, "Callback got no stale value" ); + $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" ); } public static function getWithSetCallback_provider() {