From 2e5eb693de627ef0503d380429b8c042a50f1b1b Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Fri, 2 Sep 2016 21:43:16 -0700 Subject: [PATCH] objectcache: add WANObjectCacheReaper for assuring purges * This fixes keys based on some sort of change log. Updates are wrapped in a mutex and keep track of the last known good position. * Make WANObjectReapUpdate class that cleans up title related keys using the recentchanges table. This triggers as a deferred updates on RC view. Change-Id: I7f14b9ca2533032147e62b1a3cc004a23da86579 --- autoload.php | 2 + includes/DefaultSettings.php | 13 ++ includes/cache/LinkCache.php | 14 ++ includes/deferred/WANCacheReapUpdate.php | 126 +++++++++++ includes/filerepo/file/LocalFile.php | 9 + includes/libs/objectcache/WANObjectCache.php | 59 +++++ .../libs/objectcache/WANObjectCacheReaper.php | 204 ++++++++++++++++++ includes/page/WikiPage.php | 11 + .../specialpage/ChangesListSpecialPage.php | 9 + includes/user/User.php | 11 + .../libs/objectcache/WANObjectCacheTest.php | 58 +++++ 11 files changed, 516 insertions(+) create mode 100644 includes/deferred/WANCacheReapUpdate.php create mode 100644 includes/libs/objectcache/WANObjectCacheReaper.php diff --git a/autoload.php b/autoload.php index e7c97ad049..329cdb37b5 100644 --- a/autoload.php +++ b/autoload.php @@ -1541,7 +1541,9 @@ $wgAutoloadLocalClasses = [ 'ViewCLI' => __DIR__ . '/maintenance/view.php', 'VirtualRESTService' => __DIR__ . '/includes/libs/virtualrest/VirtualRESTService.php', 'VirtualRESTServiceClient' => __DIR__ . '/includes/libs/virtualrest/VirtualRESTServiceClient.php', + 'WANCacheReapUpdate' => __DIR__ . '/includes/deferred/WANCacheReapUpdate.php', 'WANObjectCache' => __DIR__ . '/includes/libs/objectcache/WANObjectCache.php', + 'WANObjectCacheReaper' => __DIR__ . '/includes/libs/objectcache/WANObjectCacheReaper.php', 'WantedCategoriesPage' => __DIR__ . '/includes/specials/SpecialWantedcategories.php', 'WantedFilesPage' => __DIR__ . '/includes/specials/SpecialWantedfiles.php', 'WantedPagesPage' => __DIR__ . '/includes/specials/SpecialWantedpages.php', diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 086b615e42..58ddb699cd 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -2342,6 +2342,19 @@ $wgWANObjectCaches = [ */ ]; +/** + * Verify and enforce WAN cache purges using reliable DB sources as streams. + * + * These secondary cache purges are de-duplicated via simple cache mutexes. + * This improves consistency when cache purges are lost, which becomes more likely + * as more cache servers are added or if there are multiple datacenters. Only keys + * related to important mutable content will be checked. + * + * @var bool + * @since 1.29 + */ +$wgEnableWANCacheReaper = false; + /** * Main object stash type. This should be a fast storage system for storing * lightweight data like hit counters and user activity. Sites with multiple diff --git a/includes/cache/LinkCache.php b/includes/cache/LinkCache.php index 23cc26d5e2..b720decb86 100644 --- a/includes/cache/LinkCache.php +++ b/includes/cache/LinkCache.php @@ -278,6 +278,20 @@ class LinkCache { return $id; } + /** + * @param WANObjectCache $cache + * @param TitleValue $t + * @return string[] + * @since 1.28 + */ + public function getMutableCacheKeys( WANObjectCache $cache, TitleValue $t ) { + if ( $this->isCacheable( $t ) ) { + return [ $cache->makeKey( 'page', $t->getNamespace(), sha1( $t->getDBkey() ) ) ]; + } + + return []; + } + private function isCacheable( LinkTarget $title ) { return ( $title->inNamespace( NS_TEMPLATE ) || $title->inNamespace( NS_FILE ) ); } diff --git a/includes/deferred/WANCacheReapUpdate.php b/includes/deferred/WANCacheReapUpdate.php new file mode 100644 index 0000000000..33ddc59cbd --- /dev/null +++ b/includes/deferred/WANCacheReapUpdate.php @@ -0,0 +1,126 @@ +db = $db; + $this->logger = $logger; + } + + function doUpdate() { + $reaper = new WANObjectCacheReaper( + ObjectCache::getMainWANInstance(), + ObjectCache::getLocalClusterInstance(), + [ $this, 'getTitleChangeEvents' ], + [ $this, 'getEventAffectedKeys' ], + [ + 'channel' => 'table:recentchanges:' . $this->db->getWikiID(), + 'logger' => $this->logger + ] + ); + + $reaper->invoke( 100 ); + } + + /** + * @see WANObjectCacheRepear + * + * @param int $start + * @param int $id + * @param int $end + * @param int $limit + * @return TitleValue[] + */ + public function getTitleChangeEvents( $start, $id, $end, $limit ) { + $db = $this->db; + $encStart = $db->addQuotes( $db->timestamp( $start ) ); + $encEnd = $db->addQuotes( $db->timestamp( $end ) ); + $id = (int)$id; // cast NULL => 0 since rc_id is an integer + + $res = $db->select( + 'recentchanges', + [ 'rc_namespace', 'rc_title', 'rc_timestamp', 'rc_id' ], + [ + $db->makeList( [ + "rc_timestamp > $encStart", + "rc_timestamp = $encStart AND rc_id > " . $db->addQuotes( $id ) + ], LIST_OR ), + "rc_timestamp < $encEnd" + ], + __METHOD__, + [ 'ORDER BY' => 'rc_timestamp ASC, rc_id ASC', 'LIMIT' => $limit ] + ); + + $events = []; + foreach ( $res as $row ) { + $events[] = [ + 'id' => (int)$row->rc_id, + 'pos' => (int)wfTimestamp( TS_UNIX, $row->rc_timestamp ), + 'item' => new TitleValue( (int)$row->rc_namespace, $row->rc_title ) + ]; + } + + return $events; + } + + /** + * Gets a list of important cache keys associated with a title + * + * @see WANObjectCacheRepear + * @param WANObjectCache $cache + * @param TitleValue $t + * @returns string[] + */ + public function getEventAffectedKeys( WANObjectCache $cache, TitleValue $t ) { + /** @var WikiPage[]|LocalFile[]|User[] $entities */ + $entities = []; + + $entities[] = WikiPage::factory( Title::newFromTitleValue( $t ) ); + if ( $t->inNamespace( NS_FILE ) ) { + $entities[] = wfLocalFile( $t->getText() ); + } + if ( $t->inNamespace( NS_USER ) ) { + $entities[] = User::newFromName( $t->getText(), false ); + } + + $keys = []; + foreach ( $entities as $entity ) { + if ( $entity ) { + $keys = array_merge( $keys, $entity->getMutableCacheKeys( $cache ) ); + } + } + if ( $keys ) { + $this->logger->debug( __CLASS__ . ': got key(s) ' . implode( ', ', $keys ) ); + } + + return $keys; + } +} diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php index 16fe72d37c..be0751ff9a 100644 --- a/includes/filerepo/file/LocalFile.php +++ b/includes/filerepo/file/LocalFile.php @@ -240,6 +240,15 @@ class LocalFile extends File { return $this->repo->getSharedCacheKey( 'file', sha1( $this->getName() ) ); } + /** + * @param WANObjectCache $cache + * @return string[] + * @since 1.28 + */ + public function getMutableCacheKeys( WANObjectCache $cache ) { + return [ $this->getCacheKey() ]; + } + /** * Try to load file metadata from memcached, falling back to the database */ diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index 171c291cf1..75c79a98fc 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -1127,6 +1127,65 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { return $values; } + /** + * Locally set a key to expire soon if it is stale based on $purgeTimestamp + * + * This sets stale keys' time-to-live at HOLDOFF_TTL seconds, which both avoids + * broadcasting in mcrouter setups and also avoids races with new tombstones. + * + * @param string $key Cache key + * @param int $purgeTimestamp UNIX timestamp of purge + * @param bool &$isStale Whether the key is stale + * @return bool Success + * @since 1.28 + */ + public function reap( $key, $purgeTimestamp, &$isStale = false ) { + $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL; + $wrapped = $this->cache->get( self::VALUE_KEY_PREFIX . $key ); + if ( is_array( $wrapped ) && $wrapped[self::FLD_TIME] < $minAsOf ) { + $isStale = true; + $this->logger->warning( "Reaping stale value key '$key'." ); + $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation + $ok = $this->cache->changeTTL( self::VALUE_KEY_PREFIX . $key, $ttlReap ); + if ( !$ok ) { + $this->logger->error( "Could not complete reap of key '$key'." ); + } + + return $ok; + } + + $isStale = false; + + return true; + } + + /** + * Locally set a "check" key to expire soon if it is stale based on $purgeTimestamp + * + * @param string $key Cache key + * @param int $purgeTimestamp UNIX timestamp of purge + * @param bool &$isStale Whether the key is stale + * @return bool Success + * @since 1.28 + */ + public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) { + $purge = $this->parsePurgeValue( $this->cache->get( self::TIME_KEY_PREFIX . $key ) ); + if ( $purge && $purge[self::FLD_TIME] < $purgeTimestamp ) { + $isStale = true; + $this->logger->warning( "Reaping stale check key '$key'." ); + $ok = $this->cache->changeTTL( self::TIME_KEY_PREFIX . $key, 1 ); + if ( !$ok ) { + $this->logger->error( "Could not complete reap of check key '$key'." ); + } + + return $ok; + } + + $isStale = false; + + return false; + } + /** * @see BagOStuff::makeKey() * @param string ... Key component diff --git a/includes/libs/objectcache/WANObjectCacheReaper.php b/includes/libs/objectcache/WANObjectCacheReaper.php new file mode 100644 index 0000000000..62e4536fd8 --- /dev/null +++ b/includes/libs/objectcache/WANObjectCacheReaper.php @@ -0,0 +1,204 @@ +cache = $cache; + $this->store = $store; + + $this->logChunkCallback = $logCallback; + $this->keyListCallback = $keyCallback; + if ( isset( $params['channel'] ) ) { + $this->channel = $params['channel']; + } else { + throw new UnexpectedValueException( "No channel specified." ); + } + + $this->initialStartWindow = isset( $params['initialStartWindow'] ) + ? $params['initialStartWindow'] + : 3600; + $this->logger = isset( $params['logger'] ) + ? $params['logger'] + : new NullLogger(); + } + + public function setLogger( LoggerInterface $logger ) { + $this->logger = $logger; + } + + /** + * Check and reap stale keys based on a chunk of events + * + * @param int $n Number of events + * @return int Number of keys checked + */ + final public function invoke( $n = 100 ) { + $posKey = $this->store->makeGlobalKey( 'WANCache', 'reaper', $this->channel ); + $scopeLock = $this->store->getScopedLock( "$posKey:busy", 0 ); + if ( !$scopeLock ) { + return 0; + } + + $now = time(); + $status = $this->store->get( $posKey ); + if ( !$status ) { + $status = [ 'pos' => $now - $this->initialStartWindow, 'id' => null ]; + } + + // Get events for entities who's keys tombstones/hold-off should have expired by now + $events = call_user_func_array( + $this->logChunkCallback, + [ $status['pos'], $status['id'], $now - WANObjectCache::HOLDOFF_TTL - 1, $n ] + ); + + $event = null; + $keyEvents = []; + foreach ( $events as $event ) { + $keys = call_user_func_array( + $this->keyListCallback, + [ $this->cache, $event['item'] ] + ); + foreach ( $keys as $key ) { + unset( $keyEvents[$key] ); // use only the latest per key + $keyEvents[$key] = [ + 'pos' => $event['pos'], + 'id' => $event['id'] + ]; + } + } + + $purgeCount = 0; + $lastOkEvent = null; + foreach ( $keyEvents as $key => $keyEvent ) { + if ( !$this->cache->reap( $key, $keyEvent['pos'] ) ) { + break; + } + ++$purgeCount; + $lastOkEvent = $event; + } + + if ( $lastOkEvent ) { + $ok = $this->store->merge( + $posKey, + function ( $bag, $key, $curValue ) use ( $lastOkEvent ) { + if ( !$curValue ) { + // Use new position + } else { + $curCoord = [ $curValue['pos'], $curValue['id'] ]; + $newCoord = [ $lastOkEvent['pos'], $lastOkEvent['id'] ]; + if ( $newCoord < $curCoord ) { + // Keep prior position instead of rolling it back + return $curValue; + } + } + + return [ + 'pos' => $lastOkEvent['pos'], + 'id' => $lastOkEvent['id'], + 'ctime' => $curValue ? $curValue['ctime'] : date( 'c' ) + ]; + }, + IExpiringStore::TTL_INDEFINITE + ); + + $pos = $lastOkEvent['pos']; + $id = $lastOkEvent['id']; + if ( $ok ) { + $this->logger->info( "Updated cache reap position ($pos, $id)." ); + } else { + $this->logger->error( "Could not update cache reap position ($pos, $id)." ); + } + } + + ScopedCallback::consume( $scopeLock ); + + return $purgeCount; + } + + /** + * @return array|bool Returns (pos, id) map or false if not set + */ + public function getState() { + $posKey = $this->store->makeGlobalKey( 'WANCache', 'reaper', $this->channel ); + + return $this->store->get( $posKey ); + } +} diff --git a/includes/page/WikiPage.php b/includes/page/WikiPage.php index ab95eea4e3..1c1412a81f 100644 --- a/includes/page/WikiPage.php +++ b/includes/page/WikiPage.php @@ -3652,4 +3652,15 @@ class WikiPage implements Page, IDBAccessObject { public function getSourceURL() { return $this->getTitle()->getCanonicalURL(); } + + /* + * @param WANObjectCache $cache + * @return string[] + * @since 1.28 + */ + public function getMutableCacheKeys( WANObjectCache $cache ) { + $linkCache = MediaWikiServices::getInstance()->getLinkCache(); + + return $linkCache->getMutableCacheKeys( $cache, $this->getTitle()->getTitleValue() ); + } } diff --git a/includes/specialpage/ChangesListSpecialPage.php b/includes/specialpage/ChangesListSpecialPage.php index 00efeae1e2..00439a12a4 100644 --- a/includes/specialpage/ChangesListSpecialPage.php +++ b/includes/specialpage/ChangesListSpecialPage.php @@ -20,6 +20,7 @@ * @file * @ingroup SpecialPage */ +use MediaWiki\Logger\LoggerFactory; /** * Special page which uses a ChangesList to show query results. @@ -77,6 +78,14 @@ abstract class ChangesListSpecialPage extends SpecialPage { $this->webOutput( $rows, $opts ); $rows->free(); + + if ( $this->getConfig()->get( 'EnableWANCacheReaper' ) ) { + // Clean up any bad page entries for titles showing up in RC + DeferredUpdates::addUpdate( new WANCacheReapUpdate( + $this->getDB(), + LoggerFactory::getInstance( 'objectcache' ) + ) ); + } } /** diff --git a/includes/user/User.php b/includes/user/User.php index fed64c2a68..6763ec1c48 100644 --- a/includes/user/User.php +++ b/includes/user/User.php @@ -468,6 +468,17 @@ class User implements IDBAccessObject { return $cache->makeGlobalKey( 'user', 'id', wfWikiID(), $this->mId ); } + /** + * @param WANObjectCache $cache + * @return string[] + * @since 1.28 + */ + public function getMutableCacheKeys( WANObjectCache $cache ) { + $id = $this->getId(); + + return $id ? [ $this->getCacheKey( $cache ) ] : []; + } + /** * Load user data from shared cache, given mId has already been set. * diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php index aa46c966ad..d7ed4bd9ce 100644 --- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php +++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php @@ -872,6 +872,62 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase { $this->assertGreaterThan( -5.1, $curTTL, "Correct CTL" ); } + /** + * @covers WANObjectCache::reap() + * @covers WANObjectCache::reapCheckKey() + */ + public function testReap() { + $vKey1 = wfRandomString(); + $vKey2 = wfRandomString(); + $tKey1 = wfRandomString(); + $tKey2 = wfRandomString(); + $value = 'moo'; + + $knownPurge = time() - 60; + $goodTime = microtime( true ) - 5; + $badTime = microtime( true ) - 300; + + $this->internalCache->set( + WANObjectCache::VALUE_KEY_PREFIX . $vKey1, + [ + WANObjectCache::FLD_VERSION => WANObjectCache::VERSION, + WANObjectCache::FLD_VALUE => $value, + WANObjectCache::FLD_TTL => 3600, + WANObjectCache::FLD_TIME => $goodTime + ] + ); + $this->internalCache->set( + WANObjectCache::VALUE_KEY_PREFIX . $vKey2, + [ + WANObjectCache::FLD_VERSION => WANObjectCache::VERSION, + WANObjectCache::FLD_VALUE => $value, + WANObjectCache::FLD_TTL => 3600, + WANObjectCache::FLD_TIME => $badTime + ] + ); + $this->internalCache->set( + WANObjectCache::TIME_KEY_PREFIX . $tKey1, + WANObjectCache::PURGE_VAL_PREFIX . $goodTime + ); + $this->internalCache->set( + WANObjectCache::TIME_KEY_PREFIX . $tKey2, + WANObjectCache::PURGE_VAL_PREFIX . $badTime + ); + + $this->assertEquals( $value, $this->cache->get( $vKey1 ) ); + $this->assertEquals( $value, $this->cache->get( $vKey2 ) ); + $this->cache->reap( $vKey1, $knownPurge, $bad1 ); + $this->cache->reap( $vKey2, $knownPurge, $bad2 ); + + $this->assertFalse( $bad1 ); + $this->assertTrue( $bad2 ); + + $this->cache->reapCheckKey( $tKey1, $knownPurge, $tBad1 ); + $this->cache->reapCheckKey( $tKey2, $knownPurge, $tBad2 ); + $this->assertFalse( $tBad1 ); + $this->assertTrue( $tBad2 ); + } + /** * @covers WANObjectCache::set() */ @@ -926,6 +982,8 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase { $wanCache->getMulti( [ 'x', 'y' ], $ctls, [ 'check2' ] ); $wanCache->getWithSetCallback( 'p', 30, $valFunc ); $wanCache->getCheckKeyTime( 'zzz' ); + $wanCache->reap( 'x', time() - 300 ); + $wanCache->reap( 'zzz', time() - 300 ); } /** -- 2.20.1