From 97e86d934b3485a7f4992bd3342c4a6ad6f77a0b Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Mon, 25 Jun 2018 10:45:55 +0100 Subject: [PATCH] Limit the number of cached languages in MessageCache via MapCacheLRU Change-Id: I37a128cfc553e0edaab524098461776cec3fe08a --- includes/cache/MessageCache.php | 106 +++++++++++++++----------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.php index 0854a43e95..93959db21d 100644 --- a/includes/cache/MessageCache.php +++ b/includes/cache/MessageCache.php @@ -46,14 +46,11 @@ class MessageCache { const LOCK_TTL = 30; /** - * Process local cache of loaded messages that are defined in - * MediaWiki namespace. First array level is a language code, - * second level is message key and the values are either message - * content prefixed with space, or !NONEXISTENT for negative - * caching. - * @var array $mCache + * Process cache of loaded messages that are defined in MediaWiki namespace + * + * @var MapCacheLRU Map of (language code => key => " " or "!TOO BIG") */ - protected $mCache; + protected $cache; /** * @var bool[] Map of (language code => boolean) @@ -80,12 +77,6 @@ class MessageCache { /** @var Parser */ protected $mParser; - /** - * Variable for tracking which variables are already loaded - * @var array $mLoadedLanguages - */ - protected $mLoadedLanguages = []; - /** * @var bool $mInParser */ @@ -174,6 +165,8 @@ class MessageCache { $this->clusterCache = $clusterCache; $this->srvCache = $serverCache; + $this->cache = new MapCacheLRU( 5 ); // limit size for sanity + $this->mDisable = !$useDB; $this->mExpiry = $expiry; } @@ -247,7 +240,7 @@ class MessageCache { * * @param string $code Language to which load messages * @param int $mode Use MessageCache::FOR_UPDATE to skip process cache [optional] - * @throws MWException + * @throws InvalidArgumentException * @return bool */ protected function load( $code, $mode = null ) { @@ -256,7 +249,7 @@ class MessageCache { } # Don't do double loading... - if ( isset( $this->mLoadedLanguages[$code] ) && $mode != self::FOR_UPDATE ) { + if ( $this->cache->has( $code ) && $mode != self::FOR_UPDATE ) { return true; } @@ -296,8 +289,8 @@ class MessageCache { $staleCache = $cache; } else { $where[] = 'got from local cache'; + $this->cache->set( $code, $cache ); $success = true; - $this->mCache[$code] = $cache; } if ( !$success ) { @@ -326,7 +319,7 @@ class MessageCache { $staleCache = $cache; } else { $where[] = 'got from global cache'; - $this->mCache[$code] = $cache; + $this->cache->set( $code, $cache ); $this->saveToCaches( $cache, 'local-only', $code ); $success = true; } @@ -347,7 +340,7 @@ class MessageCache { } elseif ( $staleCache ) { # Use the stale cache while some other thread constructs the new one $where[] = 'using stale cache'; - $this->mCache[$code] = $staleCache; + $this->cache->set( $code, $staleCache ); $success = true; break; } elseif ( $failedAttempts > 0 ) { @@ -371,13 +364,14 @@ class MessageCache { if ( !$success ) { $where[] = 'loading FAILED - cache is disabled'; $this->mDisable = true; - $this->mCache = false; + $this->cache->set( $code, null ); wfDebugLog( 'MessageCacheError', __METHOD__ . ": Failed to load $code\n" ); # This used to throw an exception, but that led to nasty side effects like # the whole wiki being instantly down if the memcached server died - } else { - # All good, just record the success - $this->mLoadedLanguages[$code] = true; + } + + if ( !$this->cache->has( $code ) ) { // sanity + throw new LogicException( "Process cache for '$code' should be set by now." ); } $info = implode( ', ', $where ); @@ -417,7 +411,7 @@ class MessageCache { } $cache = $this->loadFromDB( $code, $mode ); - $this->mCache[$code] = $cache; + $this->cache->set( $code, $cache ); $saveSuccess = $this->saveToCaches( $cache, 'all', $code ); if ( !$saveSuccess ) { @@ -473,10 +467,10 @@ class MessageCache { $mostused = []; if ( $wgAdaptiveMessageCache && $code !== $wgLanguageCode ) { - if ( !isset( $this->mCache[$wgLanguageCode] ) ) { + if ( !$this->cache->has( $wgLanguageCode ) ) { $this->load( $wgLanguageCode ); } - $mostused = array_keys( $this->mCache[$wgLanguageCode] ); + $mostused = array_keys( $this->cache->get( $wgLanguageCode ) ); foreach ( $mostused as $key => $value ) { $mostused[$key] = "$value/$code"; } @@ -578,10 +572,10 @@ class MessageCache { // (a) Update the process cache with the new message text if ( $text === false ) { // Page deleted - $this->mCache[$code][$title] = '!NONEXISTENT'; + $this->cache->setField( $code, $title, '!NONEXISTENT' ); } else { // Ignore $wgMaxMsgCacheEntrySize so the process cache is up to date - $this->mCache[$code][$title] = ' ' . $text; + $this->cache->setField( $code, $title, ' ' . $text ); } // (b) Update the shared caches in a deferred update with a fresh DB snapshot @@ -600,24 +594,27 @@ class MessageCache { } // Load the messages from the master DB to avoid race conditions $cache = $this->loadFromDB( $code, self::FOR_UPDATE ); - $this->mCache[$code] = $cache; - // Load the process cache values and set the per-title cache keys + // Check if an individual cache key should exist and update cache accordingly $page = WikiPage::factory( Title::makeTitle( NS_MEDIAWIKI, $title ) ); $page->loadPageData( $page::READ_LATEST ); $text = $this->getMessageTextFromContent( $page->getContent() ); - // Check if an individual cache key should exist and update cache accordingly if ( is_string( $text ) && strlen( $text ) > $wgMaxMsgCacheEntrySize ) { - $titleKey = $this->bigMessageCacheKey( $this->mCache[$code]['HASH'], $title ); - $this->wanCache->set( $titleKey, ' ' . $text, $this->mExpiry ); + $this->wanCache->set( + $this->bigMessageCacheKey( $cache['HASH'], $title ), + ' ' . $text, + $this->mExpiry + ); } // Mark this cache as definitely being "latest" (non-volatile) so - // load() calls do try to refresh the cache with replica DB data - $this->mCache[$code]['LATEST'] = time(); + // load() calls do not try to refresh the cache with replica DB data + $cache['LATEST'] = time(); + // Update the process cache + $this->cache->set( $code, $cache ); // Pre-emptively update the local datacenter cache so things like edit filter and // blacklist changes are reflected immediately; these often use MediaWiki: pages. // The datacenter handling replace() calls should be the same one handling edits // as they require HTTP POST. - $this->saveToCaches( $this->mCache[$code], 'all', $code ); + $this->saveToCaches( $cache, 'all', $code ); // Release the lock now that the cache is saved ScopedCallback::consume( $scopedLock ); @@ -971,8 +968,8 @@ class MessageCache { public function getMsgFromNamespace( $title, $code ) { $this->load( $code ); - if ( isset( $this->mCache[$code][$title] ) ) { - $entry = $this->mCache[$code][$title]; + if ( $this->cache->hasField( $code, $title ) ) { + $entry = $this->cache->getField( $code, $title ); if ( substr( $entry, 0, 1 ) === ' ' ) { // The message exists and is not '!TOO BIG' return (string)substr( $entry, 1 ); @@ -984,40 +981,40 @@ class MessageCache { $message = false; Hooks::run( 'MessagesPreLoad', [ $title, &$message, $code ] ); if ( $message !== false ) { - $this->mCache[$code][$title] = ' ' . $message; + $this->cache->setField( $code, $title, ' ' . $message ); } else { - $this->mCache[$code][$title] = '!NONEXISTENT'; + $this->cache->setField( $code, $title, '!NONEXISTENT' ); } return $message; } // Individual message cache key - $titleKey = $this->bigMessageCacheKey( $this->mCache[$code]['HASH'], $title ); + $bigKey = $this->bigMessageCacheKey( $this->cache->getField( $code, 'HASH' ), $title ); if ( $this->mCacheVolatile[$code] ) { $entry = false; // Make sure that individual keys respect the WAN cache holdoff period too LoggerFactory::getInstance( 'MessageCache' )->debug( __METHOD__ . ': loading volatile key \'{titleKey}\'', - [ 'titleKey' => $titleKey, 'code' => $code ] ); + [ 'titleKey' => $bigKey, 'code' => $code ] ); } else { // Try the individual message cache - $entry = $this->wanCache->get( $titleKey ); + $entry = $this->wanCache->get( $bigKey ); } if ( $entry !== false ) { if ( substr( $entry, 0, 1 ) === ' ' ) { - $this->mCache[$code][$title] = $entry; + $this->cache->setField( $code, $title, $entry ); // The message exists, so make sure a string is returned return (string)substr( $entry, 1 ); } elseif ( $entry === '!NONEXISTENT' ) { - $this->mCache[$code][$title] = '!NONEXISTENT'; + $this->cache->setField( $code, $title, '!NONEXISTENT' ); return false; } else { // Corrupt/obsolete entry, delete it - $this->wanCache->delete( $titleKey ); + $this->wanCache->delete( $bigKey ); } } @@ -1040,14 +1037,14 @@ class MessageCache { if ( $content ) { $message = $this->getMessageTextFromContent( $content ); if ( is_string( $message ) ) { - $this->mCache[$code][$title] = ' ' . $message; - $this->wanCache->set( $titleKey, ' ' . $message, $this->mExpiry, $cacheOpts ); + $this->cache->setField( $code, $title, ' ' . $message ); + $this->wanCache->set( $bigKey, ' ' . $message, $this->mExpiry, $cacheOpts ); } } else { // A possibly temporary loading failure LoggerFactory::getInstance( 'MessageCache' )->warning( __METHOD__ . ': failed to load message page text for \'{titleKey}\'', - [ 'titleKey' => $titleKey, 'code' => $code ] ); + [ 'titleKey' => $bigKey, 'code' => $code ] ); $message = null; // no negative caching } } else { @@ -1056,8 +1053,8 @@ class MessageCache { if ( $message === false ) { // Negative caching in case a "too big" message is no longer available (deleted) - $this->mCache[$code][$title] = '!NONEXISTENT'; - $this->wanCache->set( $titleKey, '!NONEXISTENT', $this->mExpiry, $cacheOpts ); + $this->cache->setField( $code, $title, '!NONEXISTENT' ); + $this->wanCache->set( $bigKey, '!NONEXISTENT', $this->mExpiry, $cacheOpts ); } return $message; @@ -1192,13 +1189,12 @@ class MessageCache { * * Mainly used after a mass rebuild */ - function clear() { + public function clear() { $langs = Language::fetchLanguageNames( null, 'mw' ); foreach ( array_keys( $langs ) as $code ) { $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) ); } - - $this->mLoadedLanguages = []; + $this->cache->clear(); } /** @@ -1235,12 +1231,12 @@ class MessageCache { global $wgContLang; $this->load( $code ); - if ( !isset( $this->mCache[$code] ) ) { + if ( !$this->cache->has( $code ) ) { // Apparently load() failed return null; } // Remove administrative keys - $cache = $this->mCache[$code]; + $cache = $this->cache->get( $code ); unset( $cache['VERSION'] ); unset( $cache['EXPIRY'] ); unset( $cache['EXCESSIVE'] ); -- 2.20.1