From bda755617ed2eed30934d21d1e0d84f619a523d3 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Niklas=20Laxstr=C3=B6m?= Date: Wed, 24 Oct 2012 14:59:37 +0000 Subject: [PATCH] Rewritten $wgAdaptiveMessageCache This was developed for translatewiki.net, which has but millions of messages in MediaWiki namespace. To avoid exploding the message cache, not all of them can be loaded. Instead of logging what messages have been requested, now it uses the fact that if the message has not been customised in the site language, there is most likely no reason to load it in any other language either. Change-Id: I6dd81b3858acfd2b73332e46668d46015e99c748 --- includes/DefaultSettings.php | 7 +- includes/Wiki.php | 3 - includes/cache/MessageCache.php | 138 +++----------------------------- 3 files changed, 14 insertions(+), 134 deletions(-) diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index f146943cbb..6cdacc1208 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -1853,9 +1853,10 @@ $wgUseLocalMessageCache = false; $wgLocalMessageCacheSerialized = true; /** - * Instead of caching everything, keep track which messages are requested and - * load only most used messages. This only makes sense if there is lots of - * interface messages customised in the wiki (like hundreds in many languages). + * Instead of caching everything, only cache those messages which have + * been customised in the site content language. This means that + * MediaWiki:Foo/ja is ignored if MediaWiki:Foo doesn't exist. + * This option is probably only useful for translatewiki.net. */ $wgAdaptiveMessageCache = false; diff --git a/includes/Wiki.php b/includes/Wiki.php index 7a6b37d9fd..d4840cc22e 100644 --- a/includes/Wiki.php +++ b/includes/Wiki.php @@ -574,9 +574,6 @@ class MediaWiki { // Execute a job from the queue $this->doJobs(); - // Log message usage, if $wgAdaptiveMessageCache is set to true - MessageCache::logMessages(); - // Log profiling data, e.g. in the database or UDP wfLogProfilingData(); diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.php index e061101bc6..148e26b38e 100644 --- a/includes/cache/MessageCache.php +++ b/includes/cache/MessageCache.php @@ -59,26 +59,6 @@ class MessageCache { /// Variable for tracking which variables are already loaded protected $mLoadedLanguages = array(); - /** - * Used for automatic detection of most used messages. - */ - protected $mRequestedMessages = array(); - - /** - * How long the message request counts are stored. Longer period gives - * better sample, but also takes longer to adapt changes. The counts - * are aggregrated per day, regardless of the value of this variable. - */ - protected static $mAdaptiveDataAge = 604800; // Is 7*24*3600 - - /** - * Filter the tail of less used messages that are requested more seldom - * than this factor times the number of request of most requested message. - * These messages are not loaded in the default set, but are still cached - * individually on demand with the normal cache expiry time. - */ - protected static $mAdaptiveInclusionThreshold = 0.05; - /** * Singleton instance * @@ -404,19 +384,20 @@ class MessageCache { ); $mostused = array(); - if ( $wgAdaptiveMessageCache ) { - $mostused = $this->getMostUsedMessages(); - if ( $code !== $wgLanguageCode ) { - foreach ( $mostused as $key => $value ) { - $mostused[$key] = "$value/$code"; - } + if ( $wgAdaptiveMessageCache && $code !== $wgLanguageCode ) { + if ( !isset( $this->mCache[$wgLanguageCode] ) ) { + $this->load( $wgLanguageCode ); + } + $mostused = array_keys( $this->mCache[$wgLanguageCode] ); + foreach ( $mostused as $key => $value ) { + $mostused[$key] = "$value/$code"; } } if ( count( $mostused ) ) { $conds['page_title'] = $mostused; } elseif ( $code !== $wgLanguageCode ) { - $conds[] = 'page_title' . $dbr->buildLike( $dbr->anyString(), "/$code" ); + $conds[] = 'page_title' . $dbr->buildLike( $dbr->anyString(), '/', $code ); } else { # Effectively disallows use of '/' character in NS_MEDIAWIKI for uses # other than language code. @@ -459,12 +440,6 @@ class MessageCache { $cache[$row->page_title] = $entry; } - foreach ( $mostused as $key ) { - if ( !isset( $cache[$key] ) ) { - $cache[$key] = '!NONEXISTENT'; - } - } - $cache['VERSION'] = MSG_CACHE_VERSION; wfProfileOut( __METHOD__ ); return $cache; @@ -646,13 +621,6 @@ class MessageCache { $uckey = $wgContLang->ucfirst( $lckey ); } - /** - * Record each message request, but only once per request. - * This information is not used unless $wgAdaptiveMessageCache - * is enabled. - */ - $this->mRequestedMessages[$uckey] = true; - # Try the MediaWiki namespace if( !$this->mDisable && $useDB ) { $title = $uckey; @@ -719,8 +687,6 @@ class MessageCache { * @return string|bool False on failure */ function getMsgFromNamespace( $title, $code ) { - global $wgAdaptiveMessageCache; - $this->load( $code ); if ( isset( $this->mCache[$code][$title] ) ) { $entry = $this->mCache[$code][$title]; @@ -739,15 +705,7 @@ class MessageCache { return $message; } - /** - * If message cache is in normal mode, it is guaranteed - * (except bugs) that there is always entry (or placeholder) - * in the cache if message exists. Thus we can do minor - * performance improvement and return false early. - */ - if ( !$wgAdaptiveMessageCache ) { - return false; - } + return false; } # Try the individual message cache @@ -907,7 +865,7 @@ class MessageCache { */ function clear() { $langs = Language::fetchLanguageNames( null, 'mw' ); - foreach ( array_keys($langs) as $code ) { + foreach ( array_keys( $langs ) as $code ) { # Global cache $this->mMemc->delete( wfMemcKey( 'messages', $code ) ); # Invalidate all local caches @@ -936,82 +894,6 @@ class MessageCache { return array( $message, $lang ); } - public static function logMessages() { - wfProfileIn( __METHOD__ ); - global $wgAdaptiveMessageCache; - if ( !$wgAdaptiveMessageCache || !self::$instance instanceof MessageCache ) { - wfProfileOut( __METHOD__ ); - return; - } - - $cachekey = wfMemckey( 'message-profiling' ); - $cache = wfGetCache( CACHE_DB ); - $data = $cache->get( $cachekey ); - - if ( !$data ) { - $data = array(); - } - - $age = self::$mAdaptiveDataAge; - $filterDate = substr( wfTimestamp( TS_MW, time() - $age ), 0, 8 ); - foreach ( array_keys( $data ) as $key ) { - if ( $key < $filterDate ) { - unset( $data[$key] ); - } - } - - $index = substr( wfTimestampNow(), 0, 8 ); - if ( !isset( $data[$index] ) ) { - $data[$index] = array(); - } - - foreach ( self::$instance->mRequestedMessages as $message => $_ ) { - if ( !isset( $data[$index][$message] ) ) { - $data[$index][$message] = 0; - } - $data[$index][$message]++; - } - - $cache->set( $cachekey, $data ); - wfProfileOut( __METHOD__ ); - } - - /** - * @return array - */ - public function getMostUsedMessages() { - wfProfileIn( __METHOD__ ); - $cachekey = wfMemcKey( 'message-profiling' ); - $cache = wfGetCache( CACHE_DB ); - $data = $cache->get( $cachekey ); - if ( !$data ) { - wfProfileOut( __METHOD__ ); - return array(); - } - - $list = array(); - - foreach( $data as $messages ) { - foreach( $messages as $message => $count ) { - $key = $message; - if ( !isset( $list[$key] ) ) { - $list[$key] = 0; - } - $list[$key] += $count; - } - } - - $max = max( $list ); - foreach ( $list as $message => $count ) { - if ( $count < intval( $max * self::$mAdaptiveInclusionThreshold ) ) { - unset( $list[$message] ); - } - } - - wfProfileOut( __METHOD__ ); - return array_keys( $list ); - } - /** * Get all message keys stored in the message cache for a given language. * If $code is the content language code, this will return all message keys -- 2.20.1