Merge "Enable fallback languages when retrieving messages"

diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.php
index 49db857..307d301 100644 (file)
--- a/includes/cache/MessageCache.php
+++ b/includes/cache/MessageCache.php
@@ -628,51 +628,62 @@ class MessageCache {
        /**
         * Get a message from either the content language or the user language.
         *
-        * @param string $key The message cache key
-        * @param bool $useDB Get the message from the DB, false to use only
-        *               the localisation
-        * @param bool|string $langcode Code of the language to get the message for, if
-        *                  it is a valid code create a language for that language,
-        *                  if it is a string but not a valid code then make a basic
-        *                  language object, if it is a false boolean then use the
-        *                  current users language (as a fallback for the old
-        *                  parameter functionality), or if it is a true boolean
-        *                  then use the wikis content language (also as a
-        *                  fallback).
-        * @param bool $isFullKey Specifies whether $key is a two part key "msg/lang".
+        * First, assemble a list of languages to attempt getting the message from. This
+        * chain begins with the requested language and its fallbacks and then continues with
+        * the content language and its fallbacks. For each language in the chain, the following
+        * process will occur (in this order):
+        *  1. If a language-specific override, i.e., [[MW:msg/lang]], is available, use that.
+        *     Note: for the content language, there is no /lang subpage.
+        *  2. Fetch from the static CDB cache.
+        *  3. If available, check the database for fallback language overrides.
         *
-        * @throws MWException
-        * @return string|bool
+        * This process provides a number of guarantees. When changing this code, make sure all
+        * of these guarantees are preserved.
+        *  * If the requested language is *not* the content language, then the CDB cache for that
+        *    specific language will take precedence over the root database page ([[MW:msg]]).
+        *  * Fallbacks will be just that: fallbacks. A fallback language will never be reached if
+        *    the message is available *anywhere* in the language for which it is a fallback.
+        *
+        * @param string $key the message key
+        * @param bool $useDB If true, look for the message in the DB, false
+        *                    to use only the compiled l10n cache.
+        * @param bool|string|object $langcode Code of the language to get the message for.
+        *        - If string and a valid code, will create a standard language object
+        *        - If string but not a valid code, will create a basic language object
+        *        - If boolean and false, create object from the current users language
+        *        - If boolean and true, create object from the wikis content language
+        *        - If language object, use it as given
+        * @param bool $isFullKey specifies whether $key is a two part key
+        *                   "msg/lang".
+        *
+        * @throws MWException when given an invalid key
+        * @return string|bool False if the message doesn't exist, otherwise the message (which can be empty)
         */
        function get( $key, $useDB = true, $langcode = true, $isFullKey = false ) {
-               global $wgLanguageCode, $wgContLang;
+               global $wgContLang;
 
-               if ( is_int( $key ) ) {
-                       // "Non-string key given" exception sometimes happens for numerical
-                       // strings that become ints somewhere on their way here
-                       $key = strval( $key );
-               }
+               $section = new ProfileSection( __METHOD__ );
 
-               if ( !is_string( $key ) ) {
+               if ( is_int( $key ) ) {
+                       // Fix numerical strings that somehow become ints
+                       // on their way here
+                       $key = (string)$key;
+               } elseif ( !is_string( $key ) ) {
                        throw new MWException( 'Non-string key given' );
-               }
-
-               if ( strval( $key ) === '' ) {
-                       # Shortcut: the empty key is always missing
+               } elseif ( $key === '' ) {
+                       // Shortcut: the empty key is always missing
                        return false;
                }
 
-               $lang = wfGetLangObj( $langcode );
-               if ( !$lang ) {
-                       throw new MWException( "Bad lang code $langcode given" );
+               // For full keys, get the language code from the key
+               $pos = strrpos( $key, '/' );
+               if ( $isFullKey && $pos !== false ) {
+                       $langcode = substr( $key, $pos + 1 );
+                       $key = substr( $key, 0, $pos );
                }
 
-               $langcode = $lang->getCode();
-
-               $message = false;
-
-               # Normalise title-case input (with some inlining)
-               $lckey = str_replace( ' ', '_', $key );
+               // Normalise title-case input (with some inlining)
+               $lckey = strtr( $key, ' ', '_');
                if ( ord( $key ) < 128 ) {
                        $lckey[0] = strtolower( $lckey[0] );
                        $uckey = ucfirst( $lckey );
@@ -681,61 +692,135 @@ class MessageCache {
                        $uckey = $wgContLang->ucfirst( $lckey );
                }
 
-               # Try the MediaWiki namespace
-               if ( !$this->mDisable && $useDB ) {
-                       $title = $uckey;
-                       if ( !$isFullKey && ( $langcode != $wgLanguageCode ) ) {
-                               $title .= '/' . $langcode;
-                       }
-                       $message = $this->getMsgFromNamespace( $title, $langcode );
-               }
-
-               # Try the array in the language object
-               if ( $message === false ) {
-                       $message = $lang->getMessage( $lckey );
-                       if ( is_null( $message ) ) {
-                               $message = false;
-                       }
-               }
+               // Loop through each language in the fallback list until we find something useful
+               $lang = wfGetLangObj( $langcode );
+               $message = $this->getMessageFromFallbackChain( $lang, $lckey, $uckey, !$this->mDisable && $useDB );
 
-               # Try the array of another language
+               // If we still have no message, maybe the key was in fact a full key so try that
                if ( $message === false ) {
                        $parts = explode( '/', $lckey );
-                       # We may get calls for things that are http-urls from sidebar
-                       # Let's not load nonexistent languages for those
-                       # They usually have more than one slash.
+                       // We may get calls for things that are http-urls from sidebar
+                       // Let's not load nonexistent languages for those
+                       // They usually have more than one slash.
                        if ( count( $parts ) == 2 && $parts[1] !== '' ) {
                                $message = Language::getMessageFor( $parts[0], $parts[1] );
-                               if ( is_null( $message ) ) {
+                               if ( $message === null ) {
                                        $message = false;
                                }
                        }
                }
 
-               # Is this a custom message? Try the default language in the db...
-               if ( ( $message === false || $message === '-' ) &&
-                       !$this->mDisable && $useDB &&
-                       !$isFullKey && ( $langcode != $wgLanguageCode )
-               ) {
+               // Post-processing if the message exists
+               if( $message !== false ) {
+                       // Fix whitespace
+                       $message = str_replace(
+                               array(
+                                       # Fix for trailing whitespace, removed by textarea
+                                       '&#32;',
+                                       # Fix for NBSP, converted to space by firefox
+                                       '&nbsp;',
+                                       '&#160;',
+                               ),
+                               array(
+                                       ' ',
+                                       "\xc2\xa0",
+                                       "\xc2\xa0"
+                               ),
+                               $message
+                       );
+               }
+
+               return $message;
+       }
+
+       /**
+        * Given a language, try and fetch a message from that language, then the
+        * fallbacks of that language, then the site language, then the fallbacks for the
+        * site language.
+        *
+        * @param Language $lang Requested language
+        * @param string $lckey Lowercase key for the message
+        * @param string $uckey Uppercase key for the message
+        * @param bool $useDB Whether to use the database
+        *
+        * @see MessageCache::get
+        * @return string|bool The message, or false if not found
+        */
+       protected function getMessageFromFallbackChain( $lang, $lckey, $uckey, $useDB ) {
+               global $wgLanguageCode, $wgContLang;
+
+               $langcode = $lang->getCode();
+               $message = false;
+
+               // First try the requested language.
+               if ( $useDB ) {
+                       if ( $langcode === $wgLanguageCode ) {
+                               // Messages created in the content language will not have the /lang extension
+                               $message = $this->getMsgFromNamespace( $uckey, $langcode );
+                       } else {
+                               $message = $this->getMsgFromNamespace( "$uckey/$langcode", $langcode );
+                       }
+               }
+
+               if ( $message !== false ) {
+                       return $message;
+               }
+
+               // Check the CDB cache
+               $message = $lang->getMessage( $lckey );
+               if ( $message !== null ) {
+                       return $message;
+               }
+
+               list( $fallbackChain, $siteFallbackChain ) = Language::getFallbacksIncludingSiteLanguage( $langcode );
+
+               // Next try checking the database for all of the fallback languages of the requested language.
+               if ( $useDB ) {
+                       foreach ( $fallbackChain as $code ) {
+                               if ( $code === $wgLanguageCode ) {
+                                       // Messages created in the content language will not have the /lang extension
+                                       $message = $this->getMsgFromNamespace( $uckey, $code );
+                               } else {
+                                       $message = $this->getMsgFromNamespace( "$uckey/$code", $code );
+                               }
+
+                               if ( $message !== false ) {
+                                       // Found the message.
+                                       return $message;
+                               }
+                       }
+               }
+
+               // Now try checking the site language.
+               if ( $useDB ) {
                        $message = $this->getMsgFromNamespace( $uckey, $wgLanguageCode );
+                       if ( $message !== false ) {
+                               return $message;
+                       }
                }
 
-               # Final fallback
-               if ( $message === false ) {
-                       return false;
+               $message = $wgContLang->getMessage( $lckey );
+               if ( $message !== null ) {
+                       return $message;
                }
 
-               # Fix whitespace
-               $message = strtr( $message,
-                       array(
-                               # Fix for trailing whitespace, removed by textarea
-                               '&#32;' => ' ',
-                               # Fix for NBSP, converted to space by firefox
-                               '&nbsp;' => "\xc2\xa0",
-                               '&#160;' => "\xc2\xa0",
-                       ) );
+               // Finally try the DB for the site language's fallbacks.
+               if ( $useDB ) {
+                       foreach ( $siteFallbackChain as $code ) {
+                               $message = $this->getMsgFromNamespace( "$uckey/$code", $code );
+                               if ( $message === false && $code === $wgLanguageCode ) {
+                                       // Messages created in the content language will not have the /lang extension
+                                       $message = $this->getMsgFromNamespace( $uckey, $code );
+                               }
 
-               return $message;
+                               if ( $message !== false ) {
+                                       // Found the message.
+                                       return $message;
+                               }
+                       }
+               }
+
+               return false;
        }
 
        /**

diff --git a/languages/Language.php b/languages/Language.php
index 9301e54..ea34363 100644 (file)
--- a/languages/Language.php
+++ b/languages/Language.php
@@ -170,6 +170,14 @@ class Language {
                'seconds' => 1,
        );
 
+       /**
+        * Cache for language fallbacks.
+        * @see Language::getFallbacksIncludingSiteLanguage
+        * @since 1.21
+        * @var array
+        */
+       static private $fallbackLanguageCache = array();
+
        /**
         * Get a cached or new language object for a given language code
         * @param $code String
@@ -4053,6 +4061,36 @@ class Language {
                }
        }
 
+       /**
+        * Get the ordered list of fallback languages, ending with the fallback
+        * language chain for the site language.
+        *
+        * @since 1.22
+        * @param string $code Language code
+        * @return array array( fallbacks, site fallbacks )
+        */
+       public static function getFallbacksIncludingSiteLanguage( $code ) {
+               global $wgLanguageCode;
+
+               // Usually, we will only store a tiny number of fallback chains, so we
+               // keep them in static memory.
+               $cacheKey = "{$code}-{$wgLanguageCode}";
+
+               if ( !array_key_exists( $cacheKey, self::$fallbackLanguageCache ) ) {
+                       $fallbacks = self::getFallbacksFor( $code );
+
+                       // Append the site's fallback chain, including the site language itself
+                       $siteFallbacks = self::getFallbacksFor( $wgLanguageCode );
+                       array_unshift( $siteFallbacks, $wgLanguageCode );
+
+                       // Eliminate any languages already included in the chain
+                       $siteFallbacks = array_diff( $siteFallbacks, $fallbacks );
+
+                       self::$fallbackLanguageCache[$cacheKey] = array( $fallbacks, $siteFallbacks );
+               }
+               return self::$fallbackLanguageCache[$cacheKey];
+       }
+
        /**
         * Get all messages for a given language
         * WARNING: this may take a long time. If you just need all message *keys*

diff --git a/tests/phpunit/includes/cache/MessageCacheTest.php b/tests/phpunit/includes/cache/MessageCacheTest.php
new file mode 100644 (file)
index 0000000..c550150
--- /dev/null
+++ b/tests/phpunit/includes/cache/MessageCacheTest.php
@@ -0,0 +1,127 @@
+<?php
+
+/**
+ * @group Database
+ * @group Cache
+ */
+class MessageCacheTest extends MediaWikiLangTestCase {
+
+       protected function setUp() {
+               parent::setUp();
+               $this->configureLanguages();
+               MessageCache::singleton()->enable();
+       }
+
+       /**
+        * Helper function -- setup site language for testing
+        */
+       protected function configureLanguages() {
+               // for the test, we need the content language to be anything but English,
+               // let's choose e.g. German (de)
+               $langCode = 'de';
+               $langObj = Language::factory( $langCode );
+
+               $this->setMwGlobals( array(
+                       'wgLanguageCode' => $langCode,
+                       'wgLang' => $langObj,
+                       'wgContLang' => $langObj,
+               ) );
+       }
+
+       function addDBData() {
+               $this->configureLanguages();
+
+               // Set up messages and fallbacks ab -> ru -> de
+               $this->makePage( 'FallbackLanguageTest-Full', 'ab' );
+               $this->makePage( 'FallbackLanguageTest-Full', 'ru' );
+               $this->makePage( 'FallbackLanguageTest-Full', 'de' );
+
+               // Fallbacks where ab does not exist
+               $this->makePage( 'FallbackLanguageTest-Partial', 'ru' );
+               $this->makePage( 'FallbackLanguageTest-Partial', 'de' );
+
+               // Fallback to the content language
+               $this->makePage( 'FallbackLanguageTest-ContLang', 'de' );
+
+               // Add customizations for an existing message.
+               $this->makePage( 'sunday', 'ru' );
+
+               // Full key tests -- always want russian
+               $this->makePage( 'MessageCacheTest-FullKeyTest', 'ab' );
+               $this->makePage( 'MessageCacheTest-FullKeyTest', 'ru' );
+
+               // In content language -- get base if no derivative
+               $this->makePage( 'FallbackLanguageTest-NoDervContLang', 'de', 'de/none', false );
+       }
+
+       /**
+        * Helper function for addDBData -- adds a simple page to the database
+        *
+        * @param string $title Title of page to be created
+        * @param string $lang  Language and content of the created page
+        * @param string|null $content Content of the created page, or null for a generic string
+        * @param bool $createSubPage Set to false if a root page should be created
+        */
+       protected function makePage( $title, $lang, $content = null, $createSubPage = true ) {
+               global $wgContLang;
+
+               if ( $content === null ) {
+                       $content = $lang;
+               }
+               if ( $lang !== $wgContLang->getCode() || $createSubPage ) {
+                       $title = "$title/$lang";
+               }
+
+               $title = Title::newFromText( $title, NS_MEDIAWIKI );
+               $wikiPage = new WikiPage( $title );
+               $contentHandler = ContentHandler::makeContent( $content, $title );
+               $wikiPage->doEditContent( $contentHandler, "$lang translation test case" );
+       }
+
+       /**
+        * Test message fallbacks, bug #1495
+        *
+        * @dataProvider provideMessagesForFallback
+        */
+       function testMessageFallbacks( $message, $lang, $expectedContent ) {
+               $result = MessageCache::singleton()->get( $message, true, $lang );
+               $this->assertEquals( $expectedContent, $result, "Message fallback failed." );
+       }
+
+       function provideMessagesForFallback() {
+               return array(
+                       array( 'FallbackLanguageTest-Full', 'ab', 'ab' ),
+                       array( 'FallbackLanguageTest-Partial', 'ab', 'ru' ),
+                       array( 'FallbackLanguageTest-ContLang', 'ab', 'de' ),
+                       array( 'FallbackLanguageTest-None', 'ab', false ),
+
+                       // Existing message with customizations on the fallbacks
+                       array( 'sunday', 'ab', 'амҽыш' ),
+
+                       // bug 46579
+                       array( 'FallbackLanguageTest-NoDervContLang', 'de', 'de/none' ),
+                       // UI language different from content language should only use de/none as last option
+                       array( 'FallbackLanguageTest-NoDervContLang', 'fit', 'de/none' ),
+               );
+       }
+
+       /**
+        * There's a fallback case where the message key is given as fully qualified -- this
+        * should ignore the passed $lang and use the language from the key
+        *
+        * @dataProvider provideMessagesForFullKeys
+        */
+       function testFullKeyBehaviour( $message, $lang, $expectedContent ) {
+               $result = MessageCache::singleton()->get( $message, true, $lang, true );
+               $this->assertEquals( $expectedContent, $result, "Full key message fallback failed." );
+       }
+
+       function provideMessagesForFullKeys() {
+               return array(
+                       array( 'MessageCacheTest-FullKeyTest/ru', 'ru', 'ru' ),
+                       array( 'MessageCacheTest-FullKeyTest/ru', 'ab', 'ru' ),
+                       array( 'MessageCacheTest-FullKeyTest/ru/foo', 'ru', false ),
+               );
+       }
+
+}