From 67b3cdc0047f43bc9ec6a7c367ddb3c3cdf5d7e4 Mon Sep 17 00:00:00 2001 From: Lucas Werkmeister Date: Thu, 16 May 2019 11:42:05 +0200 Subject: [PATCH] Add action=query&meta=languageinfo API module MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This API module can be used to get information about all the languages supported by this MediaWiki installation. Since parts of this information, such as the fallback chain, are expensive to retrieve if the localization cache is not populated, we apply continuation if the request is taking too long (suggested by Anomie in T217239#4994301); we don’t expect this to happen in Wikimedia production, though. Bug: T74153 Bug: T220415 Change-Id: Ic66991cd85ed4439a47bfb1412dbe24c23bd9819 --- autoload.php | 1 + includes/api/ApiQuery.php | 1 + includes/api/ApiQueryLanguageinfo.php | 245 ++++++++++++++++++ includes/api/i18n/en.json | 16 ++ includes/api/i18n/qqq.json | 15 ++ .../includes/api/ApiQueryLanguageinfoTest.php | 175 +++++++++++++ 6 files changed, 453 insertions(+) create mode 100644 includes/api/ApiQueryLanguageinfo.php create mode 100644 tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php diff --git a/autoload.php b/autoload.php index 275d20e886..0d317f8b65 100644 --- a/autoload.php +++ b/autoload.php @@ -113,6 +113,7 @@ $wgAutoloadLocalClasses = [ 'ApiQueryInfo' => __DIR__ . '/includes/api/ApiQueryInfo.php', 'ApiQueryLangBacklinks' => __DIR__ . '/includes/api/ApiQueryLangBacklinks.php', 'ApiQueryLangLinks' => __DIR__ . '/includes/api/ApiQueryLangLinks.php', + 'ApiQueryLanguageinfo' => __DIR__ . '/includes/api/ApiQueryLanguageinfo.php', 'ApiQueryLinks' => __DIR__ . '/includes/api/ApiQueryLinks.php', 'ApiQueryLogEvents' => __DIR__ . '/includes/api/ApiQueryLogEvents.php', 'ApiQueryMyStashedFiles' => __DIR__ . '/includes/api/ApiQueryMyStashedFiles.php', diff --git a/includes/api/ApiQuery.php b/includes/api/ApiQuery.php index ae6b1a1dbc..0c89c1e99a 100644 --- a/includes/api/ApiQuery.php +++ b/includes/api/ApiQuery.php @@ -115,6 +115,7 @@ class ApiQuery extends ApiBase { 'userinfo' => ApiQueryUserInfo::class, 'filerepoinfo' => ApiQueryFileRepoInfo::class, 'tokens' => ApiQueryTokens::class, + 'languageinfo' => ApiQueryLanguageinfo::class, ]; /** diff --git a/includes/api/ApiQueryLanguageinfo.php b/includes/api/ApiQueryLanguageinfo.php new file mode 100644 index 0000000000..72b59b006b --- /dev/null +++ b/includes/api/ApiQueryLanguageinfo.php @@ -0,0 +1,245 @@ +microtimeFunction = $microtimeFunction; + } + + /** @return float */ + private function microtime() { + if ( $this->microtimeFunction ) { + return ( $this->microtimeFunction )(); + } else { + return microtime( true ); + } + } + + public function execute() { + $endTime = $this->microtime() + self::MAX_EXECUTE_SECONDS; + + $props = array_flip( $this->getParameter( 'prop' ) ); + $includeCode = isset( $props['code'] ); + $includeBcp47 = isset( $props['bcp47'] ); + $includeDir = isset( $props['dir'] ); + $includeAutonym = isset( $props['autonym'] ); + $includeName = isset( $props['name'] ); + $includeFallbacks = isset( $props['fallbacks'] ); + $includeVariants = isset( $props['variants'] ); + + $targetLanguageCode = $this->getLanguage()->getCode(); + $include = 'all'; + + $availableLanguageCodes = array_keys( Language::fetchLanguageNames( + // MediaWiki and extensions may return different sets of language codes + // when asked for language names in different languages; + // asking for English language names is most likely to give us the full set, + // even though we may not need those at all + 'en', + $include + ) ); + $selectedLanguageCodes = $this->getParameter( 'code' ); + if ( $selectedLanguageCodes === [ '*' ] ) { + $languageCodes = $availableLanguageCodes; + } else { + $languageCodes = array_values( array_intersect( + $availableLanguageCodes, + $selectedLanguageCodes + ) ); + $unrecognizedCodes = array_values( array_diff( + $selectedLanguageCodes, + $availableLanguageCodes + ) ); + if ( $unrecognizedCodes !== [] ) { + $this->addWarning( [ + 'apiwarn-unrecognizedvalues', + $this->encodeParamName( 'code' ), + Message::listParam( $unrecognizedCodes, 'comma' ), + count( $unrecognizedCodes ), + ] ); + } + } + // order of $languageCodes is guaranteed by Language::fetchLanguageNames() + // and preserved by array_values() + array_intersect() + + $continue = $this->getParameter( 'continue' ); + if ( $continue === null ) { + $continue = reset( $languageCodes ); + } + + $result = $this->getResult(); + $rootPath = [ + $this->getQuery()->getModuleName(), + $this->getModuleName(), + ]; + $result->addArrayType( $rootPath, 'assoc' ); + + foreach ( $languageCodes as $languageCode ) { + if ( $languageCode < $continue ) { + continue; + } + + $now = $this->microtime(); + if ( $now >= $endTime ) { + $this->setContinueEnumParameter( 'continue', $languageCode ); + break; + } + + $info = []; + ApiResult::setArrayType( $info, 'assoc' ); + + if ( $includeCode ) { + $info['code'] = $languageCode; + } + + if ( $includeBcp47 ) { + $bcp47 = LanguageCode::bcp47( $languageCode ); + $info['bcp47'] = $bcp47; + } + + if ( $includeDir ) { + $dir = Language::factory( $languageCode )->getDir(); + $info['dir'] = $dir; + } + + if ( $includeAutonym ) { + $autonym = Language::fetchLanguageName( + $languageCode, + Language::AS_AUTONYMS, + $include + ); + $info['autonym'] = $autonym; + } + + if ( $includeName ) { + $name = Language::fetchLanguageName( + $languageCode, + $targetLanguageCode, + $include + ); + $info['name'] = $name; + } + + if ( $includeFallbacks ) { + $fallbacks = Language::getFallbacksFor( + $languageCode, + // allow users to distinguish between implicit and explicit 'en' fallbacks + Language::STRICT_FALLBACKS + ); + ApiResult::setIndexedTagName( $fallbacks, 'fb' ); + $info['fallbacks'] = $fallbacks; + } + + if ( $includeVariants ) { + $variants = Language::factory( $languageCode )->getVariants(); + ApiResult::setIndexedTagName( $variants, 'var' ); + $info['variants'] = $variants; + } + + $fit = $result->addValue( $rootPath, $languageCode, $info ); + if ( !$fit ) { + $this->setContinueEnumParameter( 'continue', $languageCode ); + break; + } + } + } + + public function getCacheMode( $params ) { + return 'public'; + } + + public function getAllowedParams() { + return [ + 'prop' => [ + self::PARAM_DFLT => 'code', + self::PARAM_ISMULTI => true, + self::PARAM_TYPE => [ + 'code', + 'bcp47', + 'dir', + 'autonym', + 'name', + 'fallbacks', + 'variants', + ], + self::PARAM_HELP_MSG_PER_VALUE => [], + ], + 'code' => [ + self::PARAM_DFLT => '*', + self::PARAM_ISMULTI => true, + ], + 'continue' => [ + self::PARAM_HELP_MSG => 'api-help-param-continue', + ], + ]; + } + + protected function getExamplesMessages() { + $pathUrl = 'action=' . $this->getQuery()->getModuleName() . + '&meta=' . $this->getModuleName(); + $pathMsg = $this->getModulePath(); + $prefix = $this->getModulePrefix(); + + return [ + "$pathUrl" + => "apihelp-$pathMsg-example-simple", + "$pathUrl&{$prefix}prop=autonym|name&lang=de" + => "apihelp-$pathMsg-example-autonym-name-de", + "$pathUrl&{$prefix}prop=fallbacks|variants&{$prefix}code=oc" + => "apihelp-$pathMsg-example-fallbacks-variants-oc", + "$pathUrl&{$prefix}prop=bcp47|dir" + => "apihelp-$pathMsg-example-bcp47-dir", + ]; + } + +} diff --git a/includes/api/i18n/en.json b/includes/api/i18n/en.json index aded1f9cba..1280889914 100644 --- a/includes/api/i18n/en.json +++ b/includes/api/i18n/en.json @@ -972,6 +972,22 @@ "apihelp-query+langlinks-param-inlanguagecode": "Language code for localised language names.", "apihelp-query+langlinks-example-simple": "Get interlanguage links from the page Main Page.", + "apihelp-query+languageinfo-summary": "Return information about available languages.", + "apihelp-query+languageinfo-extended-description": "[[mw:API:Query#Continuing queries|Continuation]] may be applied if retrieving the information takes too long for one request.", + "apihelp-query+languageinfo-param-prop": "Which information to get for each language.", + "apihelp-query+languageinfo-paramvalue-prop-code": "The language code. (This code is MediaWiki-specific, though there are overlaps with other standards.)", + "apihelp-query+languageinfo-paramvalue-prop-bcp47": "The BCP-47 language code.", + "apihelp-query+languageinfo-paramvalue-prop-dir": "The writing direction of the language (either ltr or rtl).", + "apihelp-query+languageinfo-paramvalue-prop-autonym": "The autonym of the language, that is, the name in that language.", + "apihelp-query+languageinfo-paramvalue-prop-name": "The name of the language in the language specified by the lilang parameter, with language fallbacks applied if necessary.", + "apihelp-query+languageinfo-paramvalue-prop-fallbacks": "The language codes of the fallback languages configured for this language. The implicit final fallback to 'en' is not included (but some languages may fall back to 'en' explicitly).", + "apihelp-query+languageinfo-paramvalue-prop-variants": "The language codes of the variants supported by this language.", + "apihelp-query+languageinfo-param-code": "Language codes of the languages that should be returned, or * for all languages.", + "apihelp-query+languageinfo-example-simple": "Get the language codes of all supported languages.", + "apihelp-query+languageinfo-example-autonym-name-de": "Get the autonyms and German names of all supported languages.", + "apihelp-query+languageinfo-example-fallbacks-variants-oc": "Get the fallback languages and variants of Occitan.", + "apihelp-query+languageinfo-example-bcp47-dir": "Get the BCP-47 language code and direction of all supported languages.", + "apihelp-query+links-summary": "Returns all links from the given pages.", "apihelp-query+links-param-namespace": "Show links in these namespaces only.", "apihelp-query+links-param-limit": "How many links to return.", diff --git a/includes/api/i18n/qqq.json b/includes/api/i18n/qqq.json index 06ac6a732f..36c49536ca 100644 --- a/includes/api/i18n/qqq.json +++ b/includes/api/i18n/qqq.json @@ -910,6 +910,21 @@ "apihelp-query+langlinks-param-dir": "{{doc-apihelp-param|query+langlinks|dir}}", "apihelp-query+langlinks-param-inlanguagecode": "{{doc-apihelp-param|query+langlinks|inlanguagecode}}", "apihelp-query+langlinks-example-simple": "{{doc-apihelp-example|query+langlinks}}", + "apihelp-query+languageinfo-summary": "{{doc-apihelp-summary|query+languageinfo}}", + "apihelp-query+languageinfo-extended-description": "{{doc-apihelp-extended-description|query+languageinfo}}", + "apihelp-query+languageinfo-param-prop": "{{doc-apihelp-param|query+languageinfo|prop|paramvalues=1}}", + "apihelp-query+languageinfo-paramvalue-prop-code": "{{doc-apihelp-paramvalue|query+languageinfo|prop|code}}", + "apihelp-query+languageinfo-paramvalue-prop-bcp47": "{{doc-apihelp-paramvalue|query+languageinfo|prop|bcp47}}", + "apihelp-query+languageinfo-paramvalue-prop-dir": "{{doc-apihelp-paramvalue|query+languageinfo|prop|dir}}", + "apihelp-query+languageinfo-paramvalue-prop-autonym": "{{doc-apihelp-paramvalue|query+languageinfo|prop|autonym}}", + "apihelp-query+languageinfo-paramvalue-prop-name": "{{doc-apihelp-paramvalue|query+languageinfo|prop|name}}", + "apihelp-query+languageinfo-paramvalue-prop-fallbacks": "{{doc-apihelp-paramvalue|query+languageinfo|prop|fallbacks}}", + "apihelp-query+languageinfo-paramvalue-prop-variants": "{{doc-apihelp-paramvalue|query+languageinfo|prop|variants}}", + "apihelp-query+languageinfo-param-code": "{{doc-apihelp-param|query+languageinfo|code}}", + "apihelp-query+languageinfo-example-simple": "{{doc-apihelp-example|query+languageinfo}}", + "apihelp-query+languageinfo-example-autonym-name-de": "{{doc-apihelp-example|query+languageinfo}}", + "apihelp-query+languageinfo-example-fallbacks-variants-oc": "{{doc-apihelp-example|query+languageinfo}}", + "apihelp-query+languageinfo-example-bcp47-dir": "{{doc-apihelp-example|query+languageinfo}}", "apihelp-query+links-summary": "{{doc-apihelp-summary|query+links}}", "apihelp-query+links-param-namespace": "{{doc-apihelp-param|query+links|namespace}}", "apihelp-query+links-param-limit": "{{doc-apihelp-param|query+links|limit}}", diff --git a/tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php b/tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php new file mode 100644 index 0000000000..f20a0613bb --- /dev/null +++ b/tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php @@ -0,0 +1,175 @@ +setTemporaryHook( + 'LanguageGetTranslatedLanguageNames', + function ( array &$names, $code ) { + switch ( $code ) { + case 'en': + $names['sh'] = 'Serbo-Croatian'; + $names['qtp'] = 'a custom language code MediaWiki knows nothing about'; + break; + case 'pt': + $names['de'] = 'alemão'; + break; + } + } + ); + } + + private function doQuery( array $params, $microtimeFunction = null ): array { + $params += [ + 'action' => 'query', + 'meta' => 'languageinfo', + 'uselang' => 'en', + ]; + + if ( $microtimeFunction !== null ) { + // hook into the module manager to override the factory function + // so we can call the constructor with the custom $microtimeFunction + $this->setTemporaryHook( + 'ApiQuery::moduleManager', + function ( ApiModuleManager $moduleManager ) use ( $microtimeFunction ) { + $moduleManager->addModule( + 'languageinfo', + 'meta', + ApiQueryLanguageinfo::class, + function ( $parent, $name ) use ( $microtimeFunction ) { + return new ApiQueryLanguageinfo( + $parent, + $name, + $microtimeFunction + ); + } + ); + } + ); + } + + $res = $this->doApiRequest( $params ); + + $this->assertArrayNotHasKey( 'warnings', $res[0] ); + + return [ $res[0]['query']['languageinfo'], $res[0]['continue'] ?? null ]; + } + + public function testAllPropsForSingleLanguage() { + list( $response, $continue ) = $this->doQuery( [ + 'liprop' => 'code|bcp47|dir|autonym|name|fallbacks|variants', + 'licode' => 'sh', + ] ); + + $this->assertArrayEquals( [ + 'sh' => [ + 'code' => 'sh', + 'bcp47' => 'sh', + 'autonym' => 'srpskohrvatski / српскохрватски', + 'name' => 'Serbo-Croatian', + 'fallbacks' => [ 'bs', 'sr-el', 'hr' ], + 'dir' => 'ltr', + 'variants' => [ 'sh' ], + ], + ], $response ); + } + + public function testAllPropsForSingleCustomLanguage() { + list( $response, $continue ) = $this->doQuery( [ + 'liprop' => 'code|bcp47|dir|autonym|name|fallbacks|variants', + 'licode' => 'qtp', // reserved for local use by ISO 639; registered in setUp() + ] ); + + $this->assertArrayEquals( [ + 'qtp' => [ + 'code' => 'qtp', + 'bcp47' => 'qtp', + 'autonym' => '', + 'name' => 'a custom language code MediaWiki knows nothing about', + 'fallbacks' => [], + 'dir' => 'ltr', + 'variants' => [ 'qtp' ], + ], + ], $response ); + } + + public function testNameInOtherLanguageForSingleLanguage() { + list( $response, $continue ) = $this->doQuery( [ + 'liprop' => 'name', + 'licode' => 'de', + 'uselang' => 'pt', + ] ); + + $this->assertArrayEquals( [ 'de' => [ 'name' => 'alemão' ] ], $response ); + } + + public function testContinuationNecessary() { + $time = 0; + $microtimeFunction = function () use ( &$time ) { + return $time += 0.75; + }; + + list( $response, $continue ) = $this->doQuery( [], $microtimeFunction ); + + $this->assertCount( 2, $response ); + $this->assertArrayHasKey( 'licontinue', $continue ); + } + + public function testContinuationNotNecessary() { + $time = 0; + $microtimeFunction = function () use ( &$time ) { + return $time += 1.5; + }; + + list( $response, $continue ) = $this->doQuery( [ + 'licode' => 'de', + ], $microtimeFunction ); + + $this->assertNull( $continue ); + } + + public function testContinuationInAlphabeticalOrderNotParameterOrder() { + $time = 0; + $microtimeFunction = function () use ( &$time ) { + return $time += 0.75; + }; + $params = [ 'licode' => 'en|ru|zh|de|yue' ]; + + list( $response, $continue ) = $this->doQuery( $params, $microtimeFunction ); + + $this->assertCount( 2, $response ); + $this->assertArrayHasKey( 'licontinue', $continue ); + $this->assertSame( [ 'de', 'en' ], array_keys( $response ) ); + + $time = 0; + $params = $continue + $params; + list( $response, $continue ) = $this->doQuery( $params, $microtimeFunction ); + + $this->assertCount( 2, $response ); + $this->assertArrayHasKey( 'licontinue', $continue ); + $this->assertSame( [ 'ru', 'yue' ], array_keys( $response ) ); + + $time = 0; + $params = $continue + $params; + list( $response, $continue ) = $this->doQuery( $params, $microtimeFunction ); + + $this->assertCount( 1, $response ); + $this->assertNull( $continue ); + $this->assertSame( [ 'zh' ], array_keys( $response ) ); + } + + public function testResponseHasModulePathEvenIfEmpty() { + list( $response, $continue ) = $this->doQuery( [ 'licode' => '' ] ); + $this->assertEmpty( $response ); + // the real test is that $res[0]['query']['languageinfo'] in doQuery() didn’t fail + } + +} -- 2.20.1