From 467739271886574d47394bdb19da9c0fc772bcab Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Thu, 14 Mar 2019 19:46:19 +0000 Subject: [PATCH] Remove wgSitesCacheFile and rebuildSitesCache.php (unused) This seems to have been intended as optimization for SiteStore, but was never used as far as I can tell. Instead, SiteStore is already cached via LocalServerCache (APC). Keep the FileBasedSiteLookup class for one release cycle in case third parties not indexed by MediaWiki Codesearch are using it. == History * 2013: Report of high memcached usage by Wikibase via SiteStore. * 2014: Lazy-load the data in Wikibase (d3f2e99cb6). * 2014: Implement the file-based cache (via T47532 and 90f6efc360fd5). * 2015: Use local-server cache (APC), per T58602. The file-based code was never used. The related task marked invalid (T47532). Change-Id: I8e2d9edcf3880149f824cc3de37793ca57435b49 --- RELEASE-NOTES-1.33 | 6 + autoload.php | 2 - docs/sitescache.txt | 42 ------ includes/DefaultSettings.php | 12 -- includes/ServiceWiring.php | 12 +- includes/site/FileBasedSiteLookup.php | 9 +- includes/site/SitesCacheFileBuilder.php | 113 --------------- maintenance/rebuildSitesCache.php | 68 --------- .../includes/site/FileBasedSiteLookupTest.php | 103 ------------- .../site/SitesCacheFileBuilderTest.php | 137 ------------------ 10 files changed, 13 insertions(+), 491 deletions(-) delete mode 100644 docs/sitescache.txt delete mode 100644 includes/site/SitesCacheFileBuilder.php delete mode 100644 maintenance/rebuildSitesCache.php delete mode 100644 tests/phpunit/includes/site/FileBasedSiteLookupTest.php delete mode 100644 tests/phpunit/includes/site/SitesCacheFileBuilderTest.php diff --git a/RELEASE-NOTES-1.33 b/RELEASE-NOTES-1.33 index 27b1c71a05..d3001f3e59 100644 --- a/RELEASE-NOTES-1.33 +++ b/RELEASE-NOTES-1.33 @@ -41,6 +41,10 @@ production. it unset should treat it as being MIGRATION_NEW. * $wgAuth – This old setting, deprecated in 1.27, has been removed as part of the removal of AuthPlugin. +* $wgSitesCacheFile has been removed. It was introduced in 1.25 to allow sites + to configure a file in which to cache the SiteStore database table. + This was never used. SiteStore already caches its information by default + using BagOStuff (e.g. Memcached or APC). === New features in 1.33 === * (T96041) __EXPECTUNUSEDCATEGORY__ on a category page causes the category @@ -347,6 +351,8 @@ because of Phabricator reports. check block behaviour. * The api-feature-usage log channel now has log context. The text message is deprecated and will be removed in the future. +* The FileBasedSiteLookup class has been deprecated. For a cacheable SiteLookup + implementation, use CachingSiteStore instead. === Other changes in 1.33 === * (T201747) Html::openElement() warns if given an element name with a space diff --git a/autoload.php b/autoload.php index bb4de22cce..1c7c34e098 100644 --- a/autoload.php +++ b/autoload.php @@ -1197,7 +1197,6 @@ $wgAutoloadLocalClasses = [ 'RebuildLocalisationCache' => __DIR__ . '/maintenance/rebuildLocalisationCache.php', 'RebuildMessages' => __DIR__ . '/maintenance/rebuildmessages.php', 'RebuildRecentchanges' => __DIR__ . '/maintenance/rebuildrecentchanges.php', - 'RebuildSitesCache' => __DIR__ . '/maintenance/rebuildSitesCache.php', 'RebuildTextIndex' => __DIR__ . '/maintenance/rebuildtextindex.php', 'RecentChange' => __DIR__ . '/includes/changes/RecentChange.php', 'RecentChangesUpdateJob' => __DIR__ . '/includes/jobqueue/jobs/RecentChangesUpdateJob.php', @@ -1335,7 +1334,6 @@ $wgAutoloadLocalClasses = [ 'SiteStatsInit' => __DIR__ . '/includes/SiteStatsInit.php', 'SiteStatsUpdate' => __DIR__ . '/includes/deferred/SiteStatsUpdate.php', 'SiteStore' => __DIR__ . '/includes/site/SiteStore.php', - 'SitesCacheFileBuilder' => __DIR__ . '/includes/site/SitesCacheFileBuilder.php', 'Skin' => __DIR__ . '/includes/skins/Skin.php', 'SkinApi' => __DIR__ . '/includes/skins/SkinApi.php', 'SkinApiTemplate' => __DIR__ . '/includes/skins/SkinApiTemplate.php', diff --git a/docs/sitescache.txt b/docs/sitescache.txt deleted file mode 100644 index 13bf371d20..0000000000 --- a/docs/sitescache.txt +++ /dev/null @@ -1,42 +0,0 @@ -MediaWiki's SiteStore can be cached and stored in a flat file, -in a json format. If the SiteStore is frequently accessed, the -file cache may provide a performance benefit over a database -store, even with memcached in front of it. - -Configuration: - -File-based caching can be enabled by setting $wgSitesCacheFile -to the file path of the cache file. - -The file can then be generated with the rebuildSitesCache.php -maintenance script. - -Format: - -In the sites cache file, sites are listed in a key-value -map, with the key being the site's global id (e.g. "enwiki") -and a key-value map as the value. The site list is wrapped -with in a "sites" key. - -Example: - -"sites": { - "aawiktionary": { - "globalid": "aawiktionary", - "type": "mediawiki", - "group": "wiktionary", - "source": "local", - "language": "aa", - "localids": [], - "config": [], - "data": { - "paths": { - "file_path": "http:\/\/aa.wiktionary.org\/w\/$1", - "page_path": "http:\/\/aa.wiktionary.org\/wiki\/$1" - } - }, - "forward": false, - "internalid": 2666, - "identifiers": [] - } -} diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 4ec82edaf2..a215af57a7 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -3992,18 +3992,6 @@ $wgInterwikiFallbackSite = 'wiki'; /** @} */ # end of Interwiki caching settings. -/** - * @name SiteStore caching settings. - * @{ - */ - -/** - * Specify the file location for the Sites json cache file. - */ -$wgSitesCacheFile = false; - -/** @} */ # end of SiteStore caching settings. - /** * If local interwikis are set up which allow redirects, * set this regexp to restrict URLs which will be displayed diff --git a/includes/ServiceWiring.php b/includes/ServiceWiring.php index e5f891e312..12e782d34f 100644 --- a/includes/ServiceWiring.php +++ b/includes/ServiceWiring.php @@ -489,14 +489,10 @@ return [ }, 'SiteLookup' => function ( MediaWikiServices $services ) : SiteLookup { - $cacheFile = $services->getMainConfig()->get( 'SitesCacheFile' ); - - if ( $cacheFile !== false ) { - return new FileBasedSiteLookup( $cacheFile ); - } else { - // Use the default SiteStore as the SiteLookup implementation for now - return $services->getSiteStore(); - } + // Use SiteStore as the SiteLookup as well. This was originally separated + // to allow for a cacheable read-only interface (using FileBasedSiteLookup), + // but this was never used. SiteStore has caching (see below). + return $services->getSiteStore(); }, 'SiteStore' => function ( MediaWikiServices $services ) : SiteStore { diff --git a/includes/site/FileBasedSiteLookup.php b/includes/site/FileBasedSiteLookup.php index c168a47ffb..174d667651 100644 --- a/includes/site/FileBasedSiteLookup.php +++ b/includes/site/FileBasedSiteLookup.php @@ -21,14 +21,10 @@ */ /** - * Provides a file-based cache of a SiteStore. The sites are stored in - * a json file. (see docs/sitescache.txt regarding format) - * - * The cache can be built with the rebuildSitesCache.php maintenance script, - * and a MediaWiki instance can be setup to use this by setting the - * 'wgSitesCacheFile' configuration to the cache file location. + * Provides a file-based cache of a SiteStore, using a json file. * * @since 1.25 + * @deprecated since 1.33 Use CachingSiteStore instead. */ class FileBasedSiteLookup implements SiteLookup { @@ -46,6 +42,7 @@ class FileBasedSiteLookup implements SiteLookup { * @param string $cacheFile */ public function __construct( $cacheFile ) { + wfDeprecated( __CLASS__, '1.33' ); $this->cacheFile = $cacheFile; } diff --git a/includes/site/SitesCacheFileBuilder.php b/includes/site/SitesCacheFileBuilder.php deleted file mode 100644 index f0d6ce1026..0000000000 --- a/includes/site/SitesCacheFileBuilder.php +++ /dev/null @@ -1,113 +0,0 @@ -siteLookup = $siteLookup; - $this->cacheFile = $cacheFile; - } - - public function build() { - $this->sites = $this->siteLookup->getSites(); - $this->cacheSites( $this->sites->getArrayCopy() ); - } - - /** - * @param Site[] $sites - * - * @throws MWException if in manualRecache mode - * @return bool - */ - private function cacheSites( array $sites ) { - $sitesArray = []; - - foreach ( $sites as $site ) { - $globalId = $site->getGlobalId(); - $sitesArray[$globalId] = $this->getSiteAsArray( $site ); - } - - $json = json_encode( [ - 'sites' => $sitesArray - ] ); - - $result = file_put_contents( $this->cacheFile, $json ); - - return $result !== false; - } - - /** - * @param Site $site - * - * @return array - */ - private function getSiteAsArray( Site $site ) { - $siteEntry = unserialize( $site->serialize() ); - $siteIdentifiers = $this->buildLocalIdentifiers( $site ); - $identifiersArray = []; - - foreach ( $siteIdentifiers as $identifier ) { - $identifiersArray[] = $identifier; - } - - $siteEntry['identifiers'] = $identifiersArray; - - return $siteEntry; - } - - /** - * @param Site $site - * - * @return array Site local identifiers - */ - private function buildLocalIdentifiers( Site $site ) { - $localIds = []; - - foreach ( $site->getLocalIds() as $idType => $ids ) { - foreach ( $ids as $id ) { - $localIds[] = [ - 'type' => $idType, - 'key' => $id - ]; - } - } - - return $localIds; - } - -} diff --git a/maintenance/rebuildSitesCache.php b/maintenance/rebuildSitesCache.php deleted file mode 100644 index 41fd863661..0000000000 --- a/maintenance/rebuildSitesCache.php +++ /dev/null @@ -1,68 +0,0 @@ -addDescription( 'Cache sites as json for file-based lookup.' ); - $this->addOption( 'file', 'File to output the json to', false, true ); - } - - public function execute() { - $sitesCacheFileBuilder = new SitesCacheFileBuilder( - \MediaWiki\MediaWikiServices::getInstance()->getSiteLookup(), - $this->getCacheFile() - ); - - $sitesCacheFileBuilder->build(); - } - - /** - * @return string - */ - private function getCacheFile() { - if ( $this->hasOption( 'file' ) ) { - $jsonFile = $this->getOption( 'file' ); - } else { - $jsonFile = $this->getConfig()->get( 'SitesCacheFile' ); - - if ( $jsonFile === false ) { - $this->fatalError( 'Error: No file set in configuration for SitesCacheFile.' ); - } - } - - return $jsonFile; - } - -} - -$maintClass = RebuildSitesCache::class; -require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/tests/phpunit/includes/site/FileBasedSiteLookupTest.php b/tests/phpunit/includes/site/FileBasedSiteLookupTest.php deleted file mode 100644 index 69e0e38936..0000000000 --- a/tests/phpunit/includes/site/FileBasedSiteLookupTest.php +++ /dev/null @@ -1,103 +0,0 @@ - - */ -class FileBasedSiteLookupTest extends PHPUnit\Framework\TestCase { - - use MediaWikiCoversValidator; - - protected function setUp() { - $this->cacheFile = $this->getCacheFile(); - } - - protected function tearDown() { - unlink( $this->cacheFile ); - } - - public function testGetSites() { - $sites = $this->getSites(); - $cacheBuilder = $this->newSitesCacheFileBuilder( $sites ); - $cacheBuilder->build(); - - $cache = new FileBasedSiteLookup( $this->cacheFile ); - $this->assertEquals( $sites, $cache->getSites() ); - } - - public function testGetSite() { - $sites = $this->getSites(); - $cacheBuilder = $this->newSitesCacheFileBuilder( $sites ); - $cacheBuilder->build(); - - $cache = new FileBasedSiteLookup( $this->cacheFile ); - - $this->assertEquals( $sites->getSite( 'enwiktionary' ), $cache->getSite( 'enwiktionary' ) ); - } - - private function newSitesCacheFileBuilder( SiteList $sites ) { - return new SitesCacheFileBuilder( - $this->getSiteLookup( $sites ), - $this->cacheFile - ); - } - - private function getSiteLookup( SiteList $sites ) { - $siteLookup = $this->getMockBuilder( SiteLookup::class ) - ->disableOriginalConstructor() - ->getMock(); - - $siteLookup->expects( $this->any() ) - ->method( 'getSites' ) - ->will( $this->returnValue( $sites ) ); - - return $siteLookup; - } - - private function getSites() { - $sites = []; - - $site = new Site(); - $site->setGlobalId( 'foobar' ); - $sites[] = $site; - - $site = new MediaWikiSite(); - $site->setGlobalId( 'enwiktionary' ); - $site->setGroup( 'wiktionary' ); - $site->setLanguageCode( 'en' ); - $site->addNavigationId( 'enwiktionary' ); - $site->setPath( MediaWikiSite::PATH_PAGE, "https://en.wiktionary.org/wiki/$1" ); - $site->setPath( MediaWikiSite::PATH_FILE, "https://en.wiktionary.org/w/$1" ); - $sites[] = $site; - - return new SiteList( $sites ); - } - - private function getCacheFile() { - return tempnam( sys_get_temp_dir(), 'mw-test-sitelist' ); - } - -} diff --git a/tests/phpunit/includes/site/SitesCacheFileBuilderTest.php b/tests/phpunit/includes/site/SitesCacheFileBuilderTest.php deleted file mode 100644 index 8c84ce57c0..0000000000 --- a/tests/phpunit/includes/site/SitesCacheFileBuilderTest.php +++ /dev/null @@ -1,137 +0,0 @@ - - */ -class SitesCacheFileBuilderTest extends PHPUnit\Framework\TestCase { - - use MediaWikiCoversValidator; - - protected function setUp() { - $this->cacheFile = $this->getCacheFile(); - } - - protected function tearDown() { - unlink( $this->cacheFile ); - } - - public function testBuild() { - $cacheBuilder = $this->newSitesCacheFileBuilder( $this->getSites() ); - $cacheBuilder->build(); - - $contents = file_get_contents( $this->cacheFile ); - $this->assertEquals( json_encode( $this->getExpectedData() ), $contents ); - } - - private function getExpectedData() { - return [ - 'sites' => [ - 'foobar' => [ - 'globalid' => 'foobar', - 'type' => 'unknown', - 'group' => 'none', - 'source' => 'local', - 'language' => null, - 'localids' => [], - 'config' => [], - 'data' => [], - 'forward' => false, - 'internalid' => null, - 'identifiers' => [] - ], - 'enwiktionary' => [ - 'globalid' => 'enwiktionary', - 'type' => 'mediawiki', - 'group' => 'wiktionary', - 'source' => 'local', - 'language' => 'en', - 'localids' => [ - 'equivalent' => [ 'enwiktionary' ] - ], - 'config' => [], - 'data' => [ - 'paths' => [ - 'page_path' => 'https://en.wiktionary.org/wiki/$1', - 'file_path' => 'https://en.wiktionary.org/w/$1' - ] - ], - 'forward' => false, - 'internalid' => null, - 'identifiers' => [ - [ - 'type' => 'equivalent', - 'key' => 'enwiktionary' - ] - ] - ] - ] - ]; - } - - private function newSitesCacheFileBuilder( SiteList $sites ) { - return new SitesCacheFileBuilder( - $this->getSiteLookup( $sites ), - $this->cacheFile - ); - } - - private function getSiteLookup( SiteList $sites ) { - $siteLookup = $this->getMockBuilder( SiteLookup::class ) - ->disableOriginalConstructor() - ->getMock(); - - $siteLookup->expects( $this->any() ) - ->method( 'getSites' ) - ->will( $this->returnValue( $sites ) ); - - return $siteLookup; - } - - private function getSites() { - $sites = []; - - $site = new Site(); - $site->setGlobalId( 'foobar' ); - $sites[] = $site; - - $site = new MediaWikiSite(); - $site->setGlobalId( 'enwiktionary' ); - $site->setGroup( 'wiktionary' ); - $site->setLanguageCode( 'en' ); - $site->addNavigationId( 'enwiktionary' ); - $site->setPath( MediaWikiSite::PATH_PAGE, "https://en.wiktionary.org/wiki/$1" ); - $site->setPath( MediaWikiSite::PATH_FILE, "https://en.wiktionary.org/w/$1" ); - $sites[] = $site; - - return new SiteList( $sites ); - } - - private function getCacheFile() { - return tempnam( sys_get_temp_dir(), 'mw-test-sitelist' ); - } - -} -- 2.20.1