From: Tim Starling Date: Fri, 1 Nov 2013 04:59:17 +0000 (+1100) Subject: Move the DeferredUpdates hierarchy to its own directory X-Git-Tag: 1.31.0-rc.0~18304 X-Git-Url: https://git.cyclocoop.org/%7B%24admin_url%7Dmembres/supprimer.php?a=commitdiff_plain;h=64d0dc06a036c38574877fe72f940c27cf172a90;p=lhc%2Fweb%2Fwiklou.git Move the DeferredUpdates hierarchy to its own directory The grouping makes at least as much sense as job/, and certainly makes more sense than cache/. With directories named after base classes, it is fairly easy to tell what should go where. The grouping of DeferredUpdates, DataUpdate and CallableUpdate would surely be uncontroversial. The move of SearchUpdate out of search/ demonstrates the conflict between arrangement by module versus arrangement by type, which is the most difficult design question here. I think arrangement by type is more consistent with e.g. the arrangement of the core root, i.e. tests/, resources/, maintenance/, etc. where a given feature will have its files split up into a mostly type-based hierarchy. I also tidied up AutoLoader.php by moving includes/content to the correct location, sorted alphabetically by subdirectory. Verified with AutoLoaderTest. Change-Id: Ib369411d0caca38e72978084aa57348f1b892ed0 --- diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index dbba50007c..da854388c1 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -65,9 +65,6 @@ $wgAutoloadLocalClasses = array( 'Cookie' => 'includes/Cookie.php', 'CookieJar' => 'includes/Cookie.php', 'CurlHttpRequest' => 'includes/HttpFunctions.php', - 'DeferrableUpdate' => 'includes/DeferredUpdates.php', - 'DeferredUpdates' => 'includes/DeferredUpdates.php', - 'MWCallableUpdate' => 'includes/CallableUpdate.php', 'DeprecatedGlobal' => 'includes/DeprecatedGlobal.php', 'DerivativeRequest' => 'includes/WebRequest.php', 'DiffHistoryBlob' => 'includes/HistoryBlob.php', @@ -155,8 +152,6 @@ $wgAutoloadLocalClasses = array( 'Licenses' => 'includes/Licenses.php', 'Linker' => 'includes/Linker.php', 'LinkFilter' => 'includes/LinkFilter.php', - 'LinksUpdate' => 'includes/LinksUpdate.php', - 'LinksDeletionUpdate' => 'includes/LinksUpdate.php', 'LocalisationCache' => 'includes/cache/LocalisationCache.php', 'LocalisationCache_BulkLoad' => 'includes/cache/LocalisationCache.php', 'MagicWord' => 'includes/MagicWord.php', @@ -212,14 +207,11 @@ $wgAutoloadLocalClasses = array( 'RevisionList' => 'includes/RevisionList.php', 'RSSFeed' => 'includes/Feed.php', 'Sanitizer' => 'includes/Sanitizer.php', - 'DataUpdate' => 'includes/DataUpdate.php', - 'SqlDataUpdate' => 'includes/SqlDataUpdate.php', 'ScopedCallback' => 'includes/ScopedCallback.php', 'ScopedPHPTimeout' => 'includes/ScopedPHPTimeout.php', 'SiteConfiguration' => 'includes/SiteConfiguration.php', 'SiteStats' => 'includes/SiteStats.php', 'SiteStatsInit' => 'includes/SiteStats.php', - 'SiteStatsUpdate' => 'includes/SiteStats.php', 'Skin' => 'includes/Skin.php', 'SkinTemplate' => 'includes/SkinTemplate.php', 'SpecialCreateAccount' => 'includes/SpecialPage.php', @@ -261,7 +253,6 @@ $wgAutoloadLocalClasses = array( 'UserCache' => 'includes/cache/UserCache.php', 'UserMailer' => 'includes/UserMailer.php', 'UserRightsProxy' => 'includes/UserRightsProxy.php', - 'ViewCountUpdate' => 'includes/ViewCountUpdate.php', 'WantedQueryPage' => 'includes/QueryPage.php', 'WatchedItem' => 'includes/WatchedItem.php', 'WebRequest' => 'includes/WebRequest.php', @@ -288,21 +279,6 @@ $wgAutoloadLocalClasses = array( 'ZipDirectoryReader' => 'includes/ZipDirectoryReader.php', 'ZipDirectoryReaderError' => 'includes/ZipDirectoryReader.php', - # content handler - 'AbstractContent' => 'includes/content/AbstractContent.php', - 'ContentHandler' => 'includes/content/ContentHandler.php', - 'Content' => 'includes/content/Content.php', - 'CssContentHandler' => 'includes/content/CssContentHandler.php', - 'CssContent' => 'includes/content/CssContent.php', - 'JavaScriptContentHandler' => 'includes/content/JavaScriptContentHandler.php', - 'JavaScriptContent' => 'includes/content/JavaScriptContent.php', - 'MessageContent' => 'includes/content/MessageContent.php', - 'MWContentSerializationException' => 'includes/content/ContentHandler.php', - 'TextContentHandler' => 'includes/content/TextContentHandler.php', - 'TextContent' => 'includes/content/TextContent.php', - 'WikitextContentHandler' => 'includes/content/WikitextContentHandler.php', - 'WikitextContent' => 'includes/content/WikitextContent.php', - # includes/actions 'CachedAction' => 'includes/actions/CachedAction.php', 'CreditsAction' => 'includes/actions/CreditsAction.php', @@ -440,7 +416,6 @@ $wgAutoloadLocalClasses = array( 'FileDependency' => 'includes/cache/CacheDependency.php', 'GenderCache' => 'includes/cache/GenderCache.php', 'GlobalDependency' => 'includes/cache/CacheDependency.php', - 'HTMLCacheUpdate' => 'includes/cache/HTMLCacheUpdate.php', 'HTMLFileCache' => 'includes/cache/HTMLFileCache.php', 'LinkBatch' => 'includes/cache/LinkBatch.php', 'LinkCache' => 'includes/cache/LinkCache.php', @@ -448,7 +423,6 @@ $wgAutoloadLocalClasses = array( 'ObjectFileCache' => 'includes/cache/ObjectFileCache.php', 'ProcessCacheLRU' => 'includes/cache/ProcessCacheLRU.php', 'ResourceFileCache' => 'includes/cache/ResourceFileCache.php', - 'SquidUpdate' => 'includes/cache/SquidUpdate.php', 'TitleDependency' => 'includes/cache/CacheDependency.php', 'TitleListDependency' => 'includes/cache/CacheDependency.php', @@ -463,6 +437,21 @@ $wgAutoloadLocalClasses = array( 'RedisConnectionPool' => 'includes/clientpool/RedisConnectionPool.php', 'RedisConnRef' => 'includes/clientpool/RedisConnectionPool.php', + # includes/content + 'AbstractContent' => 'includes/content/AbstractContent.php', + 'ContentHandler' => 'includes/content/ContentHandler.php', + 'Content' => 'includes/content/Content.php', + 'CssContentHandler' => 'includes/content/CssContentHandler.php', + 'CssContent' => 'includes/content/CssContent.php', + 'JavaScriptContentHandler' => 'includes/content/JavaScriptContentHandler.php', + 'JavaScriptContent' => 'includes/content/JavaScriptContent.php', + 'MessageContent' => 'includes/content/MessageContent.php', + 'MWContentSerializationException' => 'includes/content/ContentHandler.php', + 'TextContentHandler' => 'includes/content/TextContentHandler.php', + 'TextContent' => 'includes/content/TextContent.php', + 'WikitextContentHandler' => 'includes/content/WikitextContentHandler.php', + 'WikitextContent' => 'includes/content/WikitextContent.php', + # includes/context 'ContextSource' => 'includes/context/ContextSource.php', 'DerivativeContext' => 'includes/context/DerivativeContext.php', @@ -530,6 +519,20 @@ $wgAutoloadLocalClasses = array( # includes/debug 'MWDebug' => 'includes/debug/Debug.php', + # includes/deferred + 'DataUpdate' => 'includes/deferred/DataUpdate.php', + 'DeferrableUpdate' => 'includes/deferred/DeferredUpdates.php', + 'DeferredUpdates' => 'includes/deferred/DeferredUpdates.php', + 'HTMLCacheUpdate' => 'includes/deferred/HTMLCacheUpdate.php', + 'LinksDeletionUpdate' => 'includes/deferred/LinksUpdate.php', + 'LinksUpdate' => 'includes/deferred/LinksUpdate.php', + 'MWCallableUpdate' => 'includes/deferred/CallableUpdate.php', + 'SearchUpdate' => 'includes/deferred/SearchUpdate.php', + 'SiteStatsUpdate' => 'includes/deferred/SiteStatsUpdate.php', + 'SqlDataUpdate' => 'includes/deferred/SqlDataUpdate.php', + 'SquidUpdate' => 'includes/deferred/SquidUpdate.php', + 'ViewCountUpdate' => 'includes/deferred/ViewCountUpdate.php', + # includes/diff '_DiffEngine' => 'includes/diff/DairikiDiff.php', '_DiffOp' => 'includes/diff/DairikiDiff.php', @@ -909,7 +912,6 @@ $wgAutoloadLocalClasses = array( 'SearchResultSet' => 'includes/search/SearchEngine.php', 'SearchResultTooMany' => 'includes/search/SearchEngine.php', 'SearchSqlite' => 'includes/search/SearchSqlite.php', - 'SearchUpdate' => 'includes/search/SearchUpdate.php', 'SqliteSearchResultSet' => 'includes/search/SearchSqlite.php', 'SqlSearchResultSet' => 'includes/search/SearchEngine.php', diff --git a/includes/CallableUpdate.php b/includes/CallableUpdate.php deleted file mode 100644 index 6eb5541393..0000000000 --- a/includes/CallableUpdate.php +++ /dev/null @@ -1,30 +0,0 @@ -callback = $callback; - } - - /** - * Run the update - */ - public function doUpdate() { - call_user_func( $this->callback ); - } - -} diff --git a/includes/DataUpdate.php b/includes/DataUpdate.php deleted file mode 100644 index 7b9ac2811d..0000000000 --- a/includes/DataUpdate.php +++ /dev/null @@ -1,126 +0,0 @@ -beginTransaction(); - $open_transactions[] = $update; - } - - // do work - foreach ( $updates as $update ) { - $update->doUpdate(); - } - - // commit transactions - while ( count( $open_transactions ) > 0 ) { - $trans = array_pop( $open_transactions ); - $trans->commitTransaction(); - } - } catch ( Exception $ex ) { - $exception = $ex; - wfDebug( "Caught exception, will rethrow after rollback: " . $ex->getMessage() ); - } - - // rollback remaining transactions - while ( count( $open_transactions ) > 0 ) { - $trans = array_pop( $open_transactions ); - $trans->rollbackTransaction(); - } - - if ( $exception ) { - throw $exception; // rethrow after cleanup - } - } - -} diff --git a/includes/DeferredUpdates.php b/includes/DeferredUpdates.php deleted file mode 100644 index c385f138e9..0000000000 --- a/includes/DeferredUpdates.php +++ /dev/null @@ -1,129 +0,0 @@ -doUpdate(); - - if ( $doCommit && $dbw->trxLevel() ) { - $dbw->commit( __METHOD__, 'flush' ); - } - } catch ( MWException $e ) { - // We don't want exceptions thrown during deferred updates to - // be reported to the user since the output is already sent. - // Instead we just log them. - if ( !$e instanceof ErrorPageError ) { - MWExceptionHandler::logException( $e ); - } - } - } - - self::clearPendingUpdates(); - wfProfileOut( __METHOD__ ); - } - - /** - * Clear all pending updates without performing them. Generally, you don't - * want or need to call this. Unit tests need it though. - */ - public static function clearPendingUpdates() { - global $wgDeferredUpdateList; - $wgDeferredUpdateList = self::$updates = array(); - } -} diff --git a/includes/LinksUpdate.php b/includes/LinksUpdate.php deleted file mode 100644 index fdd0e3c116..0000000000 --- a/includes/LinksUpdate.php +++ /dev/null @@ -1,892 +0,0 @@ -mTitle = $title; - $this->mId = $title->getArticleID(); - - if ( !$this->mId ) { - throw new MWException( "The Title object did not provide an article ID. Perhaps the page doesn't exist?" ); - } - - $this->mParserOutput = $parserOutput; - - $this->mLinks = $parserOutput->getLinks(); - $this->mImages = $parserOutput->getImages(); - $this->mTemplates = $parserOutput->getTemplates(); - $this->mExternals = $parserOutput->getExternalLinks(); - $this->mCategories = $parserOutput->getCategories(); - $this->mProperties = $parserOutput->getProperties(); - $this->mInterwikis = $parserOutput->getInterwikiLinks(); - - # Convert the format of the interlanguage links - # I didn't want to change it in the ParserOutput, because that array is passed all - # the way back to the skin, so either a skin API break would be required, or an - # inefficient back-conversion. - $ill = $parserOutput->getLanguageLinks(); - $this->mInterlangs = array(); - foreach ( $ill as $link ) { - list( $key, $title ) = explode( ':', $link, 2 ); - $this->mInterlangs[$key] = $title; - } - - foreach ( $this->mCategories as &$sortkey ) { - # If the sortkey is longer then 255 bytes, - # it truncated by DB, and then doesn't get - # matched when comparing existing vs current - # categories, causing bug 25254. - # Also. substr behaves weird when given "". - if ( $sortkey !== '' ) { - $sortkey = substr( $sortkey, 0, 255 ); - } - } - - $this->mRecursive = $recursive; - - wfRunHooks( 'LinksUpdateConstructed', array( &$this ) ); - } - - /** - * Update link tables with outgoing links from an updated article - */ - public function doUpdate() { - wfRunHooks( 'LinksUpdate', array( &$this ) ); - $this->doIncrementalUpdate(); - wfRunHooks( 'LinksUpdateComplete', array( &$this ) ); - } - - protected function doIncrementalUpdate() { - wfProfileIn( __METHOD__ ); - - # Page links - $existing = $this->getExistingLinks(); - $this->linkDeletions = $this->getLinkDeletions( $existing ); - $this->linkInsertions = $this->getLinkInsertions( $existing ); - $this->incrTableUpdate( 'pagelinks', 'pl', $this->linkDeletions, $this->linkInsertions ); - - # Image links - $existing = $this->getExistingImages(); - - $imageDeletes = $this->getImageDeletions( $existing ); - $this->incrTableUpdate( 'imagelinks', 'il', $imageDeletes, - $this->getImageInsertions( $existing ) ); - - # Invalidate all image description pages which had links added or removed - $imageUpdates = $imageDeletes + array_diff_key( $this->mImages, $existing ); - $this->invalidateImageDescriptions( $imageUpdates ); - - # External links - $existing = $this->getExistingExternals(); - $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ), - $this->getExternalInsertions( $existing ) ); - - # Language links - $existing = $this->getExistingInterlangs(); - $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ), - $this->getInterlangInsertions( $existing ) ); - - # Inline interwiki links - $existing = $this->getExistingInterwikis(); - $this->incrTableUpdate( 'iwlinks', 'iwl', $this->getInterwikiDeletions( $existing ), - $this->getInterwikiInsertions( $existing ) ); - - # Template links - $existing = $this->getExistingTemplates(); - $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ), - $this->getTemplateInsertions( $existing ) ); - - # Category links - $existing = $this->getExistingCategories(); - - $categoryDeletes = $this->getCategoryDeletions( $existing ); - - $this->incrTableUpdate( 'categorylinks', 'cl', $categoryDeletes, - $this->getCategoryInsertions( $existing ) ); - - # Invalidate all categories which were added, deleted or changed (set symmetric difference) - $categoryInserts = array_diff_assoc( $this->mCategories, $existing ); - $categoryUpdates = $categoryInserts + $categoryDeletes; - $this->invalidateCategories( $categoryUpdates ); - $this->updateCategoryCounts( $categoryInserts, $categoryDeletes ); - - # Page properties - $existing = $this->getExistingProperties(); - - $propertiesDeletes = $this->getPropertyDeletions( $existing ); - - $this->incrTableUpdate( 'page_props', 'pp', $propertiesDeletes, - $this->getPropertyInsertions( $existing ) ); - - # Invalidate the necessary pages - $changed = $propertiesDeletes + array_diff_assoc( $this->mProperties, $existing ); - $this->invalidateProperties( $changed ); - - # Refresh links of all pages including this page - # This will be in a separate transaction - if ( $this->mRecursive ) { - $this->queueRecursiveJobs(); - } - - wfProfileOut( __METHOD__ ); - } - - /** - * Queue recursive jobs for this page - * - * Which means do LinksUpdate on all templates - * that include the current page, using the job queue. - */ - function queueRecursiveJobs() { - self::queueRecursiveJobsForTable( $this->mTitle, 'templatelinks' ); - } - - /** - * Queue a RefreshLinks job for any table. - * - * @param Title $title Title to do job for - * @param String $table Table to use (e.g. 'templatelinks') - */ - public static function queueRecursiveJobsForTable( Title $title, $table ) { - wfProfileIn( __METHOD__ ); - if ( $title->getBacklinkCache()->hasLinks( $table ) ) { - $job = new RefreshLinksJob2( - $title, - array( - 'table' => $table, - ) + Job::newRootJobParams( // "overall" refresh links job info - "refreshlinks:{$table}:{$title->getPrefixedText()}" - ) - ); - JobQueueGroup::singleton()->push( $job ); - JobQueueGroup::singleton()->deduplicateRootJob( $job ); - } - wfProfileOut( __METHOD__ ); - } - - /** - * @param $cats - */ - function invalidateCategories( $cats ) { - $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) ); - } - - /** - * Update all the appropriate counts in the category table. - * @param array $added associative array of category name => sort key - * @param array $deleted associative array of category name => sort key - */ - function updateCategoryCounts( $added, $deleted ) { - $a = WikiPage::factory( $this->mTitle ); - $a->updateCategoryCounts( - array_keys( $added ), array_keys( $deleted ) - ); - } - - /** - * @param $images - */ - function invalidateImageDescriptions( $images ) { - $this->invalidatePages( NS_FILE, array_keys( $images ) ); - } - - /** - * Update a table by doing a delete query then an insert query - * @param $table - * @param $prefix - * @param $deletions - * @param $insertions - */ - function incrTableUpdate( $table, $prefix, $deletions, $insertions ) { - if ( $table == 'page_props' ) { - $fromField = 'pp_page'; - } else { - $fromField = "{$prefix}_from"; - } - $where = array( $fromField => $this->mId ); - if ( $table == 'pagelinks' || $table == 'templatelinks' || $table == 'iwlinks' ) { - if ( $table == 'iwlinks' ) { - $baseKey = 'iwl_prefix'; - } else { - $baseKey = "{$prefix}_namespace"; - } - $clause = $this->mDb->makeWhereFrom2d( $deletions, $baseKey, "{$prefix}_title" ); - if ( $clause ) { - $where[] = $clause; - } else { - $where = false; - } - } else { - if ( $table == 'langlinks' ) { - $toField = 'll_lang'; - } elseif ( $table == 'page_props' ) { - $toField = 'pp_propname'; - } else { - $toField = $prefix . '_to'; - } - if ( count( $deletions ) ) { - $where[] = "$toField IN (" . $this->mDb->makeList( array_keys( $deletions ) ) . ')'; - } else { - $where = false; - } - } - if ( $where ) { - $this->mDb->delete( $table, $where, __METHOD__ ); - } - if ( count( $insertions ) ) { - $this->mDb->insert( $table, $insertions, __METHOD__, 'IGNORE' ); - wfRunHooks( 'LinksUpdateAfterInsert', array( $this, $table, $insertions ) ); - } - } - - /** - * Get an array of pagelinks insertions for passing to the DB - * Skips the titles specified by the 2-D array $existing - * @param $existing array - * @return array - */ - private function getLinkInsertions( $existing = array() ) { - $arr = array(); - foreach ( $this->mLinks as $ns => $dbkeys ) { - $diffs = isset( $existing[$ns] ) - ? array_diff_key( $dbkeys, $existing[$ns] ) - : $dbkeys; - foreach ( $diffs as $dbk => $id ) { - $arr[] = array( - 'pl_from' => $this->mId, - 'pl_namespace' => $ns, - 'pl_title' => $dbk - ); - } - } - return $arr; - } - - /** - * Get an array of template insertions. Like getLinkInsertions() - * @param $existing array - * @return array - */ - private function getTemplateInsertions( $existing = array() ) { - $arr = array(); - foreach ( $this->mTemplates as $ns => $dbkeys ) { - $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys; - foreach ( $diffs as $dbk => $id ) { - $arr[] = array( - 'tl_from' => $this->mId, - 'tl_namespace' => $ns, - 'tl_title' => $dbk - ); - } - } - return $arr; - } - - /** - * Get an array of image insertions - * Skips the names specified in $existing - * @param $existing array - * @return array - */ - private function getImageInsertions( $existing = array() ) { - $arr = array(); - $diffs = array_diff_key( $this->mImages, $existing ); - foreach ( $diffs as $iname => $dummy ) { - $arr[] = array( - 'il_from' => $this->mId, - 'il_to' => $iname - ); - } - return $arr; - } - - /** - * Get an array of externallinks insertions. Skips the names specified in $existing - * @param $existing array - * @return array - */ - private function getExternalInsertions( $existing = array() ) { - $arr = array(); - $diffs = array_diff_key( $this->mExternals, $existing ); - foreach ( $diffs as $url => $dummy ) { - foreach ( wfMakeUrlIndexes( $url ) as $index ) { - $arr[] = array( - 'el_from' => $this->mId, - 'el_to' => $url, - 'el_index' => $index, - ); - } - } - return $arr; - } - - /** - * Get an array of category insertions - * - * @param array $existing mapping existing category names to sort keys. If both - * match a link in $this, the link will be omitted from the output - * - * @return array - */ - private function getCategoryInsertions( $existing = array() ) { - global $wgContLang, $wgCategoryCollation; - $diffs = array_diff_assoc( $this->mCategories, $existing ); - $arr = array(); - foreach ( $diffs as $name => $prefix ) { - $nt = Title::makeTitleSafe( NS_CATEGORY, $name ); - $wgContLang->findVariantLink( $name, $nt, true ); - - if ( $this->mTitle->getNamespace() == NS_CATEGORY ) { - $type = 'subcat'; - } elseif ( $this->mTitle->getNamespace() == NS_FILE ) { - $type = 'file'; - } else { - $type = 'page'; - } - - # Treat custom sortkeys as a prefix, so that if multiple - # things are forced to sort as '*' or something, they'll - # sort properly in the category rather than in page_id - # order or such. - $sortkey = Collation::singleton()->getSortKey( - $this->mTitle->getCategorySortkey( $prefix ) ); - - $arr[] = array( - 'cl_from' => $this->mId, - 'cl_to' => $name, - 'cl_sortkey' => $sortkey, - 'cl_timestamp' => $this->mDb->timestamp(), - 'cl_sortkey_prefix' => $prefix, - 'cl_collation' => $wgCategoryCollation, - 'cl_type' => $type, - ); - } - return $arr; - } - - /** - * Get an array of interlanguage link insertions - * - * @param array $existing mapping existing language codes to titles - * - * @return array - */ - private function getInterlangInsertions( $existing = array() ) { - $diffs = array_diff_assoc( $this->mInterlangs, $existing ); - $arr = array(); - foreach ( $diffs as $lang => $title ) { - $arr[] = array( - 'll_from' => $this->mId, - 'll_lang' => $lang, - 'll_title' => $title - ); - } - return $arr; - } - - /** - * Get an array of page property insertions - * @param $existing array - * @return array - */ - function getPropertyInsertions( $existing = array() ) { - $diffs = array_diff_assoc( $this->mProperties, $existing ); - $arr = array(); - foreach ( $diffs as $name => $value ) { - $arr[] = array( - 'pp_page' => $this->mId, - 'pp_propname' => $name, - 'pp_value' => $value, - ); - } - return $arr; - } - - /** - * Get an array of interwiki insertions for passing to the DB - * Skips the titles specified by the 2-D array $existing - * @param $existing array - * @return array - */ - private function getInterwikiInsertions( $existing = array() ) { - $arr = array(); - foreach ( $this->mInterwikis as $prefix => $dbkeys ) { - $diffs = isset( $existing[$prefix] ) ? array_diff_key( $dbkeys, $existing[$prefix] ) : $dbkeys; - foreach ( $diffs as $dbk => $id ) { - $arr[] = array( - 'iwl_from' => $this->mId, - 'iwl_prefix' => $prefix, - 'iwl_title' => $dbk - ); - } - } - return $arr; - } - - /** - * Given an array of existing links, returns those links which are not in $this - * and thus should be deleted. - * @param $existing array - * @return array - */ - private function getLinkDeletions( $existing ) { - $del = array(); - foreach ( $existing as $ns => $dbkeys ) { - if ( isset( $this->mLinks[$ns] ) ) { - $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] ); - } else { - $del[$ns] = $existing[$ns]; - } - } - return $del; - } - - /** - * Given an array of existing templates, returns those templates which are not in $this - * and thus should be deleted. - * @param $existing array - * @return array - */ - private function getTemplateDeletions( $existing ) { - $del = array(); - foreach ( $existing as $ns => $dbkeys ) { - if ( isset( $this->mTemplates[$ns] ) ) { - $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] ); - } else { - $del[$ns] = $existing[$ns]; - } - } - return $del; - } - - /** - * Given an array of existing images, returns those images which are not in $this - * and thus should be deleted. - * @param $existing array - * @return array - */ - private function getImageDeletions( $existing ) { - return array_diff_key( $existing, $this->mImages ); - } - - /** - * Given an array of existing external links, returns those links which are not - * in $this and thus should be deleted. - * @param $existing array - * @return array - */ - private function getExternalDeletions( $existing ) { - return array_diff_key( $existing, $this->mExternals ); - } - - /** - * Given an array of existing categories, returns those categories which are not in $this - * and thus should be deleted. - * @param $existing array - * @return array - */ - private function getCategoryDeletions( $existing ) { - return array_diff_assoc( $existing, $this->mCategories ); - } - - /** - * Given an array of existing interlanguage links, returns those links which are not - * in $this and thus should be deleted. - * @param $existing array - * @return array - */ - private function getInterlangDeletions( $existing ) { - return array_diff_assoc( $existing, $this->mInterlangs ); - } - - /** - * Get array of properties which should be deleted. - * @param $existing array - * @return array - */ - function getPropertyDeletions( $existing ) { - return array_diff_assoc( $existing, $this->mProperties ); - } - - /** - * Given an array of existing interwiki links, returns those links which are not in $this - * and thus should be deleted. - * @param $existing array - * @return array - */ - private function getInterwikiDeletions( $existing ) { - $del = array(); - foreach ( $existing as $prefix => $dbkeys ) { - if ( isset( $this->mInterwikis[$prefix] ) ) { - $del[$prefix] = array_diff_key( $existing[$prefix], $this->mInterwikis[$prefix] ); - } else { - $del[$prefix] = $existing[$prefix]; - } - } - return $del; - } - - /** - * Get an array of existing links, as a 2-D array - * - * @return array - */ - private function getExistingLinks() { - $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ), - array( 'pl_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - if ( !isset( $arr[$row->pl_namespace] ) ) { - $arr[$row->pl_namespace] = array(); - } - $arr[$row->pl_namespace][$row->pl_title] = 1; - } - return $arr; - } - - /** - * Get an array of existing templates, as a 2-D array - * - * @return array - */ - private function getExistingTemplates() { - $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ), - array( 'tl_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - if ( !isset( $arr[$row->tl_namespace] ) ) { - $arr[$row->tl_namespace] = array(); - } - $arr[$row->tl_namespace][$row->tl_title] = 1; - } - return $arr; - } - - /** - * Get an array of existing images, image names in the keys - * - * @return array - */ - private function getExistingImages() { - $res = $this->mDb->select( 'imagelinks', array( 'il_to' ), - array( 'il_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - $arr[$row->il_to] = 1; - } - return $arr; - } - - /** - * Get an array of existing external links, URLs in the keys - * - * @return array - */ - private function getExistingExternals() { - $res = $this->mDb->select( 'externallinks', array( 'el_to' ), - array( 'el_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - $arr[$row->el_to] = 1; - } - return $arr; - } - - /** - * Get an array of existing categories, with the name in the key and sort key in the value. - * - * @return array - */ - private function getExistingCategories() { - $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey_prefix' ), - array( 'cl_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - $arr[$row->cl_to] = $row->cl_sortkey_prefix; - } - return $arr; - } - - /** - * Get an array of existing interlanguage links, with the language code in the key and the - * title in the value. - * - * @return array - */ - private function getExistingInterlangs() { - $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ), - array( 'll_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - $arr[$row->ll_lang] = $row->ll_title; - } - return $arr; - } - - /** - * Get an array of existing inline interwiki links, as a 2-D array - * @return array (prefix => array(dbkey => 1)) - */ - protected function getExistingInterwikis() { - $res = $this->mDb->select( 'iwlinks', array( 'iwl_prefix', 'iwl_title' ), - array( 'iwl_from' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - if ( !isset( $arr[$row->iwl_prefix] ) ) { - $arr[$row->iwl_prefix] = array(); - } - $arr[$row->iwl_prefix][$row->iwl_title] = 1; - } - return $arr; - } - - /** - * Get an array of existing categories, with the name in the key and sort key in the value. - * - * @return array - */ - private function getExistingProperties() { - $res = $this->mDb->select( 'page_props', array( 'pp_propname', 'pp_value' ), - array( 'pp_page' => $this->mId ), __METHOD__, $this->mOptions ); - $arr = array(); - foreach ( $res as $row ) { - $arr[$row->pp_propname] = $row->pp_value; - } - return $arr; - } - - /** - * Return the title object of the page being updated - * @return Title - */ - public function getTitle() { - return $this->mTitle; - } - - /** - * Returns parser output - * @since 1.19 - * @return ParserOutput - */ - public function getParserOutput() { - return $this->mParserOutput; - } - - /** - * Return the list of images used as generated by the parser - * @return array - */ - public function getImages() { - return $this->mImages; - } - - /** - * Invalidate any necessary link lists related to page property changes - * @param $changed - */ - private function invalidateProperties( $changed ) { - global $wgPagePropLinkInvalidations; - - foreach ( $changed as $name => $value ) { - if ( isset( $wgPagePropLinkInvalidations[$name] ) ) { - $inv = $wgPagePropLinkInvalidations[$name]; - if ( !is_array( $inv ) ) { - $inv = array( $inv ); - } - foreach ( $inv as $table ) { - $update = new HTMLCacheUpdate( $this->mTitle, $table ); - $update->doUpdate(); - } - } - } - } - - /** - * Fetch page links added by this LinksUpdate. Only available after the update is complete. - * @since 1.22 - * @return null|array of Titles - */ - public function getAddedLinks() { - if ( $this->linkInsertions === null ) { - return null; - } - $result = array(); - foreach ( $this->linkInsertions as $insertion ) { - $result[] = Title::makeTitle( $insertion[ 'pl_namespace' ], $insertion[ 'pl_title' ] ); - } - return $result; - } - - /** - * Fetch page links removed by this LinksUpdate. Only available after the update is complete. - * @since 1.22 - * @return null|array of Titles - */ - public function getRemovedLinks() { - if ( $this->linkDeletions === null ) { - return null; - } - $result = array(); - foreach ( $this->linkDeletions as $ns => $titles ) { - foreach ( $titles as $title => $unused ) { - $result[] = Title::makeTitle( $ns, $title ); - } - } - return $result; - } -} - -/** - * Update object handling the cleanup of links tables after a page was deleted. - **/ -class LinksDeletionUpdate extends SqlDataUpdate { - - protected $mPage; //!< WikiPage the wikipage that was deleted - - /** - * Constructor - * - * @param $page WikiPage Page we are updating - * @throws MWException - */ - function __construct( WikiPage $page ) { - parent::__construct( false ); // no implicit transaction - - $this->mPage = $page; - - if ( !$page->exists() ) { - throw new MWException( "Page ID not known, perhaps the page doesn't exist?" ); - } - } - - /** - * Do some database updates after deletion - */ - public function doUpdate() { - $title = $this->mPage->getTitle(); - $id = $this->mPage->getId(); - - # Delete restrictions for it - $this->mDb->delete( 'page_restrictions', array( 'pr_page' => $id ), __METHOD__ ); - - # Fix category table counts - $cats = array(); - $res = $this->mDb->select( 'categorylinks', 'cl_to', array( 'cl_from' => $id ), __METHOD__ ); - - foreach ( $res as $row ) { - $cats[] = $row->cl_to; - } - - $this->mPage->updateCategoryCounts( array(), $cats ); - - # If using cascading deletes, we can skip some explicit deletes - if ( !$this->mDb->cascadingDeletes() ) { - # Delete outgoing links - $this->mDb->delete( 'pagelinks', array( 'pl_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'imagelinks', array( 'il_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'categorylinks', array( 'cl_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'templatelinks', array( 'tl_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'externallinks', array( 'el_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'langlinks', array( 'll_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'iwlinks', array( 'iwl_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'redirect', array( 'rd_from' => $id ), __METHOD__ ); - $this->mDb->delete( 'page_props', array( 'pp_page' => $id ), __METHOD__ ); - } - - # If using cleanup triggers, we can skip some manual deletes - if ( !$this->mDb->cleanupTriggers() ) { - # Clean up recentchanges entries... - $this->mDb->delete( 'recentchanges', - array( 'rc_type != ' . RC_LOG, - 'rc_namespace' => $title->getNamespace(), - 'rc_title' => $title->getDBkey() ), - __METHOD__ ); - $this->mDb->delete( 'recentchanges', - array( 'rc_type != ' . RC_LOG, 'rc_cur_id' => $id ), - __METHOD__ ); - } - } - - /** - * Update all the appropriate counts in the category table. - * @param array $added associative array of category name => sort key - * @param array $deleted associative array of category name => sort key - */ - function updateCategoryCounts( $added, $deleted ) { - $a = WikiPage::factory( $this->mTitle ); - $a->updateCategoryCounts( - array_keys( $added ), array_keys( $deleted ) - ); - } -} diff --git a/includes/SiteStats.php b/includes/SiteStats.php index 355993c6ec..0df6d907cc 100644 --- a/includes/SiteStats.php +++ b/includes/SiteStats.php @@ -251,231 +251,6 @@ class SiteStats { } } -/** - * Class for handling updates to the site_stats table - */ -class SiteStatsUpdate implements DeferrableUpdate { - protected $views = 0; - protected $edits = 0; - protected $pages = 0; - protected $articles = 0; - protected $users = 0; - protected $images = 0; - - // @todo deprecate this constructor - function __construct( $views, $edits, $good, $pages = 0, $users = 0 ) { - $this->views = $views; - $this->edits = $edits; - $this->articles = $good; - $this->pages = $pages; - $this->users = $users; - } - - /** - * @param $deltas Array - * @return SiteStatsUpdate - */ - public static function factory( array $deltas ) { - $update = new self( 0, 0, 0 ); - - $fields = array( 'views', 'edits', 'pages', 'articles', 'users', 'images' ); - foreach ( $fields as $field ) { - if ( isset( $deltas[$field] ) && $deltas[$field] ) { - $update->$field = $deltas[$field]; - } - } - - return $update; - } - - public function doUpdate() { - global $wgSiteStatsAsyncFactor; - - $rate = $wgSiteStatsAsyncFactor; // convenience - // If set to do so, only do actual DB updates 1 every $rate times. - // The other times, just update "pending delta" values in memcached. - if ( $rate && ( $rate < 0 || mt_rand( 0, $rate - 1 ) != 0 ) ) { - $this->doUpdatePendingDeltas(); - } else { - // Need a separate transaction because this a global lock - wfGetDB( DB_MASTER )->onTransactionIdle( array( $this, 'tryDBUpdateInternal' ) ); - } - } - - /** - * Do not call this outside of SiteStatsUpdate - * - * @return void - */ - public function tryDBUpdateInternal() { - global $wgSiteStatsAsyncFactor; - - $dbw = wfGetDB( DB_MASTER ); - $lockKey = wfMemcKey( 'site_stats' ); // prepend wiki ID - if ( $wgSiteStatsAsyncFactor ) { - // Lock the table so we don't have double DB/memcached updates - if ( !$dbw->lockIsFree( $lockKey, __METHOD__ ) - || !$dbw->lock( $lockKey, __METHOD__, 1 ) // 1 sec timeout - ) { - $this->doUpdatePendingDeltas(); - return; - } - $pd = $this->getPendingDeltas(); - // Piggy-back the async deltas onto those of this stats update.... - $this->views += ( $pd['ss_total_views']['+'] - $pd['ss_total_views']['-'] ); - $this->edits += ( $pd['ss_total_edits']['+'] - $pd['ss_total_edits']['-'] ); - $this->articles += ( $pd['ss_good_articles']['+'] - $pd['ss_good_articles']['-'] ); - $this->pages += ( $pd['ss_total_pages']['+'] - $pd['ss_total_pages']['-'] ); - $this->users += ( $pd['ss_users']['+'] - $pd['ss_users']['-'] ); - $this->images += ( $pd['ss_images']['+'] - $pd['ss_images']['-'] ); - } - - // Build up an SQL query of deltas and apply them... - $updates = ''; - $this->appendUpdate( $updates, 'ss_total_views', $this->views ); - $this->appendUpdate( $updates, 'ss_total_edits', $this->edits ); - $this->appendUpdate( $updates, 'ss_good_articles', $this->articles ); - $this->appendUpdate( $updates, 'ss_total_pages', $this->pages ); - $this->appendUpdate( $updates, 'ss_users', $this->users ); - $this->appendUpdate( $updates, 'ss_images', $this->images ); - if ( $updates != '' ) { - $dbw->update( 'site_stats', array( $updates ), array(), __METHOD__ ); - } - - if ( $wgSiteStatsAsyncFactor ) { - // Decrement the async deltas now that we applied them - $this->removePendingDeltas( $pd ); - // Commit the updates and unlock the table - $dbw->unlock( $lockKey, __METHOD__ ); - } - } - - /** - * @param $dbw DatabaseBase - * @return bool|mixed - */ - public static function cacheUpdate( $dbw ) { - global $wgActiveUserDays; - $dbr = wfGetDB( DB_SLAVE, array( 'SpecialStatistics', 'vslow' ) ); - # Get non-bot users than did some recent action other than making accounts. - # If account creation is included, the number gets inflated ~20+ fold on enwiki. - $activeUsers = $dbr->selectField( - 'recentchanges', - 'COUNT( DISTINCT rc_user_text )', - array( - 'rc_user != 0', - 'rc_bot' => 0, - 'rc_log_type != ' . $dbr->addQuotes( 'newusers' ) . ' OR rc_log_type IS NULL', - 'rc_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( wfTimestamp( TS_UNIX ) - $wgActiveUserDays * 24 * 3600 ) ), - ), - __METHOD__ - ); - $dbw->update( - 'site_stats', - array( 'ss_active_users' => intval( $activeUsers ) ), - array( 'ss_row_id' => 1 ), - __METHOD__ - ); - return $activeUsers; - } - - protected function doUpdatePendingDeltas() { - $this->adjustPending( 'ss_total_views', $this->views ); - $this->adjustPending( 'ss_total_edits', $this->edits ); - $this->adjustPending( 'ss_good_articles', $this->articles ); - $this->adjustPending( 'ss_total_pages', $this->pages ); - $this->adjustPending( 'ss_users', $this->users ); - $this->adjustPending( 'ss_images', $this->images ); - } - - /** - * @param $sql string - * @param $field string - * @param $delta integer - */ - protected function appendUpdate( &$sql, $field, $delta ) { - if ( $delta ) { - if ( $sql ) { - $sql .= ','; - } - if ( $delta < 0 ) { - $sql .= "$field=$field-" . abs( $delta ); - } else { - $sql .= "$field=$field+" . abs( $delta ); - } - } - } - - /** - * @param $type string - * @param string $sign ('+' or '-') - * @return string - */ - private function getTypeCacheKey( $type, $sign ) { - return wfMemcKey( 'sitestatsupdate', 'pendingdelta', $type, $sign ); - } - - /** - * Adjust the pending deltas for a stat type. - * Each stat type has two pending counters, one for increments and decrements - * @param $type string - * @param $delta integer Delta (positive or negative) - * @return void - */ - protected function adjustPending( $type, $delta ) { - global $wgMemc; - - if ( $delta < 0 ) { // decrement - $key = $this->getTypeCacheKey( $type, '-' ); - } else { // increment - $key = $this->getTypeCacheKey( $type, '+' ); - } - - $magnitude = abs( $delta ); - if ( !$wgMemc->incr( $key, $magnitude ) ) { // not there? - if ( !$wgMemc->add( $key, $magnitude ) ) { // race? - $wgMemc->incr( $key, $magnitude ); - } - } - } - - /** - * Get pending delta counters for each stat type - * @return Array Positive and negative deltas for each type - * @return void - */ - protected function getPendingDeltas() { - global $wgMemc; - - $pending = array(); - foreach ( array( 'ss_total_views', 'ss_total_edits', - 'ss_good_articles', 'ss_total_pages', 'ss_users', 'ss_images' ) as $type ) - { - // Get pending increments and pending decrements - $pending[$type]['+'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '+' ) ); - $pending[$type]['-'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '-' ) ); - } - - return $pending; - } - - /** - * Reduce pending delta counters after updates have been applied - * @param array $pd Result of getPendingDeltas(), used for DB update - * @return void - */ - protected function removePendingDeltas( array $pd ) { - global $wgMemc; - - foreach ( $pd as $type => $deltas ) { - foreach ( $deltas as $sign => $magnitude ) { - // Lower the pending counter now that we applied these changes - $wgMemc->decr( $this->getTypeCacheKey( $type, $sign ), $magnitude ); - } - } - } -} - /** * Class designed for counting of stats. */ diff --git a/includes/SqlDataUpdate.php b/includes/SqlDataUpdate.php deleted file mode 100644 index 51188d85c9..0000000000 --- a/includes/SqlDataUpdate.php +++ /dev/null @@ -1,152 +0,0 @@ -mOptions = array(); - } else { - $this->mOptions = array( 'FOR UPDATE' ); - } - - // @todo get connection only when it's needed? make sure that doesn't break anything, especially transactions! - $this->mDb = wfGetDB( DB_MASTER ); - - $this->mWithTransaction = $withTransaction; - $this->mHasTransaction = false; - } - - /** - * Begin a database transaction, if $withTransaction was given as true in the constructor for this SqlDataUpdate. - * - * Because nested transactions are not supported by the Database class, this implementation - * checks Database::trxLevel() and only opens a transaction if none is already active. - */ - public function beginTransaction() { - if ( !$this->mWithTransaction ) { - return; - } - - // NOTE: nested transactions are not supported, only start a transaction if none is open - if ( $this->mDb->trxLevel() === 0 ) { - $this->mDb->begin( get_class( $this ) . '::beginTransaction' ); - $this->mHasTransaction = true; - } - } - - /** - * Commit the database transaction started via beginTransaction (if any). - */ - public function commitTransaction() { - if ( $this->mHasTransaction ) { - $this->mDb->commit( get_class( $this ) . '::commitTransaction' ); - $this->mHasTransaction = false; - } - } - - /** - * Abort the database transaction started via beginTransaction (if any). - */ - public function abortTransaction() { - if ( $this->mHasTransaction ) { //XXX: actually... maybe always? - $this->mDb->rollback( get_class( $this ) . '::abortTransaction' ); - $this->mHasTransaction = false; - } - } - - /** - * Invalidate the cache of a list of pages from a single namespace. - * This is intended for use by subclasses. - * - * @param $namespace Integer - * @param $dbkeys Array - */ - protected function invalidatePages( $namespace, array $dbkeys ) { - if ( $dbkeys === array() ) { - return; - } - - /** - * Determine which pages need to be updated - * This is necessary to prevent the job queue from smashing the DB with - * large numbers of concurrent invalidations of the same page - */ - $now = $this->mDb->timestamp(); - $ids = array(); - $res = $this->mDb->select( 'page', array( 'page_id' ), - array( - 'page_namespace' => $namespace, - 'page_title' => $dbkeys, - 'page_touched < ' . $this->mDb->addQuotes( $now ) - ), __METHOD__ - ); - - foreach ( $res as $row ) { - $ids[] = $row->page_id; - } - - if ( $ids === array() ) { - return; - } - - /** - * Do the update - * We still need the page_touched condition, in case the row has changed since - * the non-locking select above. - */ - $this->mDb->update( 'page', array( 'page_touched' => $now ), - array( - 'page_id' => $ids, - 'page_touched < ' . $this->mDb->addQuotes( $now ) - ), __METHOD__ - ); - } - -} diff --git a/includes/ViewCountUpdate.php b/includes/ViewCountUpdate.php deleted file mode 100644 index 22a4649364..0000000000 --- a/includes/ViewCountUpdate.php +++ /dev/null @@ -1,105 +0,0 @@ -id = intval( $id ); - } - - /** - * Run the update - */ - public function doUpdate() { - global $wgHitcounterUpdateFreq; - - $dbw = wfGetDB( DB_MASTER ); - - if ( $wgHitcounterUpdateFreq <= 1 || $dbw->getType() == 'sqlite' ) { - $dbw->update( 'page', array( 'page_counter = page_counter + 1' ), array( 'page_id' => $this->id ), __METHOD__ ); - return; - } - - # Not important enough to warrant an error page in case of failure - try { - $dbw->insert( 'hitcounter', array( 'hc_id' => $this->id ), __METHOD__ ); - $checkfreq = intval( $wgHitcounterUpdateFreq / 25 + 1 ); - if ( rand() % $checkfreq == 0 && $dbw->lastErrno() == 0 ) { - $this->collect(); - } - } catch ( DBError $e ) {} - } - - protected function collect() { - global $wgHitcounterUpdateFreq; - - $dbw = wfGetDB( DB_MASTER ); - - $rown = $dbw->selectField( 'hitcounter', 'COUNT(*)', array(), __METHOD__ ); - - if ( $rown < $wgHitcounterUpdateFreq ) { - return; - } - - wfProfileIn( __METHOD__ . '-collect' ); - $old_user_abort = ignore_user_abort( true ); - - $dbw->lockTables( array(), array( 'hitcounter' ), __METHOD__, false ); - - $dbType = $dbw->getType(); - $tabletype = $dbType == 'mysql' ? "ENGINE=HEAP " : ''; - $hitcounterTable = $dbw->tableName( 'hitcounter' ); - $acchitsTable = $dbw->tableName( 'acchits' ); - $pageTable = $dbw->tableName( 'page' ); - - $dbw->query( "CREATE TEMPORARY TABLE $acchitsTable $tabletype AS " . - "SELECT hc_id,COUNT(*) AS hc_n FROM $hitcounterTable " . - 'GROUP BY hc_id', __METHOD__ ); - $dbw->delete( 'hitcounter', '*', __METHOD__ ); - $dbw->unlockTables( __METHOD__ ); - - if ( $dbType == 'mysql' ) { - $dbw->query( "UPDATE $pageTable,$acchitsTable SET page_counter=page_counter + hc_n " . - 'WHERE page_id = hc_id', __METHOD__ ); - } else { - $dbw->query( "UPDATE $pageTable SET page_counter=page_counter + hc_n " . - "FROM $acchitsTable WHERE page_id = hc_id", __METHOD__ ); - } - $dbw->query( "DROP TABLE $acchitsTable", __METHOD__ ); - - ignore_user_abort( $old_user_abort ); - wfProfileOut( __METHOD__ . '-collect' ); - } -} diff --git a/includes/cache/HTMLCacheUpdate.php b/includes/cache/HTMLCacheUpdate.php deleted file mode 100644 index 4147424c2a..0000000000 --- a/includes/cache/HTMLCacheUpdate.php +++ /dev/null @@ -1,71 +0,0 @@ -mTitle = $titleTo; - $this->mTable = $table; - } - - public function doUpdate() { - wfProfileIn( __METHOD__ ); - - $job = new HTMLCacheUpdateJob( - $this->mTitle, - array( - 'table' => $this->mTable, - ) + Job::newRootJobParams( // "overall" refresh links job info - "htmlCacheUpdate:{$this->mTable}:{$this->mTitle->getPrefixedText()}" - ) - ); - - $count = $this->mTitle->getBacklinkCache()->getNumLinks( $this->mTable, 200 ); - if ( $count >= 200 ) { // many backlinks - JobQueueGroup::singleton()->push( $job ); - JobQueueGroup::singleton()->deduplicateRootJob( $job ); - } else { // few backlinks ($count might be off even if 0) - $dbw = wfGetDB( DB_MASTER ); - $dbw->onTransactionIdle( function() use ( $job ) { - $job->run(); // just do the purge query now - } ); - } - - wfProfileOut( __METHOD__ ); - } -} diff --git a/includes/cache/SquidUpdate.php b/includes/cache/SquidUpdate.php deleted file mode 100644 index 71afeba924..0000000000 --- a/includes/cache/SquidUpdate.php +++ /dev/null @@ -1,300 +0,0 @@ - $maxTitles ) { - // Truncate to desired maximum URL count - $urlArr = array_slice( $urlArr, 0, $maxTitles ); - } - $this->urlArr = $urlArr; - } - - /** - * Create a SquidUpdate from the given Title object. - * - * The resulting SquidUpdate will purge the given Title's URLs as well as - * the pages that link to it. Capped at $wgMaxSquidPurgeTitles total URLs. - * - * @param Title $title - * @return SquidUpdate - */ - public static function newFromLinksTo( Title $title ) { - global $wgMaxSquidPurgeTitles; - wfProfileIn( __METHOD__ ); - - # Get a list of URLs linking to this page - $dbr = wfGetDB( DB_SLAVE ); - $res = $dbr->select( array( 'links', 'page' ), - array( 'page_namespace', 'page_title' ), - array( - 'pl_namespace' => $title->getNamespace(), - 'pl_title' => $title->getDBkey(), - 'pl_from=page_id' ), - __METHOD__ ); - $blurlArr = $title->getSquidURLs(); - if ( $res->numRows() <= $wgMaxSquidPurgeTitles ) { - foreach ( $res as $BL ) { - $tobj = Title::makeTitle( $BL->page_namespace, $BL->page_title ); - $blurlArr[] = $tobj->getInternalURL(); - } - } - - wfProfileOut( __METHOD__ ); - return new SquidUpdate( $blurlArr ); - } - - /** - * Create a SquidUpdate from an array of Title objects, or a TitleArray object - * - * @param array $titles - * @param array $urlArr - * @return SquidUpdate - */ - public static function newFromTitles( $titles, $urlArr = array() ) { - global $wgMaxSquidPurgeTitles; - $i = 0; - foreach ( $titles as $title ) { - $urlArr[] = $title->getInternalURL(); - if ( $i++ > $wgMaxSquidPurgeTitles ) { - break; - } - } - return new SquidUpdate( $urlArr ); - } - - /** - * @param Title $title - * @return SquidUpdate - */ - public static function newSimplePurge( Title $title ) { - $urlArr = $title->getSquidURLs(); - return new SquidUpdate( $urlArr ); - } - - /** - * Purges the list of URLs passed to the constructor. - */ - public function doUpdate() { - self::purge( $this->urlArr ); - } - - /** - * Purges a list of Squids defined in $wgSquidServers. - * $urlArr should contain the full URLs to purge as values - * (example: $urlArr[] = 'http://my.host/something') - * XXX report broken Squids per mail or log - * - * @param array $urlArr List of full URLs to purge - */ - public static function purge( $urlArr ) { - global $wgSquidServers, $wgHTCPRouting; - - if ( !$urlArr ) { - return; - } - - wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) . "\n" ); - - if ( $wgHTCPRouting ) { - self::HTCPPurge( $urlArr ); - } - - wfProfileIn( __METHOD__ ); - - // Remove duplicate URLs - $urlArr = array_unique( $urlArr ); - // Maximum number of parallel connections per squid - $maxSocketsPerSquid = 8; - // Number of requests to send per socket - // 400 seems to be a good tradeoff, opening a socket takes a while - $urlsPerSocket = 400; - $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket ); - if ( $socketsPerSquid > $maxSocketsPerSquid ) { - $socketsPerSquid = $maxSocketsPerSquid; - } - - $pool = new SquidPurgeClientPool; - $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) ); - foreach ( $wgSquidServers as $server ) { - foreach ( $chunks as $chunk ) { - $client = new SquidPurgeClient( $server ); - foreach ( $chunk as $url ) { - $client->queuePurge( $url ); - } - $pool->addClient( $client ); - } - } - $pool->run(); - - wfProfileOut( __METHOD__ ); - } - - /** - * Send Hyper Text Caching Protocol (HTCP) CLR requests. - * - * @throws MWException - * @param array $urlArr Collection of URLs to purge - */ - public static function HTCPPurge( $urlArr ) { - global $wgHTCPRouting, $wgHTCPMulticastTTL; - wfProfileIn( __METHOD__ ); - - // HTCP CLR operation - $htcpOpCLR = 4; - - // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h) - if ( !defined( "IPPROTO_IP" ) ) { - define( "IPPROTO_IP", 0 ); - define( "IP_MULTICAST_LOOP", 34 ); - define( "IP_MULTICAST_TTL", 33 ); - } - - // pfsockopen doesn't work because we need set_sock_opt - $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP ); - if ( ! $conn ) { - $errstr = socket_strerror( socket_last_error() ); - wfDebugLog( 'squid', __METHOD__ . - ": Error opening UDP socket: $errstr\n" ); - wfProfileOut( __METHOD__ ); - return; - } - - // Set socket options - socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 ); - if ( $wgHTCPMulticastTTL != 1 ) { - // Set multicast time to live (hop count) option on socket - socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL, - $wgHTCPMulticastTTL ); - } - - // Remove duplicate URLs from collection - $urlArr = array_unique( $urlArr ); - foreach ( $urlArr as $url ) { - if ( !is_string( $url ) ) { - wfProfileOut( __METHOD__ ); - throw new MWException( 'Bad purge URL' ); - } - $url = self::expand( $url ); - $conf = self::getRuleForURL( $url, $wgHTCPRouting ); - if ( !$conf ) { - wfDebugLog( 'squid', __METHOD__ . - "No HTCP rule configured for URL {$url} , skipping\n" ); - continue; - } - - if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) { - // Normalize single entries - $conf = array( $conf ); - } - foreach ( $conf as $subconf ) { - if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) { - wfProfileOut( __METHOD__ ); - throw new MWException( "Invalid HTCP rule for URL $url\n" ); - } - } - - // Construct a minimal HTCP request diagram - // as per RFC 2756 - // Opcode 'CLR', no response desired, no auth - $htcpTransID = rand(); - - $htcpSpecifier = pack( 'na4na*na8n', - 4, 'HEAD', strlen( $url ), $url, - 8, 'HTTP/1.0', 0 ); - - $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier ); - $htcpLen = 4 + $htcpDataLen + 2; - - // Note! Squid gets the bit order of the first - // word wrong, wrt the RFC. Apparently no other - // implementation exists, so adapt to Squid - $htcpPacket = pack( 'nxxnCxNxxa*n', - $htcpLen, $htcpDataLen, $htcpOpCLR, - $htcpTransID, $htcpSpecifier, 2 ); - - wfDebugLog( 'squid', __METHOD__ . - "Purging URL $url via HTCP\n" ); - foreach ( $conf as $subconf ) { - socket_sendto( $conn, $htcpPacket, $htcpLen, 0, - $subconf['host'], $subconf['port'] ); - } - } - wfProfileOut( __METHOD__ ); - } - - /** - * Expand local URLs to fully-qualified URLs using the internal protocol - * and host defined in $wgInternalServer. Input that's already fully- - * qualified will be passed through unchanged. - * - * This is used to generate purge URLs that may be either local to the - * main wiki or include a non-native host, such as images hosted on a - * second internal server. - * - * Client functions should not need to call this. - * - * @param string $url - * @return string - */ - public static function expand( $url ) { - return wfExpandUrl( $url, PROTO_INTERNAL ); - } - - /** - * Find the HTCP routing rule to use for a given URL. - * @param string $url URL to match - * @param array $rules Array of rules, see $wgHTCPRouting for format and behavior - * @return mixed Element of $rules that matched, or false if nothing matched - */ - private static function getRuleForURL( $url, $rules ) { - foreach ( $rules as $regex => $routing ) { - if ( $regex === '' || preg_match( $regex, $url ) ) { - return $routing; - } - } - return false; - } -} diff --git a/includes/deferred/CallableUpdate.php b/includes/deferred/CallableUpdate.php new file mode 100644 index 0000000000..6eb5541393 --- /dev/null +++ b/includes/deferred/CallableUpdate.php @@ -0,0 +1,30 @@ +callback = $callback; + } + + /** + * Run the update + */ + public function doUpdate() { + call_user_func( $this->callback ); + } + +} diff --git a/includes/deferred/DataUpdate.php b/includes/deferred/DataUpdate.php new file mode 100644 index 0000000000..7b9ac2811d --- /dev/null +++ b/includes/deferred/DataUpdate.php @@ -0,0 +1,126 @@ +beginTransaction(); + $open_transactions[] = $update; + } + + // do work + foreach ( $updates as $update ) { + $update->doUpdate(); + } + + // commit transactions + while ( count( $open_transactions ) > 0 ) { + $trans = array_pop( $open_transactions ); + $trans->commitTransaction(); + } + } catch ( Exception $ex ) { + $exception = $ex; + wfDebug( "Caught exception, will rethrow after rollback: " . $ex->getMessage() ); + } + + // rollback remaining transactions + while ( count( $open_transactions ) > 0 ) { + $trans = array_pop( $open_transactions ); + $trans->rollbackTransaction(); + } + + if ( $exception ) { + throw $exception; // rethrow after cleanup + } + } + +} diff --git a/includes/deferred/DeferredUpdates.php b/includes/deferred/DeferredUpdates.php new file mode 100644 index 0000000000..c385f138e9 --- /dev/null +++ b/includes/deferred/DeferredUpdates.php @@ -0,0 +1,129 @@ +doUpdate(); + + if ( $doCommit && $dbw->trxLevel() ) { + $dbw->commit( __METHOD__, 'flush' ); + } + } catch ( MWException $e ) { + // We don't want exceptions thrown during deferred updates to + // be reported to the user since the output is already sent. + // Instead we just log them. + if ( !$e instanceof ErrorPageError ) { + MWExceptionHandler::logException( $e ); + } + } + } + + self::clearPendingUpdates(); + wfProfileOut( __METHOD__ ); + } + + /** + * Clear all pending updates without performing them. Generally, you don't + * want or need to call this. Unit tests need it though. + */ + public static function clearPendingUpdates() { + global $wgDeferredUpdateList; + $wgDeferredUpdateList = self::$updates = array(); + } +} diff --git a/includes/deferred/HTMLCacheUpdate.php b/includes/deferred/HTMLCacheUpdate.php new file mode 100644 index 0000000000..4147424c2a --- /dev/null +++ b/includes/deferred/HTMLCacheUpdate.php @@ -0,0 +1,71 @@ +mTitle = $titleTo; + $this->mTable = $table; + } + + public function doUpdate() { + wfProfileIn( __METHOD__ ); + + $job = new HTMLCacheUpdateJob( + $this->mTitle, + array( + 'table' => $this->mTable, + ) + Job::newRootJobParams( // "overall" refresh links job info + "htmlCacheUpdate:{$this->mTable}:{$this->mTitle->getPrefixedText()}" + ) + ); + + $count = $this->mTitle->getBacklinkCache()->getNumLinks( $this->mTable, 200 ); + if ( $count >= 200 ) { // many backlinks + JobQueueGroup::singleton()->push( $job ); + JobQueueGroup::singleton()->deduplicateRootJob( $job ); + } else { // few backlinks ($count might be off even if 0) + $dbw = wfGetDB( DB_MASTER ); + $dbw->onTransactionIdle( function() use ( $job ) { + $job->run(); // just do the purge query now + } ); + } + + wfProfileOut( __METHOD__ ); + } +} diff --git a/includes/deferred/LinksUpdate.php b/includes/deferred/LinksUpdate.php new file mode 100644 index 0000000000..fdd0e3c116 --- /dev/null +++ b/includes/deferred/LinksUpdate.php @@ -0,0 +1,892 @@ +mTitle = $title; + $this->mId = $title->getArticleID(); + + if ( !$this->mId ) { + throw new MWException( "The Title object did not provide an article ID. Perhaps the page doesn't exist?" ); + } + + $this->mParserOutput = $parserOutput; + + $this->mLinks = $parserOutput->getLinks(); + $this->mImages = $parserOutput->getImages(); + $this->mTemplates = $parserOutput->getTemplates(); + $this->mExternals = $parserOutput->getExternalLinks(); + $this->mCategories = $parserOutput->getCategories(); + $this->mProperties = $parserOutput->getProperties(); + $this->mInterwikis = $parserOutput->getInterwikiLinks(); + + # Convert the format of the interlanguage links + # I didn't want to change it in the ParserOutput, because that array is passed all + # the way back to the skin, so either a skin API break would be required, or an + # inefficient back-conversion. + $ill = $parserOutput->getLanguageLinks(); + $this->mInterlangs = array(); + foreach ( $ill as $link ) { + list( $key, $title ) = explode( ':', $link, 2 ); + $this->mInterlangs[$key] = $title; + } + + foreach ( $this->mCategories as &$sortkey ) { + # If the sortkey is longer then 255 bytes, + # it truncated by DB, and then doesn't get + # matched when comparing existing vs current + # categories, causing bug 25254. + # Also. substr behaves weird when given "". + if ( $sortkey !== '' ) { + $sortkey = substr( $sortkey, 0, 255 ); + } + } + + $this->mRecursive = $recursive; + + wfRunHooks( 'LinksUpdateConstructed', array( &$this ) ); + } + + /** + * Update link tables with outgoing links from an updated article + */ + public function doUpdate() { + wfRunHooks( 'LinksUpdate', array( &$this ) ); + $this->doIncrementalUpdate(); + wfRunHooks( 'LinksUpdateComplete', array( &$this ) ); + } + + protected function doIncrementalUpdate() { + wfProfileIn( __METHOD__ ); + + # Page links + $existing = $this->getExistingLinks(); + $this->linkDeletions = $this->getLinkDeletions( $existing ); + $this->linkInsertions = $this->getLinkInsertions( $existing ); + $this->incrTableUpdate( 'pagelinks', 'pl', $this->linkDeletions, $this->linkInsertions ); + + # Image links + $existing = $this->getExistingImages(); + + $imageDeletes = $this->getImageDeletions( $existing ); + $this->incrTableUpdate( 'imagelinks', 'il', $imageDeletes, + $this->getImageInsertions( $existing ) ); + + # Invalidate all image description pages which had links added or removed + $imageUpdates = $imageDeletes + array_diff_key( $this->mImages, $existing ); + $this->invalidateImageDescriptions( $imageUpdates ); + + # External links + $existing = $this->getExistingExternals(); + $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ), + $this->getExternalInsertions( $existing ) ); + + # Language links + $existing = $this->getExistingInterlangs(); + $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ), + $this->getInterlangInsertions( $existing ) ); + + # Inline interwiki links + $existing = $this->getExistingInterwikis(); + $this->incrTableUpdate( 'iwlinks', 'iwl', $this->getInterwikiDeletions( $existing ), + $this->getInterwikiInsertions( $existing ) ); + + # Template links + $existing = $this->getExistingTemplates(); + $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ), + $this->getTemplateInsertions( $existing ) ); + + # Category links + $existing = $this->getExistingCategories(); + + $categoryDeletes = $this->getCategoryDeletions( $existing ); + + $this->incrTableUpdate( 'categorylinks', 'cl', $categoryDeletes, + $this->getCategoryInsertions( $existing ) ); + + # Invalidate all categories which were added, deleted or changed (set symmetric difference) + $categoryInserts = array_diff_assoc( $this->mCategories, $existing ); + $categoryUpdates = $categoryInserts + $categoryDeletes; + $this->invalidateCategories( $categoryUpdates ); + $this->updateCategoryCounts( $categoryInserts, $categoryDeletes ); + + # Page properties + $existing = $this->getExistingProperties(); + + $propertiesDeletes = $this->getPropertyDeletions( $existing ); + + $this->incrTableUpdate( 'page_props', 'pp', $propertiesDeletes, + $this->getPropertyInsertions( $existing ) ); + + # Invalidate the necessary pages + $changed = $propertiesDeletes + array_diff_assoc( $this->mProperties, $existing ); + $this->invalidateProperties( $changed ); + + # Refresh links of all pages including this page + # This will be in a separate transaction + if ( $this->mRecursive ) { + $this->queueRecursiveJobs(); + } + + wfProfileOut( __METHOD__ ); + } + + /** + * Queue recursive jobs for this page + * + * Which means do LinksUpdate on all templates + * that include the current page, using the job queue. + */ + function queueRecursiveJobs() { + self::queueRecursiveJobsForTable( $this->mTitle, 'templatelinks' ); + } + + /** + * Queue a RefreshLinks job for any table. + * + * @param Title $title Title to do job for + * @param String $table Table to use (e.g. 'templatelinks') + */ + public static function queueRecursiveJobsForTable( Title $title, $table ) { + wfProfileIn( __METHOD__ ); + if ( $title->getBacklinkCache()->hasLinks( $table ) ) { + $job = new RefreshLinksJob2( + $title, + array( + 'table' => $table, + ) + Job::newRootJobParams( // "overall" refresh links job info + "refreshlinks:{$table}:{$title->getPrefixedText()}" + ) + ); + JobQueueGroup::singleton()->push( $job ); + JobQueueGroup::singleton()->deduplicateRootJob( $job ); + } + wfProfileOut( __METHOD__ ); + } + + /** + * @param $cats + */ + function invalidateCategories( $cats ) { + $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) ); + } + + /** + * Update all the appropriate counts in the category table. + * @param array $added associative array of category name => sort key + * @param array $deleted associative array of category name => sort key + */ + function updateCategoryCounts( $added, $deleted ) { + $a = WikiPage::factory( $this->mTitle ); + $a->updateCategoryCounts( + array_keys( $added ), array_keys( $deleted ) + ); + } + + /** + * @param $images + */ + function invalidateImageDescriptions( $images ) { + $this->invalidatePages( NS_FILE, array_keys( $images ) ); + } + + /** + * Update a table by doing a delete query then an insert query + * @param $table + * @param $prefix + * @param $deletions + * @param $insertions + */ + function incrTableUpdate( $table, $prefix, $deletions, $insertions ) { + if ( $table == 'page_props' ) { + $fromField = 'pp_page'; + } else { + $fromField = "{$prefix}_from"; + } + $where = array( $fromField => $this->mId ); + if ( $table == 'pagelinks' || $table == 'templatelinks' || $table == 'iwlinks' ) { + if ( $table == 'iwlinks' ) { + $baseKey = 'iwl_prefix'; + } else { + $baseKey = "{$prefix}_namespace"; + } + $clause = $this->mDb->makeWhereFrom2d( $deletions, $baseKey, "{$prefix}_title" ); + if ( $clause ) { + $where[] = $clause; + } else { + $where = false; + } + } else { + if ( $table == 'langlinks' ) { + $toField = 'll_lang'; + } elseif ( $table == 'page_props' ) { + $toField = 'pp_propname'; + } else { + $toField = $prefix . '_to'; + } + if ( count( $deletions ) ) { + $where[] = "$toField IN (" . $this->mDb->makeList( array_keys( $deletions ) ) . ')'; + } else { + $where = false; + } + } + if ( $where ) { + $this->mDb->delete( $table, $where, __METHOD__ ); + } + if ( count( $insertions ) ) { + $this->mDb->insert( $table, $insertions, __METHOD__, 'IGNORE' ); + wfRunHooks( 'LinksUpdateAfterInsert', array( $this, $table, $insertions ) ); + } + } + + /** + * Get an array of pagelinks insertions for passing to the DB + * Skips the titles specified by the 2-D array $existing + * @param $existing array + * @return array + */ + private function getLinkInsertions( $existing = array() ) { + $arr = array(); + foreach ( $this->mLinks as $ns => $dbkeys ) { + $diffs = isset( $existing[$ns] ) + ? array_diff_key( $dbkeys, $existing[$ns] ) + : $dbkeys; + foreach ( $diffs as $dbk => $id ) { + $arr[] = array( + 'pl_from' => $this->mId, + 'pl_namespace' => $ns, + 'pl_title' => $dbk + ); + } + } + return $arr; + } + + /** + * Get an array of template insertions. Like getLinkInsertions() + * @param $existing array + * @return array + */ + private function getTemplateInsertions( $existing = array() ) { + $arr = array(); + foreach ( $this->mTemplates as $ns => $dbkeys ) { + $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys; + foreach ( $diffs as $dbk => $id ) { + $arr[] = array( + 'tl_from' => $this->mId, + 'tl_namespace' => $ns, + 'tl_title' => $dbk + ); + } + } + return $arr; + } + + /** + * Get an array of image insertions + * Skips the names specified in $existing + * @param $existing array + * @return array + */ + private function getImageInsertions( $existing = array() ) { + $arr = array(); + $diffs = array_diff_key( $this->mImages, $existing ); + foreach ( $diffs as $iname => $dummy ) { + $arr[] = array( + 'il_from' => $this->mId, + 'il_to' => $iname + ); + } + return $arr; + } + + /** + * Get an array of externallinks insertions. Skips the names specified in $existing + * @param $existing array + * @return array + */ + private function getExternalInsertions( $existing = array() ) { + $arr = array(); + $diffs = array_diff_key( $this->mExternals, $existing ); + foreach ( $diffs as $url => $dummy ) { + foreach ( wfMakeUrlIndexes( $url ) as $index ) { + $arr[] = array( + 'el_from' => $this->mId, + 'el_to' => $url, + 'el_index' => $index, + ); + } + } + return $arr; + } + + /** + * Get an array of category insertions + * + * @param array $existing mapping existing category names to sort keys. If both + * match a link in $this, the link will be omitted from the output + * + * @return array + */ + private function getCategoryInsertions( $existing = array() ) { + global $wgContLang, $wgCategoryCollation; + $diffs = array_diff_assoc( $this->mCategories, $existing ); + $arr = array(); + foreach ( $diffs as $name => $prefix ) { + $nt = Title::makeTitleSafe( NS_CATEGORY, $name ); + $wgContLang->findVariantLink( $name, $nt, true ); + + if ( $this->mTitle->getNamespace() == NS_CATEGORY ) { + $type = 'subcat'; + } elseif ( $this->mTitle->getNamespace() == NS_FILE ) { + $type = 'file'; + } else { + $type = 'page'; + } + + # Treat custom sortkeys as a prefix, so that if multiple + # things are forced to sort as '*' or something, they'll + # sort properly in the category rather than in page_id + # order or such. + $sortkey = Collation::singleton()->getSortKey( + $this->mTitle->getCategorySortkey( $prefix ) ); + + $arr[] = array( + 'cl_from' => $this->mId, + 'cl_to' => $name, + 'cl_sortkey' => $sortkey, + 'cl_timestamp' => $this->mDb->timestamp(), + 'cl_sortkey_prefix' => $prefix, + 'cl_collation' => $wgCategoryCollation, + 'cl_type' => $type, + ); + } + return $arr; + } + + /** + * Get an array of interlanguage link insertions + * + * @param array $existing mapping existing language codes to titles + * + * @return array + */ + private function getInterlangInsertions( $existing = array() ) { + $diffs = array_diff_assoc( $this->mInterlangs, $existing ); + $arr = array(); + foreach ( $diffs as $lang => $title ) { + $arr[] = array( + 'll_from' => $this->mId, + 'll_lang' => $lang, + 'll_title' => $title + ); + } + return $arr; + } + + /** + * Get an array of page property insertions + * @param $existing array + * @return array + */ + function getPropertyInsertions( $existing = array() ) { + $diffs = array_diff_assoc( $this->mProperties, $existing ); + $arr = array(); + foreach ( $diffs as $name => $value ) { + $arr[] = array( + 'pp_page' => $this->mId, + 'pp_propname' => $name, + 'pp_value' => $value, + ); + } + return $arr; + } + + /** + * Get an array of interwiki insertions for passing to the DB + * Skips the titles specified by the 2-D array $existing + * @param $existing array + * @return array + */ + private function getInterwikiInsertions( $existing = array() ) { + $arr = array(); + foreach ( $this->mInterwikis as $prefix => $dbkeys ) { + $diffs = isset( $existing[$prefix] ) ? array_diff_key( $dbkeys, $existing[$prefix] ) : $dbkeys; + foreach ( $diffs as $dbk => $id ) { + $arr[] = array( + 'iwl_from' => $this->mId, + 'iwl_prefix' => $prefix, + 'iwl_title' => $dbk + ); + } + } + return $arr; + } + + /** + * Given an array of existing links, returns those links which are not in $this + * and thus should be deleted. + * @param $existing array + * @return array + */ + private function getLinkDeletions( $existing ) { + $del = array(); + foreach ( $existing as $ns => $dbkeys ) { + if ( isset( $this->mLinks[$ns] ) ) { + $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] ); + } else { + $del[$ns] = $existing[$ns]; + } + } + return $del; + } + + /** + * Given an array of existing templates, returns those templates which are not in $this + * and thus should be deleted. + * @param $existing array + * @return array + */ + private function getTemplateDeletions( $existing ) { + $del = array(); + foreach ( $existing as $ns => $dbkeys ) { + if ( isset( $this->mTemplates[$ns] ) ) { + $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] ); + } else { + $del[$ns] = $existing[$ns]; + } + } + return $del; + } + + /** + * Given an array of existing images, returns those images which are not in $this + * and thus should be deleted. + * @param $existing array + * @return array + */ + private function getImageDeletions( $existing ) { + return array_diff_key( $existing, $this->mImages ); + } + + /** + * Given an array of existing external links, returns those links which are not + * in $this and thus should be deleted. + * @param $existing array + * @return array + */ + private function getExternalDeletions( $existing ) { + return array_diff_key( $existing, $this->mExternals ); + } + + /** + * Given an array of existing categories, returns those categories which are not in $this + * and thus should be deleted. + * @param $existing array + * @return array + */ + private function getCategoryDeletions( $existing ) { + return array_diff_assoc( $existing, $this->mCategories ); + } + + /** + * Given an array of existing interlanguage links, returns those links which are not + * in $this and thus should be deleted. + * @param $existing array + * @return array + */ + private function getInterlangDeletions( $existing ) { + return array_diff_assoc( $existing, $this->mInterlangs ); + } + + /** + * Get array of properties which should be deleted. + * @param $existing array + * @return array + */ + function getPropertyDeletions( $existing ) { + return array_diff_assoc( $existing, $this->mProperties ); + } + + /** + * Given an array of existing interwiki links, returns those links which are not in $this + * and thus should be deleted. + * @param $existing array + * @return array + */ + private function getInterwikiDeletions( $existing ) { + $del = array(); + foreach ( $existing as $prefix => $dbkeys ) { + if ( isset( $this->mInterwikis[$prefix] ) ) { + $del[$prefix] = array_diff_key( $existing[$prefix], $this->mInterwikis[$prefix] ); + } else { + $del[$prefix] = $existing[$prefix]; + } + } + return $del; + } + + /** + * Get an array of existing links, as a 2-D array + * + * @return array + */ + private function getExistingLinks() { + $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ), + array( 'pl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + if ( !isset( $arr[$row->pl_namespace] ) ) { + $arr[$row->pl_namespace] = array(); + } + $arr[$row->pl_namespace][$row->pl_title] = 1; + } + return $arr; + } + + /** + * Get an array of existing templates, as a 2-D array + * + * @return array + */ + private function getExistingTemplates() { + $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ), + array( 'tl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + if ( !isset( $arr[$row->tl_namespace] ) ) { + $arr[$row->tl_namespace] = array(); + } + $arr[$row->tl_namespace][$row->tl_title] = 1; + } + return $arr; + } + + /** + * Get an array of existing images, image names in the keys + * + * @return array + */ + private function getExistingImages() { + $res = $this->mDb->select( 'imagelinks', array( 'il_to' ), + array( 'il_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->il_to] = 1; + } + return $arr; + } + + /** + * Get an array of existing external links, URLs in the keys + * + * @return array + */ + private function getExistingExternals() { + $res = $this->mDb->select( 'externallinks', array( 'el_to' ), + array( 'el_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->el_to] = 1; + } + return $arr; + } + + /** + * Get an array of existing categories, with the name in the key and sort key in the value. + * + * @return array + */ + private function getExistingCategories() { + $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey_prefix' ), + array( 'cl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->cl_to] = $row->cl_sortkey_prefix; + } + return $arr; + } + + /** + * Get an array of existing interlanguage links, with the language code in the key and the + * title in the value. + * + * @return array + */ + private function getExistingInterlangs() { + $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ), + array( 'll_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->ll_lang] = $row->ll_title; + } + return $arr; + } + + /** + * Get an array of existing inline interwiki links, as a 2-D array + * @return array (prefix => array(dbkey => 1)) + */ + protected function getExistingInterwikis() { + $res = $this->mDb->select( 'iwlinks', array( 'iwl_prefix', 'iwl_title' ), + array( 'iwl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + if ( !isset( $arr[$row->iwl_prefix] ) ) { + $arr[$row->iwl_prefix] = array(); + } + $arr[$row->iwl_prefix][$row->iwl_title] = 1; + } + return $arr; + } + + /** + * Get an array of existing categories, with the name in the key and sort key in the value. + * + * @return array + */ + private function getExistingProperties() { + $res = $this->mDb->select( 'page_props', array( 'pp_propname', 'pp_value' ), + array( 'pp_page' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->pp_propname] = $row->pp_value; + } + return $arr; + } + + /** + * Return the title object of the page being updated + * @return Title + */ + public function getTitle() { + return $this->mTitle; + } + + /** + * Returns parser output + * @since 1.19 + * @return ParserOutput + */ + public function getParserOutput() { + return $this->mParserOutput; + } + + /** + * Return the list of images used as generated by the parser + * @return array + */ + public function getImages() { + return $this->mImages; + } + + /** + * Invalidate any necessary link lists related to page property changes + * @param $changed + */ + private function invalidateProperties( $changed ) { + global $wgPagePropLinkInvalidations; + + foreach ( $changed as $name => $value ) { + if ( isset( $wgPagePropLinkInvalidations[$name] ) ) { + $inv = $wgPagePropLinkInvalidations[$name]; + if ( !is_array( $inv ) ) { + $inv = array( $inv ); + } + foreach ( $inv as $table ) { + $update = new HTMLCacheUpdate( $this->mTitle, $table ); + $update->doUpdate(); + } + } + } + } + + /** + * Fetch page links added by this LinksUpdate. Only available after the update is complete. + * @since 1.22 + * @return null|array of Titles + */ + public function getAddedLinks() { + if ( $this->linkInsertions === null ) { + return null; + } + $result = array(); + foreach ( $this->linkInsertions as $insertion ) { + $result[] = Title::makeTitle( $insertion[ 'pl_namespace' ], $insertion[ 'pl_title' ] ); + } + return $result; + } + + /** + * Fetch page links removed by this LinksUpdate. Only available after the update is complete. + * @since 1.22 + * @return null|array of Titles + */ + public function getRemovedLinks() { + if ( $this->linkDeletions === null ) { + return null; + } + $result = array(); + foreach ( $this->linkDeletions as $ns => $titles ) { + foreach ( $titles as $title => $unused ) { + $result[] = Title::makeTitle( $ns, $title ); + } + } + return $result; + } +} + +/** + * Update object handling the cleanup of links tables after a page was deleted. + **/ +class LinksDeletionUpdate extends SqlDataUpdate { + + protected $mPage; //!< WikiPage the wikipage that was deleted + + /** + * Constructor + * + * @param $page WikiPage Page we are updating + * @throws MWException + */ + function __construct( WikiPage $page ) { + parent::__construct( false ); // no implicit transaction + + $this->mPage = $page; + + if ( !$page->exists() ) { + throw new MWException( "Page ID not known, perhaps the page doesn't exist?" ); + } + } + + /** + * Do some database updates after deletion + */ + public function doUpdate() { + $title = $this->mPage->getTitle(); + $id = $this->mPage->getId(); + + # Delete restrictions for it + $this->mDb->delete( 'page_restrictions', array( 'pr_page' => $id ), __METHOD__ ); + + # Fix category table counts + $cats = array(); + $res = $this->mDb->select( 'categorylinks', 'cl_to', array( 'cl_from' => $id ), __METHOD__ ); + + foreach ( $res as $row ) { + $cats[] = $row->cl_to; + } + + $this->mPage->updateCategoryCounts( array(), $cats ); + + # If using cascading deletes, we can skip some explicit deletes + if ( !$this->mDb->cascadingDeletes() ) { + # Delete outgoing links + $this->mDb->delete( 'pagelinks', array( 'pl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'imagelinks', array( 'il_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'categorylinks', array( 'cl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'templatelinks', array( 'tl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'externallinks', array( 'el_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'langlinks', array( 'll_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'iwlinks', array( 'iwl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'redirect', array( 'rd_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'page_props', array( 'pp_page' => $id ), __METHOD__ ); + } + + # If using cleanup triggers, we can skip some manual deletes + if ( !$this->mDb->cleanupTriggers() ) { + # Clean up recentchanges entries... + $this->mDb->delete( 'recentchanges', + array( 'rc_type != ' . RC_LOG, + 'rc_namespace' => $title->getNamespace(), + 'rc_title' => $title->getDBkey() ), + __METHOD__ ); + $this->mDb->delete( 'recentchanges', + array( 'rc_type != ' . RC_LOG, 'rc_cur_id' => $id ), + __METHOD__ ); + } + } + + /** + * Update all the appropriate counts in the category table. + * @param array $added associative array of category name => sort key + * @param array $deleted associative array of category name => sort key + */ + function updateCategoryCounts( $added, $deleted ) { + $a = WikiPage::factory( $this->mTitle ); + $a->updateCategoryCounts( + array_keys( $added ), array_keys( $deleted ) + ); + } +} diff --git a/includes/deferred/SearchUpdate.php b/includes/deferred/SearchUpdate.php new file mode 100644 index 0000000000..82a413e920 --- /dev/null +++ b/includes/deferred/SearchUpdate.php @@ -0,0 +1,185 @@ +id = $id; + // is_string() check is back-compat for ApprovedRevs + if ( is_string( $c ) ) { + $this->content = new TextContent( $c ); + } else { + $this->content = $c ?: false; + } + $this->title = $nt; + } else { + wfDebug( "SearchUpdate object created with invalid title '$title'\n" ); + } + } + + /** + * Perform actual update for the entry + */ + public function doUpdate() { + global $wgDisableSearchUpdate; + + if ( $wgDisableSearchUpdate || !$this->id ) { + return; + } + + wfProfileIn( __METHOD__ ); + + $page = WikiPage::newFromId( $this->id, WikiPage::READ_LATEST ); + $indexTitle = Title::indexTitle( $this->title->getNamespace(), $this->title->getText() ); + + foreach ( SearchEngine::getSearchTypes() as $type ) { + $search = SearchEngine::create( $type ); + if ( !$search->supports( 'search-update' ) ) { + continue; + } + + $normalTitle = $search->normalizeText( $indexTitle ); + + if ( $page === null ) { + $search->delete( $this->id, $normalTitle ); + continue; + } elseif ( $this->content === false ) { + $search->updateTitle( $this->id, $normalTitle ); + continue; + } + + $text = $search->getTextFromContent( $this->title, $this->content ); + if ( !$search->textAlreadyUpdatedForIndex() ) { + $text = self::updateText( $text ); + } + + # Perform the actual update + $search->update( $this->id, $normalTitle, $search->normalizeText( $text ) ); + } + + wfProfileOut( __METHOD__ ); + } + + /** + * Clean text for indexing. Only really suitable for indexing in databases. + * If you're using a real search engine, you'll probably want to override + * this behavior and do something nicer with the original wikitext. + */ + public static function updateText( $text ) { + global $wgContLang; + + # Language-specific strip/conversion + $text = $wgContLang->normalizeForSearch( $text ); + $lc = SearchEngine::legalSearchChars() . '&#;'; + + wfProfileIn( __METHOD__ . '-regexps' ); + $text = preg_replace( "/<\\/?\\s*[A-Za-z][^>]*?>/", + ' ', $wgContLang->lc( " " . $text . " " ) ); # Strip HTML markup + $text = preg_replace( "/(^|\\n)==\\s*([^\\n]+)\\s*==(\\s)/sD", + "\\1\\2 \\2 \\2\\3", $text ); # Emphasize headings + + # Strip external URLs + $uc = "A-Za-z0-9_\\/:.,~%\\-+&;#?!=()@\\x80-\\xFF"; + $protos = "http|https|ftp|mailto|news|gopher"; + $pat = "/(^|[^\\[])({$protos}):[{$uc}]+([^{$uc}]|$)/"; + $text = preg_replace( $pat, "\\1 \\3", $text ); + + $p1 = "/([^\\[])\\[({$protos}):[{$uc}]+]/"; + $p2 = "/([^\\[])\\[({$protos}):[{$uc}]+\\s+([^\\]]+)]/"; + $text = preg_replace( $p1, "\\1 ", $text ); + $text = preg_replace( $p2, "\\1 \\3 ", $text ); + + # Internal image links + $pat2 = "/\\[\\[image:([{$uc}]+)\\.(gif|png|jpg|jpeg)([^{$uc}])/i"; + $text = preg_replace( $pat2, " \\1 \\3", $text ); + + $text = preg_replace( "/([^{$lc}])([{$lc}]+)]]([a-z]+)/", + "\\1\\2 \\2\\3", $text ); # Handle [[game]]s + + # Strip all remaining non-search characters + $text = preg_replace( "/[^{$lc}]+/", " ", $text ); + + # Handle 's, s' + # + # $text = preg_replace( "/([{$lc}]+)'s /", "\\1 \\1's ", $text ); + # $text = preg_replace( "/([{$lc}]+)s' /", "\\1s ", $text ); + # + # These tail-anchored regexps are insanely slow. The worst case comes + # when Japanese or Chinese text (ie, no word spacing) is written on + # a wiki configured for Western UTF-8 mode. The Unicode characters are + # expanded to hex codes and the "words" are very long paragraph-length + # monstrosities. On a large page the above regexps may take over 20 + # seconds *each* on a 1GHz-level processor. + # + # Following are reversed versions which are consistently fast + # (about 3 milliseconds on 1GHz-level processor). + # + $text = strrev( preg_replace( "/ s'([{$lc}]+)/", " s'\\1 \\1", strrev( $text ) ) ); + $text = strrev( preg_replace( "/ 's([{$lc}]+)/", " s\\1", strrev( $text ) ) ); + + # Strip wiki '' and ''' + $text = preg_replace( "/''[']*/", " ", $text ); + wfProfileOut( __METHOD__ . '-regexps' ); + return $text; + } +} diff --git a/includes/deferred/SiteStatsUpdate.php b/includes/deferred/SiteStatsUpdate.php new file mode 100644 index 0000000000..09ff87dd3b --- /dev/null +++ b/includes/deferred/SiteStatsUpdate.php @@ -0,0 +1,245 @@ +views = $views; + $this->edits = $edits; + $this->articles = $good; + $this->pages = $pages; + $this->users = $users; + } + + /** + * @param $deltas Array + * @return SiteStatsUpdate + */ + public static function factory( array $deltas ) { + $update = new self( 0, 0, 0 ); + + $fields = array( 'views', 'edits', 'pages', 'articles', 'users', 'images' ); + foreach ( $fields as $field ) { + if ( isset( $deltas[$field] ) && $deltas[$field] ) { + $update->$field = $deltas[$field]; + } + } + + return $update; + } + + public function doUpdate() { + global $wgSiteStatsAsyncFactor; + + $rate = $wgSiteStatsAsyncFactor; // convenience + // If set to do so, only do actual DB updates 1 every $rate times. + // The other times, just update "pending delta" values in memcached. + if ( $rate && ( $rate < 0 || mt_rand( 0, $rate - 1 ) != 0 ) ) { + $this->doUpdatePendingDeltas(); + } else { + // Need a separate transaction because this a global lock + wfGetDB( DB_MASTER )->onTransactionIdle( array( $this, 'tryDBUpdateInternal' ) ); + } + } + + /** + * Do not call this outside of SiteStatsUpdate + * + * @return void + */ + public function tryDBUpdateInternal() { + global $wgSiteStatsAsyncFactor; + + $dbw = wfGetDB( DB_MASTER ); + $lockKey = wfMemcKey( 'site_stats' ); // prepend wiki ID + if ( $wgSiteStatsAsyncFactor ) { + // Lock the table so we don't have double DB/memcached updates + if ( !$dbw->lockIsFree( $lockKey, __METHOD__ ) + || !$dbw->lock( $lockKey, __METHOD__, 1 ) // 1 sec timeout + ) { + $this->doUpdatePendingDeltas(); + return; + } + $pd = $this->getPendingDeltas(); + // Piggy-back the async deltas onto those of this stats update.... + $this->views += ( $pd['ss_total_views']['+'] - $pd['ss_total_views']['-'] ); + $this->edits += ( $pd['ss_total_edits']['+'] - $pd['ss_total_edits']['-'] ); + $this->articles += ( $pd['ss_good_articles']['+'] - $pd['ss_good_articles']['-'] ); + $this->pages += ( $pd['ss_total_pages']['+'] - $pd['ss_total_pages']['-'] ); + $this->users += ( $pd['ss_users']['+'] - $pd['ss_users']['-'] ); + $this->images += ( $pd['ss_images']['+'] - $pd['ss_images']['-'] ); + } + + // Build up an SQL query of deltas and apply them... + $updates = ''; + $this->appendUpdate( $updates, 'ss_total_views', $this->views ); + $this->appendUpdate( $updates, 'ss_total_edits', $this->edits ); + $this->appendUpdate( $updates, 'ss_good_articles', $this->articles ); + $this->appendUpdate( $updates, 'ss_total_pages', $this->pages ); + $this->appendUpdate( $updates, 'ss_users', $this->users ); + $this->appendUpdate( $updates, 'ss_images', $this->images ); + if ( $updates != '' ) { + $dbw->update( 'site_stats', array( $updates ), array(), __METHOD__ ); + } + + if ( $wgSiteStatsAsyncFactor ) { + // Decrement the async deltas now that we applied them + $this->removePendingDeltas( $pd ); + // Commit the updates and unlock the table + $dbw->unlock( $lockKey, __METHOD__ ); + } + } + + /** + * @param $dbw DatabaseBase + * @return bool|mixed + */ + public static function cacheUpdate( $dbw ) { + global $wgActiveUserDays; + $dbr = wfGetDB( DB_SLAVE, array( 'SpecialStatistics', 'vslow' ) ); + # Get non-bot users than did some recent action other than making accounts. + # If account creation is included, the number gets inflated ~20+ fold on enwiki. + $activeUsers = $dbr->selectField( + 'recentchanges', + 'COUNT( DISTINCT rc_user_text )', + array( + 'rc_user != 0', + 'rc_bot' => 0, + 'rc_log_type != ' . $dbr->addQuotes( 'newusers' ) . ' OR rc_log_type IS NULL', + 'rc_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( wfTimestamp( TS_UNIX ) - $wgActiveUserDays * 24 * 3600 ) ), + ), + __METHOD__ + ); + $dbw->update( + 'site_stats', + array( 'ss_active_users' => intval( $activeUsers ) ), + array( 'ss_row_id' => 1 ), + __METHOD__ + ); + return $activeUsers; + } + + protected function doUpdatePendingDeltas() { + $this->adjustPending( 'ss_total_views', $this->views ); + $this->adjustPending( 'ss_total_edits', $this->edits ); + $this->adjustPending( 'ss_good_articles', $this->articles ); + $this->adjustPending( 'ss_total_pages', $this->pages ); + $this->adjustPending( 'ss_users', $this->users ); + $this->adjustPending( 'ss_images', $this->images ); + } + + /** + * @param $sql string + * @param $field string + * @param $delta integer + */ + protected function appendUpdate( &$sql, $field, $delta ) { + if ( $delta ) { + if ( $sql ) { + $sql .= ','; + } + if ( $delta < 0 ) { + $sql .= "$field=$field-" . abs( $delta ); + } else { + $sql .= "$field=$field+" . abs( $delta ); + } + } + } + + /** + * @param $type string + * @param string $sign ('+' or '-') + * @return string + */ + private function getTypeCacheKey( $type, $sign ) { + return wfMemcKey( 'sitestatsupdate', 'pendingdelta', $type, $sign ); + } + + /** + * Adjust the pending deltas for a stat type. + * Each stat type has two pending counters, one for increments and decrements + * @param $type string + * @param $delta integer Delta (positive or negative) + * @return void + */ + protected function adjustPending( $type, $delta ) { + global $wgMemc; + + if ( $delta < 0 ) { // decrement + $key = $this->getTypeCacheKey( $type, '-' ); + } else { // increment + $key = $this->getTypeCacheKey( $type, '+' ); + } + + $magnitude = abs( $delta ); + if ( !$wgMemc->incr( $key, $magnitude ) ) { // not there? + if ( !$wgMemc->add( $key, $magnitude ) ) { // race? + $wgMemc->incr( $key, $magnitude ); + } + } + } + + /** + * Get pending delta counters for each stat type + * @return Array Positive and negative deltas for each type + * @return void + */ + protected function getPendingDeltas() { + global $wgMemc; + + $pending = array(); + foreach ( array( 'ss_total_views', 'ss_total_edits', + 'ss_good_articles', 'ss_total_pages', 'ss_users', 'ss_images' ) as $type ) + { + // Get pending increments and pending decrements + $pending[$type]['+'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '+' ) ); + $pending[$type]['-'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '-' ) ); + } + + return $pending; + } + + /** + * Reduce pending delta counters after updates have been applied + * @param array $pd Result of getPendingDeltas(), used for DB update + * @return void + */ + protected function removePendingDeltas( array $pd ) { + global $wgMemc; + + foreach ( $pd as $type => $deltas ) { + foreach ( $deltas as $sign => $magnitude ) { + // Lower the pending counter now that we applied these changes + $wgMemc->decr( $this->getTypeCacheKey( $type, $sign ), $magnitude ); + } + } + } +} + diff --git a/includes/deferred/SqlDataUpdate.php b/includes/deferred/SqlDataUpdate.php new file mode 100644 index 0000000000..51188d85c9 --- /dev/null +++ b/includes/deferred/SqlDataUpdate.php @@ -0,0 +1,152 @@ +mOptions = array(); + } else { + $this->mOptions = array( 'FOR UPDATE' ); + } + + // @todo get connection only when it's needed? make sure that doesn't break anything, especially transactions! + $this->mDb = wfGetDB( DB_MASTER ); + + $this->mWithTransaction = $withTransaction; + $this->mHasTransaction = false; + } + + /** + * Begin a database transaction, if $withTransaction was given as true in the constructor for this SqlDataUpdate. + * + * Because nested transactions are not supported by the Database class, this implementation + * checks Database::trxLevel() and only opens a transaction if none is already active. + */ + public function beginTransaction() { + if ( !$this->mWithTransaction ) { + return; + } + + // NOTE: nested transactions are not supported, only start a transaction if none is open + if ( $this->mDb->trxLevel() === 0 ) { + $this->mDb->begin( get_class( $this ) . '::beginTransaction' ); + $this->mHasTransaction = true; + } + } + + /** + * Commit the database transaction started via beginTransaction (if any). + */ + public function commitTransaction() { + if ( $this->mHasTransaction ) { + $this->mDb->commit( get_class( $this ) . '::commitTransaction' ); + $this->mHasTransaction = false; + } + } + + /** + * Abort the database transaction started via beginTransaction (if any). + */ + public function abortTransaction() { + if ( $this->mHasTransaction ) { //XXX: actually... maybe always? + $this->mDb->rollback( get_class( $this ) . '::abortTransaction' ); + $this->mHasTransaction = false; + } + } + + /** + * Invalidate the cache of a list of pages from a single namespace. + * This is intended for use by subclasses. + * + * @param $namespace Integer + * @param $dbkeys Array + */ + protected function invalidatePages( $namespace, array $dbkeys ) { + if ( $dbkeys === array() ) { + return; + } + + /** + * Determine which pages need to be updated + * This is necessary to prevent the job queue from smashing the DB with + * large numbers of concurrent invalidations of the same page + */ + $now = $this->mDb->timestamp(); + $ids = array(); + $res = $this->mDb->select( 'page', array( 'page_id' ), + array( + 'page_namespace' => $namespace, + 'page_title' => $dbkeys, + 'page_touched < ' . $this->mDb->addQuotes( $now ) + ), __METHOD__ + ); + + foreach ( $res as $row ) { + $ids[] = $row->page_id; + } + + if ( $ids === array() ) { + return; + } + + /** + * Do the update + * We still need the page_touched condition, in case the row has changed since + * the non-locking select above. + */ + $this->mDb->update( 'page', array( 'page_touched' => $now ), + array( + 'page_id' => $ids, + 'page_touched < ' . $this->mDb->addQuotes( $now ) + ), __METHOD__ + ); + } + +} diff --git a/includes/deferred/SquidUpdate.php b/includes/deferred/SquidUpdate.php new file mode 100644 index 0000000000..71afeba924 --- /dev/null +++ b/includes/deferred/SquidUpdate.php @@ -0,0 +1,300 @@ + $maxTitles ) { + // Truncate to desired maximum URL count + $urlArr = array_slice( $urlArr, 0, $maxTitles ); + } + $this->urlArr = $urlArr; + } + + /** + * Create a SquidUpdate from the given Title object. + * + * The resulting SquidUpdate will purge the given Title's URLs as well as + * the pages that link to it. Capped at $wgMaxSquidPurgeTitles total URLs. + * + * @param Title $title + * @return SquidUpdate + */ + public static function newFromLinksTo( Title $title ) { + global $wgMaxSquidPurgeTitles; + wfProfileIn( __METHOD__ ); + + # Get a list of URLs linking to this page + $dbr = wfGetDB( DB_SLAVE ); + $res = $dbr->select( array( 'links', 'page' ), + array( 'page_namespace', 'page_title' ), + array( + 'pl_namespace' => $title->getNamespace(), + 'pl_title' => $title->getDBkey(), + 'pl_from=page_id' ), + __METHOD__ ); + $blurlArr = $title->getSquidURLs(); + if ( $res->numRows() <= $wgMaxSquidPurgeTitles ) { + foreach ( $res as $BL ) { + $tobj = Title::makeTitle( $BL->page_namespace, $BL->page_title ); + $blurlArr[] = $tobj->getInternalURL(); + } + } + + wfProfileOut( __METHOD__ ); + return new SquidUpdate( $blurlArr ); + } + + /** + * Create a SquidUpdate from an array of Title objects, or a TitleArray object + * + * @param array $titles + * @param array $urlArr + * @return SquidUpdate + */ + public static function newFromTitles( $titles, $urlArr = array() ) { + global $wgMaxSquidPurgeTitles; + $i = 0; + foreach ( $titles as $title ) { + $urlArr[] = $title->getInternalURL(); + if ( $i++ > $wgMaxSquidPurgeTitles ) { + break; + } + } + return new SquidUpdate( $urlArr ); + } + + /** + * @param Title $title + * @return SquidUpdate + */ + public static function newSimplePurge( Title $title ) { + $urlArr = $title->getSquidURLs(); + return new SquidUpdate( $urlArr ); + } + + /** + * Purges the list of URLs passed to the constructor. + */ + public function doUpdate() { + self::purge( $this->urlArr ); + } + + /** + * Purges a list of Squids defined in $wgSquidServers. + * $urlArr should contain the full URLs to purge as values + * (example: $urlArr[] = 'http://my.host/something') + * XXX report broken Squids per mail or log + * + * @param array $urlArr List of full URLs to purge + */ + public static function purge( $urlArr ) { + global $wgSquidServers, $wgHTCPRouting; + + if ( !$urlArr ) { + return; + } + + wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) . "\n" ); + + if ( $wgHTCPRouting ) { + self::HTCPPurge( $urlArr ); + } + + wfProfileIn( __METHOD__ ); + + // Remove duplicate URLs + $urlArr = array_unique( $urlArr ); + // Maximum number of parallel connections per squid + $maxSocketsPerSquid = 8; + // Number of requests to send per socket + // 400 seems to be a good tradeoff, opening a socket takes a while + $urlsPerSocket = 400; + $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket ); + if ( $socketsPerSquid > $maxSocketsPerSquid ) { + $socketsPerSquid = $maxSocketsPerSquid; + } + + $pool = new SquidPurgeClientPool; + $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) ); + foreach ( $wgSquidServers as $server ) { + foreach ( $chunks as $chunk ) { + $client = new SquidPurgeClient( $server ); + foreach ( $chunk as $url ) { + $client->queuePurge( $url ); + } + $pool->addClient( $client ); + } + } + $pool->run(); + + wfProfileOut( __METHOD__ ); + } + + /** + * Send Hyper Text Caching Protocol (HTCP) CLR requests. + * + * @throws MWException + * @param array $urlArr Collection of URLs to purge + */ + public static function HTCPPurge( $urlArr ) { + global $wgHTCPRouting, $wgHTCPMulticastTTL; + wfProfileIn( __METHOD__ ); + + // HTCP CLR operation + $htcpOpCLR = 4; + + // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h) + if ( !defined( "IPPROTO_IP" ) ) { + define( "IPPROTO_IP", 0 ); + define( "IP_MULTICAST_LOOP", 34 ); + define( "IP_MULTICAST_TTL", 33 ); + } + + // pfsockopen doesn't work because we need set_sock_opt + $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP ); + if ( ! $conn ) { + $errstr = socket_strerror( socket_last_error() ); + wfDebugLog( 'squid', __METHOD__ . + ": Error opening UDP socket: $errstr\n" ); + wfProfileOut( __METHOD__ ); + return; + } + + // Set socket options + socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 ); + if ( $wgHTCPMulticastTTL != 1 ) { + // Set multicast time to live (hop count) option on socket + socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL, + $wgHTCPMulticastTTL ); + } + + // Remove duplicate URLs from collection + $urlArr = array_unique( $urlArr ); + foreach ( $urlArr as $url ) { + if ( !is_string( $url ) ) { + wfProfileOut( __METHOD__ ); + throw new MWException( 'Bad purge URL' ); + } + $url = self::expand( $url ); + $conf = self::getRuleForURL( $url, $wgHTCPRouting ); + if ( !$conf ) { + wfDebugLog( 'squid', __METHOD__ . + "No HTCP rule configured for URL {$url} , skipping\n" ); + continue; + } + + if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) { + // Normalize single entries + $conf = array( $conf ); + } + foreach ( $conf as $subconf ) { + if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) { + wfProfileOut( __METHOD__ ); + throw new MWException( "Invalid HTCP rule for URL $url\n" ); + } + } + + // Construct a minimal HTCP request diagram + // as per RFC 2756 + // Opcode 'CLR', no response desired, no auth + $htcpTransID = rand(); + + $htcpSpecifier = pack( 'na4na*na8n', + 4, 'HEAD', strlen( $url ), $url, + 8, 'HTTP/1.0', 0 ); + + $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier ); + $htcpLen = 4 + $htcpDataLen + 2; + + // Note! Squid gets the bit order of the first + // word wrong, wrt the RFC. Apparently no other + // implementation exists, so adapt to Squid + $htcpPacket = pack( 'nxxnCxNxxa*n', + $htcpLen, $htcpDataLen, $htcpOpCLR, + $htcpTransID, $htcpSpecifier, 2 ); + + wfDebugLog( 'squid', __METHOD__ . + "Purging URL $url via HTCP\n" ); + foreach ( $conf as $subconf ) { + socket_sendto( $conn, $htcpPacket, $htcpLen, 0, + $subconf['host'], $subconf['port'] ); + } + } + wfProfileOut( __METHOD__ ); + } + + /** + * Expand local URLs to fully-qualified URLs using the internal protocol + * and host defined in $wgInternalServer. Input that's already fully- + * qualified will be passed through unchanged. + * + * This is used to generate purge URLs that may be either local to the + * main wiki or include a non-native host, such as images hosted on a + * second internal server. + * + * Client functions should not need to call this. + * + * @param string $url + * @return string + */ + public static function expand( $url ) { + return wfExpandUrl( $url, PROTO_INTERNAL ); + } + + /** + * Find the HTCP routing rule to use for a given URL. + * @param string $url URL to match + * @param array $rules Array of rules, see $wgHTCPRouting for format and behavior + * @return mixed Element of $rules that matched, or false if nothing matched + */ + private static function getRuleForURL( $url, $rules ) { + foreach ( $rules as $regex => $routing ) { + if ( $regex === '' || preg_match( $regex, $url ) ) { + return $routing; + } + } + return false; + } +} diff --git a/includes/deferred/ViewCountUpdate.php b/includes/deferred/ViewCountUpdate.php new file mode 100644 index 0000000000..22a4649364 --- /dev/null +++ b/includes/deferred/ViewCountUpdate.php @@ -0,0 +1,105 @@ +id = intval( $id ); + } + + /** + * Run the update + */ + public function doUpdate() { + global $wgHitcounterUpdateFreq; + + $dbw = wfGetDB( DB_MASTER ); + + if ( $wgHitcounterUpdateFreq <= 1 || $dbw->getType() == 'sqlite' ) { + $dbw->update( 'page', array( 'page_counter = page_counter + 1' ), array( 'page_id' => $this->id ), __METHOD__ ); + return; + } + + # Not important enough to warrant an error page in case of failure + try { + $dbw->insert( 'hitcounter', array( 'hc_id' => $this->id ), __METHOD__ ); + $checkfreq = intval( $wgHitcounterUpdateFreq / 25 + 1 ); + if ( rand() % $checkfreq == 0 && $dbw->lastErrno() == 0 ) { + $this->collect(); + } + } catch ( DBError $e ) {} + } + + protected function collect() { + global $wgHitcounterUpdateFreq; + + $dbw = wfGetDB( DB_MASTER ); + + $rown = $dbw->selectField( 'hitcounter', 'COUNT(*)', array(), __METHOD__ ); + + if ( $rown < $wgHitcounterUpdateFreq ) { + return; + } + + wfProfileIn( __METHOD__ . '-collect' ); + $old_user_abort = ignore_user_abort( true ); + + $dbw->lockTables( array(), array( 'hitcounter' ), __METHOD__, false ); + + $dbType = $dbw->getType(); + $tabletype = $dbType == 'mysql' ? "ENGINE=HEAP " : ''; + $hitcounterTable = $dbw->tableName( 'hitcounter' ); + $acchitsTable = $dbw->tableName( 'acchits' ); + $pageTable = $dbw->tableName( 'page' ); + + $dbw->query( "CREATE TEMPORARY TABLE $acchitsTable $tabletype AS " . + "SELECT hc_id,COUNT(*) AS hc_n FROM $hitcounterTable " . + 'GROUP BY hc_id', __METHOD__ ); + $dbw->delete( 'hitcounter', '*', __METHOD__ ); + $dbw->unlockTables( __METHOD__ ); + + if ( $dbType == 'mysql' ) { + $dbw->query( "UPDATE $pageTable,$acchitsTable SET page_counter=page_counter + hc_n " . + 'WHERE page_id = hc_id', __METHOD__ ); + } else { + $dbw->query( "UPDATE $pageTable SET page_counter=page_counter + hc_n " . + "FROM $acchitsTable WHERE page_id = hc_id", __METHOD__ ); + } + $dbw->query( "DROP TABLE $acchitsTable", __METHOD__ ); + + ignore_user_abort( $old_user_abort ); + wfProfileOut( __METHOD__ . '-collect' ); + } +} diff --git a/includes/search/SearchUpdate.php b/includes/search/SearchUpdate.php deleted file mode 100644 index 82a413e920..0000000000 --- a/includes/search/SearchUpdate.php +++ /dev/null @@ -1,185 +0,0 @@ -id = $id; - // is_string() check is back-compat for ApprovedRevs - if ( is_string( $c ) ) { - $this->content = new TextContent( $c ); - } else { - $this->content = $c ?: false; - } - $this->title = $nt; - } else { - wfDebug( "SearchUpdate object created with invalid title '$title'\n" ); - } - } - - /** - * Perform actual update for the entry - */ - public function doUpdate() { - global $wgDisableSearchUpdate; - - if ( $wgDisableSearchUpdate || !$this->id ) { - return; - } - - wfProfileIn( __METHOD__ ); - - $page = WikiPage::newFromId( $this->id, WikiPage::READ_LATEST ); - $indexTitle = Title::indexTitle( $this->title->getNamespace(), $this->title->getText() ); - - foreach ( SearchEngine::getSearchTypes() as $type ) { - $search = SearchEngine::create( $type ); - if ( !$search->supports( 'search-update' ) ) { - continue; - } - - $normalTitle = $search->normalizeText( $indexTitle ); - - if ( $page === null ) { - $search->delete( $this->id, $normalTitle ); - continue; - } elseif ( $this->content === false ) { - $search->updateTitle( $this->id, $normalTitle ); - continue; - } - - $text = $search->getTextFromContent( $this->title, $this->content ); - if ( !$search->textAlreadyUpdatedForIndex() ) { - $text = self::updateText( $text ); - } - - # Perform the actual update - $search->update( $this->id, $normalTitle, $search->normalizeText( $text ) ); - } - - wfProfileOut( __METHOD__ ); - } - - /** - * Clean text for indexing. Only really suitable for indexing in databases. - * If you're using a real search engine, you'll probably want to override - * this behavior and do something nicer with the original wikitext. - */ - public static function updateText( $text ) { - global $wgContLang; - - # Language-specific strip/conversion - $text = $wgContLang->normalizeForSearch( $text ); - $lc = SearchEngine::legalSearchChars() . '&#;'; - - wfProfileIn( __METHOD__ . '-regexps' ); - $text = preg_replace( "/<\\/?\\s*[A-Za-z][^>]*?>/", - ' ', $wgContLang->lc( " " . $text . " " ) ); # Strip HTML markup - $text = preg_replace( "/(^|\\n)==\\s*([^\\n]+)\\s*==(\\s)/sD", - "\\1\\2 \\2 \\2\\3", $text ); # Emphasize headings - - # Strip external URLs - $uc = "A-Za-z0-9_\\/:.,~%\\-+&;#?!=()@\\x80-\\xFF"; - $protos = "http|https|ftp|mailto|news|gopher"; - $pat = "/(^|[^\\[])({$protos}):[{$uc}]+([^{$uc}]|$)/"; - $text = preg_replace( $pat, "\\1 \\3", $text ); - - $p1 = "/([^\\[])\\[({$protos}):[{$uc}]+]/"; - $p2 = "/([^\\[])\\[({$protos}):[{$uc}]+\\s+([^\\]]+)]/"; - $text = preg_replace( $p1, "\\1 ", $text ); - $text = preg_replace( $p2, "\\1 \\3 ", $text ); - - # Internal image links - $pat2 = "/\\[\\[image:([{$uc}]+)\\.(gif|png|jpg|jpeg)([^{$uc}])/i"; - $text = preg_replace( $pat2, " \\1 \\3", $text ); - - $text = preg_replace( "/([^{$lc}])([{$lc}]+)]]([a-z]+)/", - "\\1\\2 \\2\\3", $text ); # Handle [[game]]s - - # Strip all remaining non-search characters - $text = preg_replace( "/[^{$lc}]+/", " ", $text ); - - # Handle 's, s' - # - # $text = preg_replace( "/([{$lc}]+)'s /", "\\1 \\1's ", $text ); - # $text = preg_replace( "/([{$lc}]+)s' /", "\\1s ", $text ); - # - # These tail-anchored regexps are insanely slow. The worst case comes - # when Japanese or Chinese text (ie, no word spacing) is written on - # a wiki configured for Western UTF-8 mode. The Unicode characters are - # expanded to hex codes and the "words" are very long paragraph-length - # monstrosities. On a large page the above regexps may take over 20 - # seconds *each* on a 1GHz-level processor. - # - # Following are reversed versions which are consistently fast - # (about 3 milliseconds on 1GHz-level processor). - # - $text = strrev( preg_replace( "/ s'([{$lc}]+)/", " s'\\1 \\1", strrev( $text ) ) ); - $text = strrev( preg_replace( "/ 's([{$lc}]+)/", " s\\1", strrev( $text ) ) ); - - # Strip wiki '' and ''' - $text = preg_replace( "/''[']*/", " ", $text ); - wfProfileOut( __METHOD__ . '-regexps' ); - return $text; - } -}