From 87be24dbdf8235e381110393011b86e0f5be395f Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Wed, 29 Jan 2014 17:50:08 -0800 Subject: [PATCH] Made ActiveUsers use querycache and do staggered updates on view Bug: 41078 Change-Id: Id9c2377e0d2636fa9aaed01839678e2b8abe824e --- includes/DefaultSettings.php | 3 +- includes/specialpage/SpecialPageFactory.php | 5 +- includes/specials/SpecialActiveusers.php | 176 +++++++++++++++++--- 3 files changed, 159 insertions(+), 25 deletions(-) diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 326a1c2fba..d0ceafcec6 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -6162,7 +6162,8 @@ $wgJobQueueAggregator = array( * Expensive Querypages are already updated. */ $wgSpecialPageCacheUpdates = array( - 'Statistics' => array( 'SiteStatsUpdate', 'cacheUpdate' ) + 'Statistics' => array( 'SiteStatsUpdate', 'cacheUpdate' ), + 'Activeusers' => array( 'SpecialActiveUsers', 'cacheUpdate' ), ); /** diff --git a/includes/specialpage/SpecialPageFactory.php b/includes/specialpage/SpecialPageFactory.php index 792d0a64ef..dea65f3534 100644 --- a/includes/specialpage/SpecialPageFactory.php +++ b/includes/specialpage/SpecialPageFactory.php @@ -180,7 +180,6 @@ class SpecialPageFactory { global $wgSpecialPages; global $wgDisableCounters, $wgDisableInternalSearch, $wgEmailAuthentication; global $wgEnableEmail, $wgEnableJavaScriptTest; - global $wgMiserMode; if ( !is_object( self::$list ) ) { wfProfileIn( __METHOD__ ); @@ -206,9 +205,7 @@ class SpecialPageFactory { self::$list['JavaScriptTest'] = 'SpecialJavaScriptTest'; } - if ( !$wgMiserMode ) { - self::$list['Activeusers'] = 'SpecialActiveUsers'; - } + self::$list['Activeusers'] = 'SpecialActiveUsers'; // Add extension special pages self::$list = array_merge( self::$list, $wgSpecialPages ); diff --git a/includes/specials/SpecialActiveusers.php b/includes/specials/SpecialActiveusers.php index 705dab5514..b417548d7c 100644 --- a/includes/specials/SpecialActiveusers.php +++ b/includes/specials/SpecialActiveusers.php @@ -87,39 +87,31 @@ class ActiveUsersPager extends UsersPager { } function getIndexField() { - return 'rc_user_text'; + return 'qcc_title'; } function getQueryInfo() { $dbr = $this->getDatabase(); - $conds = array( 'rc_user > 0' ); // Users - no anons - $conds[] = 'rc_log_type IS NULL OR rc_log_type != ' . $dbr->addQuotes( 'newusers' ); - $conds[] = 'rc_timestamp >= ' . $dbr->addQuotes( - $dbr->timestamp( wfTimestamp( TS_UNIX ) - $this->RCMaxAge * 24 * 3600 ) ); - + $conds = array( + 'qcc_type' => 'activeusers', + 'qcc_namespace' => NS_USER, + 'user_name = qcc_title', + 'rc_user_text = qcc_title' + ); if ( $this->requestedUser != '' ) { - $conds[] = 'rc_user_text >= ' . $dbr->addQuotes( $this->requestedUser ); + $conds[] = 'qcc_title >= ' . $dbr->addQuotes( $this->requestedUser ); } - if ( !$this->getUser()->isAllowed( 'hideuser' ) ) { $conds[] = 'NOT EXISTS (' . $dbr->selectSQLText( - 'ipblocks', '1', array( 'rc_user=ipb_user', 'ipb_deleted' => 1 ) + 'ipblocks', '1', array( 'ipb_user=user_id', 'ipb_deleted' => 1 ) ) . ')'; } return array( - 'tables' => array( 'recentchanges' ), - 'fields' => array( - 'user_name' => 'rc_user_text', // for Pager inheritance - 'rc_user_text', // for Pager - 'user_id' => 'MAX(rc_user)', // Postgres - 'recentedits' => 'COUNT(*)' - ), - 'options' => array( - 'GROUP BY' => array( 'rc_user_text' ), - 'USE INDEX' => array( 'recentchanges' => 'rc_user_text' ) - ), + 'tables' => array( 'querycachetwo', 'user', 'recentchanges' ), + 'fields' => array( 'user_name', 'user_id', 'recentedits' => 'COUNT(*)', 'qcc_title' ), + 'options' => array( 'GROUP BY' => array( 'qcc_title' ) ), 'conds' => $conds ); } @@ -249,6 +241,12 @@ class SpecialActiveUsers extends SpecialPage { $out->wrapWikiMsg( "
\n$1\n
", array( 'activeusers-intro', $this->getLanguage()->formatNum( $wgActiveUserDays ) ) ); + // Occasionally merge in new updates + $seconds = self::mergeActiveUsers( 600 ); + // Mention the level of staleness + $out->addWikiMsg( 'cachedspecial-viewing-cached-ttl', + $this->getLanguage()->formatDuration( $seconds ) ); + $up = new ActiveUsersPager( $this->getContext(), null, $par ); # getBody() first to check, if empty @@ -269,4 +267,142 @@ class SpecialActiveUsers extends SpecialPage { protected function getGroupName() { return 'users'; } + + /** + * @param integer $period Seconds (do updates no more often than this) + * @return integer How many seconds old the cache is + */ + public static function mergeActiveUsers( $period ) { + global $wgActiveUserDays; + + $dbr = wfGetDB( DB_SLAVE ); + $cTime = $dbr->selectField( 'querycache_info', + 'qci_timestamp', + array( 'qci_type' => 'activeusers' ) + ); + if ( !wfReadOnly() ) { + if ( !$cTime || ( time() - wfTimestamp( TS_UNIX, $cTime ) ) > $period ) { + $dbw = wfGetDB( DB_MASTER ); + self::doQueryCacheUpdate( $dbw, 2 * $period ); + } + } + return ( time() - + ( $cTime ? wfTimestamp( TS_UNIX, $cTime ) : $wgActiveUserDays * 86400 ) ); + } + + /** + * @param DatabaseBase $dbw Passed in from updateSpecialPages.php + * @return void + */ + public static function cacheUpdate( DatabaseBase $dbw ) { + global $wgActiveUserDays; + + self::doQueryCacheUpdate( $dbw, $wgActiveUserDays * 86400 ); + } + + /** + * Update the query cache as needed + * + * @param DatabaseBase $dbw + * @param integer $window Maximum time range of new data to scan (in seconds) + * @return bool Success + */ + protected static function doQueryCacheUpdate( DatabaseBase $dbw, $window ) { + global $wgActiveUserDays; + + $lockKey = wfWikiID() . '-activeusers'; + if ( !$dbw->lock( $lockKey, __METHOD__, 1 ) ) { + return false; // exclusive update (avoids duplicate entries) + } + + $now = time(); + $cTime = $dbw->selectField( 'querycache_info', + 'qci_timestamp', + array( 'qci_type' => 'activeusers' ) + ); + $cTimeUnix = $cTime ? wfTimestamp( TS_UNIX, $cTime ) : 1; + + // Pick the date range to fetch from. This is normally from the last + // update to till the present time, but has a limited window for sanity. + // If the window is limited, multiple runs are need to fully populate it. + $sTimestamp = max( $cTimeUnix, $now - $wgActiveUserDays * 86400 ); + $eTimestamp = min( $sTimestamp + $window, $now ); + + // Get all the users active since the last update + $res = $dbw->select( + array( 'recentchanges' ), + array( 'rc_user_text', 'lastedittime' => 'MAX(rc_timestamp)' ), + array( + 'rc_user > 0', // actual accounts + 'rc_log_type IS NULL OR rc_log_type != ' . $dbw->addQuotes( 'newusers' ), + 'rc_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $sTimestamp ) ), + 'rc_timestamp <= ' . $dbw->addQuotes( $dbw->timestamp( $eTimestamp ) ) + ), + __METHOD__, + array( + 'GROUP BY' => array( 'rc_user_text' ), + 'ORDER BY' => 'NULL' // avoid filesort + ) + ); + $names = array(); + foreach ( $res as $row ) { + $names[$row->rc_user_text] = $row->lastedittime; + } + + // Rotate out users that have not edited in too long (according to old data set) + $dbw->delete( 'querycachetwo', + array( + 'qcc_type' => 'activeusers', + 'qcc_value < ' . $dbw->addQuotes( + $dbw->timestamp( $now - $wgActiveUserDays * 86400 ) ) + ), + __METHOD__ + ); + + // Find which of the recently active users are already accounted for + if ( count( $names ) ) { + $res = $dbw->select( 'querycachetwo', + array( 'user_name' => 'qcc_title' ), + array( + 'qcc_type' => 'activeusers', + 'qcc_namespace' => NS_USER, + 'qcc_title' => array_keys( $names ) ), + __METHOD__ + ); + foreach ( $res as $row ) { + unset( $names[$row->user_name] ); + } + } + + // Insert the users that need to be added to the list (which their last edit time + if ( count( $names ) ) { + $newRows = array(); + foreach ( $names as $name => $lastEditTime ) { + $newRows[] = array( + 'qcc_type' => 'activeusers', + 'qcc_namespace' => NS_USER, + 'qcc_title' => $name, + 'qcc_value' => $lastEditTime, + 'qcc_namespacetwo' => 0, // unused + 'qcc_titletwo' => '' // unused + ); + } + foreach ( array_chunk( $newRows, 500 ) as $rowBatch ) { + $dbw->insert( 'querycachetwo', $rowBatch, __METHOD__ ); + wfWaitForSlaves(); + } + } + + // Touch the data freshness timestamp + $dbw->replace( 'querycache_info', + array( 'qci_type' ), + array( 'qci_type' => 'activeusers', + 'qci_timestamp' => $dbw->timestamp( $eTimestamp ) ), // not always $now + __METHOD__ + ); + + $dbw->unlock( $lockKey, __METHOD__ ); + + return true; + } } -- 2.20.1