From: Ori Livneh Date: Fri, 20 Feb 2015 08:23:18 +0000 (-0800) Subject: Add StatsD metric logging X-Git-Tag: 1.31.0-rc.0~12149^2 X-Git-Url: https://git.cyclocoop.org/admin/?a=commitdiff_plain;h=87dfc20b1e19491f90ecd2c8ee82268c244f92c7;p=lhc%2Fweb%2Fwiklou.git Add StatsD metric logging This patch adds a metric data service object to the IContextSource interface, with full support for StatsD meters, gauges, counters and timing metrics, via the liuggio/statsd-php-client, which this patch also introduces. Usage example: $stats = $context->getStats(); $stats->increment( 'resourceloader.cache.hits' ); $stats->timing( 'resourceloader.cache.rtt', $rtt ); The metrics are flushed to a StatsD server, which may be specified via the 'StatsdServer' configuration key. If no such configuration key exists, the metrics are discarded. The StatsD client supplants MediaWiki's StatCounter class. wfIncrStats() will continue to work, but it will delegate to the StatsD data object. Change-Id: Ie10db1c154d225971398e189737de7c560bf0f90 --- diff --git a/RELEASE-NOTES-1.25 b/RELEASE-NOTES-1.25 index efaaafabf7..ddb8153830 100644 --- a/RELEASE-NOTES-1.25 +++ b/RELEASE-NOTES-1.25 @@ -111,6 +111,8 @@ production. used for conditional registration of API modules. * New hook 'EnhancedChangesList::getLogText' to alter, remove or add to the links of a group of changes in EnhancedChangesList. +* A full interface for StatsD metric reporting has been added to the context + interface, reachable via IContextSource::getStats(). ==== External libraries ==== * MediaWiki now requires certain external libraries to be installed. In the past @@ -137,6 +139,8 @@ production. This library was formerly a part of MediaWiki core, and has been moved into a separate library. It provides CDB functions which are used in the Interwiki and Localization caches. More information about the library can be found at https://www.mediawiki.org/wiki/CDB. +** liuggio/statsd-php-client + This library provides a StatsD client API for logging application metrics to a remote server. === Bug fixes in 1.25 === * (T73003) No additional code will be generated to try to load CSS-embedded diff --git a/autoload.php b/autoload.php index faf8252481..c700dec1d0 100644 --- a/autoload.php +++ b/autoload.php @@ -174,6 +174,7 @@ $wgAutoloadLocalClasses = array( 'BloomCacheRedis' => __DIR__ . '/includes/cache/bloom/BloomCacheRedis.php', 'BloomFilterTitleHasLogs' => __DIR__ . '/includes/cache/bloom/BloomFilters.php', 'BmpHandler' => __DIR__ . '/includes/media/BMP.php', + 'BufferingStatsdDataFactory' => __DIR__ . '/includes/libs/BufferingStatsdDataFactory.php', 'BrokenRedirectsPage' => __DIR__ . '/includes/specials/SpecialBrokenRedirects.php', 'CLDRPluralRuleConverter' => __DIR__ . '/languages/utils/CLDRPluralRuleConverter.php', 'CLDRPluralRuleConverterExpression' => __DIR__ . '/languages/utils/CLDRPluralRuleConverterExpression.php', @@ -1160,7 +1161,6 @@ $wgAutoloadLocalClasses = array( 'SquidPurgeClientPool' => __DIR__ . '/includes/SquidPurgeClient.php', 'SquidUpdate' => __DIR__ . '/includes/deferred/SquidUpdate.php', 'SrConverter' => __DIR__ . '/languages/classes/LanguageSr.php', - 'StatCounter' => __DIR__ . '/includes/StatCounter.php', 'StatsOutput' => __DIR__ . '/maintenance/language/StatOutputs.php', 'Status' => __DIR__ . '/includes/Status.php', 'StatusValue' => __DIR__ . '/includes/libs/StatusValue.php', diff --git a/composer.json b/composer.json index 9e32a0cef6..5b37a9240d 100644 --- a/composer.json +++ b/composer.json @@ -19,6 +19,7 @@ "cssjanus/cssjanus": "1.1.1", "ext-iconv": "*", "leafo/lessphp": "0.5.0", + "liuggio/statsd-php-client": "1.0.12", "oojs/oojs-ui": "0.9.0", "php": ">=5.3.3", "psr/log": "1.0.0", diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php index dfced1c0bb..ace52e0189 100644 --- a/includes/GlobalFunctions.php +++ b/includes/GlobalFunctions.php @@ -24,6 +24,9 @@ if ( !defined( 'MEDIAWIKI' ) ) { die( "This file is part of MediaWiki, it is not a valid entry point" ); } +use Liuggio\StatsdClient\StatsdClient; +use Liuggio\StatsdClient\Sender\SocketSender; + // Hide compatibility functions from Doxygen /// @cond @@ -1271,7 +1274,16 @@ function wfLogProfilingData() { global $wgRequestTime, $wgDebugLogGroups, $wgDebugRawPage; global $wgProfileLimit, $wgUser, $wgRequest; - StatCounter::singleton()->flush(); + $context = RequestContext::getMain(); + $config = $context->getConfig(); + if ( $config->has( 'StatsdServer' ) ) { + $statsdServer = explode( ':', $config->get( 'StatsdServer' ) ); + $statsdHost = $statsdServer[0]; + $statsdPort = isset( $statsdServer[1] ) ? $statsdServer[1] : 8125; + $statsdSender = new SocketSender( $statsdHost, $statsdPort ); + $statsdClient = new StatsdClient( $statsdSender ); + $statsdClient->send( $context->getStats()->getBuffer() ); + } $profiler = Profiler::instance(); @@ -1346,7 +1358,8 @@ function wfLogProfilingData() { * @return void */ function wfIncrStats( $key, $count = 1 ) { - StatCounter::singleton()->incr( $key, $count ); + $stats = RequestContext::getMain()->getStats(); + $stats->updateCount( $key, $count ); } /** diff --git a/includes/StatCounter.php b/includes/StatCounter.php deleted file mode 100644 index 5fc8f2f5d8..0000000000 --- a/includes/StatCounter.php +++ /dev/null @@ -1,154 +0,0 @@ - count) - - /** @var Config */ - protected $config; - - protected function __construct( Config $config ) { - $this->config = $config; - } - - /** - * @return StatCounter - */ - public static function singleton() { - static $instance = null; - if ( !$instance ) { - $instance = new self( - ConfigFactory::getDefaultInstance()->makeConfig( 'main' ) - ); - } - return $instance; - } - - /** - * Increment a key by delta $count - * - * @param string $key - * @param int $count - * @return void - */ - public function incr( $key, $count = 1 ) { - $this->deltas[$key] = isset( $this->deltas[$key] ) ? $this->deltas[$key] : 0; - $this->deltas[$key] += $count; - if ( PHP_SAPI === 'cli' ) { - $this->flush(); - } - } - - /** - * Flush all pending deltas to persistent storage - * - * @return void - */ - public function flush() { - $statsMethod = $this->config->get( 'StatsMethod' ); - $deltas = array_filter( $this->deltas ); // remove 0 valued entries - if ( $statsMethod === 'udp' ) { - $this->sendDeltasUDP( $deltas ); - } elseif ( $statsMethod === 'cache' ) { - $this->sendDeltasMemc( $deltas ); - } else { - // disabled - } - $this->deltas = array(); - } - - /** - * @param array $deltas - * @return void - */ - protected function sendDeltasUDP( array $deltas ) { - $aggregateStatsID = $this->config->get( 'AggregateStatsID' ); - $id = strlen( $aggregateStatsID ) ? $aggregateStatsID : wfWikiID(); - - $lines = array(); - foreach ( $deltas as $key => $count ) { - $lines[] = sprintf( $this->config->get( 'StatsFormatString' ), $id, $count, $key ); - } - - if ( count( $lines ) ) { - static $socket = null; - if ( !$socket ) { - $socket = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP ); - } - $packet = ''; - $packets = array(); - foreach ( $lines as $line ) { - if ( ( strlen( $packet ) + strlen( $line ) ) > 1450 ) { - $packets[] = $packet; - $packet = ''; - } - $packet .= $line; - } - if ( $packet != '' ) { - $packets[] = $packet; - } - foreach ( $packets as $packet ) { - wfSuppressWarnings(); - socket_sendto( - $socket, - $packet, - strlen( $packet ), - 0, - $this->config->get( 'UDPProfilerHost' ), - $this->config->get( 'UDPProfilerPort' ) - ); - wfRestoreWarnings(); - } - } - } - - /** - * @param array $deltas - * @return void - */ - protected function sendDeltasMemc( array $deltas ) { - global $wgMemc; - - foreach ( $deltas as $key => $count ) { - $ckey = wfMemcKey( 'stats', $key ); - if ( $wgMemc->incr( $ckey, $count ) === null ) { - $wgMemc->add( $ckey, $count ); - } - } - } -} diff --git a/includes/context/ContextSource.php b/includes/context/ContextSource.php index 83e8ef6448..d526d84be6 100644 --- a/includes/context/ContextSource.php +++ b/includes/context/ContextSource.php @@ -152,6 +152,17 @@ abstract class ContextSource implements IContextSource { return $this->getContext()->getSkin(); } + /** + * Get the Stats object + * + * @since 1.25 + * @return BufferingStatsdDataFactory + */ + public function getStats() { + return $this->getContext()->getStats(); + } + + /** * Get a Message object with context set * Parameters are the same as wfMessage() diff --git a/includes/context/DerivativeContext.php b/includes/context/DerivativeContext.php index 836aef98b4..00323cae77 100644 --- a/includes/context/DerivativeContext.php +++ b/includes/context/DerivativeContext.php @@ -97,6 +97,19 @@ class DerivativeContext extends ContextSource { } } + /** + * Get the stats object + * + * @return BufferingStatsdDataFactory + */ + public function getStats() { + if ( !is_null( $this->stats ) ) { + return $this->stats; + } else { + return $this->getContext()->getStats(); + } + } + /** * Set the WebRequest object * diff --git a/includes/context/IContextSource.php b/includes/context/IContextSource.php index 925666dd9a..66df7fc659 100644 --- a/includes/context/IContextSource.php +++ b/includes/context/IContextSource.php @@ -121,6 +121,14 @@ interface IContextSource { */ public function getConfig(); + /** + * Get the stats object + * + * @since 1.25 + * @return BufferingStatsdDataFactory + */ + public function getStats(); + /** * Get a Message object with context set * diff --git a/includes/context/RequestContext.php b/includes/context/RequestContext.php index 3abc986921..4e790c0446 100644 --- a/includes/context/RequestContext.php +++ b/includes/context/RequestContext.php @@ -61,6 +61,11 @@ class RequestContext implements IContextSource { */ private $skin; + /** + * @var StatsdDataFactory + */ + private $stats; + /** * @var Config */ @@ -118,6 +123,22 @@ class RequestContext implements IContextSource { return $this->request; } + /** + * Get the Stats object + * + * @return BufferingStatsdDataFactory + */ + public function getStats() { + if ( $this->stats === null ) { + $config = $this->getConfig(); + $prefix = $config->has( 'StatsdMetricPrefix' ) + ? rtrim( $config->get( 'StatsdMetricPrefix' ), '.' ) + : 'MediaWiki'; + $this->stats = new BufferingStatsdDataFactory( $prefix ); + } + return $this->stats; + } + /** * Set the Title object * diff --git a/includes/libs/BufferingStatsdDataFactory.php b/includes/libs/BufferingStatsdDataFactory.php new file mode 100644 index 0000000000..ea5b09dc7e --- /dev/null +++ b/includes/libs/BufferingStatsdDataFactory.php @@ -0,0 +1,59 @@ +prefix = $prefix; + } + + public function produceStatsdData( $key, $value = 1, $metric = self::STATSD_METRIC_COUNT ) { + $this->buffer[] = $entity = $this->produceStatsdDataEntity(); + if ( $key !== null ) { + $prefixedKey = ltrim( $this->prefix . '.' . $key, '.' ); + $entity->setKey( $prefixedKey ); + } + if ( $value !== null ) { + $entity->setValue( $value ); + } + if ( $metric !== null ) { + $entity->setMetric( $metric ); + } + return $entity; + } + + public function getBuffer() { + return $this->buffer; + } +} diff --git a/includes/page/Article.php b/includes/page/Article.php index cc87a10fcf..6ebd8a197a 100644 --- a/includes/page/Article.php +++ b/includes/page/Article.php @@ -574,7 +574,7 @@ class Article implements Page { $useParserCache = $this->mPage->isParserCacheUsed( $parserOptions, $oldid ); wfDebug( 'Article::view using parser cache: ' . ( $useParserCache ? 'yes' : 'no' ) . "\n" ); if ( $user->getStubThreshold() ) { - wfIncrStats( 'pcache_miss_stub' ); + $this->getContext()->getStats()->increment( 'pcache_miss_stub' ); } $this->showRedirectedFromHeader();