From 766c11b51669030ff5728d716b4683085fc21057 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Mon, 30 Dec 2013 22:42:34 -0800 Subject: [PATCH] Add RunningStat class, tests RunningStat computes the central tendency, shape, and extrema of a set of points online, in constant space. It uses a neat one-pass algorithm for calculating variance, described here: This particular implementation adapts a sample C++ implementation by John D. Cook to PHP. See and . RunningStat instances can be combined. The resultant RunningStat has the same state it would have had if it had been used to accumulate each point. This property is attractive because it allows separate threads of execution to process a stream in parallel. More importantly, individual points can be accumulated in stages, without loss of fidelity, at intermediate points in the aggregation process. JavaScript profiling samples can be accumulated in the user's browser and be combined with measurements from other browsers on the profiling data aggregator. Functions that are called multiple times in the course of a profiled web request can be accumulated in MediaWiki prior to being transmitted to the profiling data aggregator. Usage will be introduced in a dependent commit. Change-Id: Ifedda276dfe8e0783cb8c4a95626e2aedd4ad368 --- includes/AutoLoader.php | 1 + includes/profiler/RunningStat.php | 176 +++++++++++++++++++++ tests/phpunit/includes/RunningStatTest.php | 81 ++++++++++ 3 files changed, 258 insertions(+) create mode 100644 includes/profiler/RunningStat.php create mode 100644 tests/phpunit/includes/RunningStatTest.php diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index b172f71c37..1f81249fe9 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -185,6 +185,7 @@ $wgAutoloadLocalClasses = array( 'Revision' => 'includes/Revision.php', 'RevisionList' => 'includes/RevisionList.php', 'RSSFeed' => 'includes/Feed.php', + 'RunningStat' => 'includes/profiler/RunningStat.php', 'Sanitizer' => 'includes/Sanitizer.php', 'SiteConfiguration' => 'includes/SiteConfiguration.php', 'SiteStats' => 'includes/SiteStats.php', diff --git a/includes/profiler/RunningStat.php b/includes/profiler/RunningStat.php new file mode 100644 index 0000000000..dda5254e3b --- /dev/null +++ b/includes/profiler/RunningStat.php @@ -0,0 +1,176 @@ +. +define( 'NEGATIVE_INF', -INF ); + +/** + * Represents a running summary of a stream of numbers. + * + * RunningStat instances are accumulator-like objects that provide a set of + * continuously-updated summary statistics for a stream of numbers, without + * requiring that each value be stored. The measures it provides are the + * arithmetic mean, variance, standard deviation, and extrema (min and max); + * together they describe the central tendency and statistical dispersion of a + * set of values. + * + * One RunningStat instance can be merged into another; the resultant + * RunningStat has the state it would have had if it had accumulated each + * individual point. This allows data to be summarized in parallel and in + * stages without loss of fidelity. + * + * Based on a C++ implementation by John D. Cook: + * + * + * + * The in-line documentation for this class incorporates content from the + * English Wikipedia articles "Variance", "Algorithms for calculating + * variance", and "Standard deviation". + * + * @since 1.23 + */ +class RunningStat implements Countable { + + /** @var int Number of samples. **/ + public $n = 0; + + /** @var float The first moment (or mean, or expected value). **/ + public $m1 = 0.0; + + /** @var float The second central moment (or variance). **/ + public $m2 = 0.0; + + /** @var float The least value in the the set. **/ + public $min = INF; + + /** @var float The most value in the set. **/ + public $max = NEGATIVE_INF; + + /** + * Count the number of accumulated values. + * @return int Number of values + */ + public function count() { + return $this->n; + } + + /** + * Add a number to the data set. + * @param int|float $x Value to add + */ + public function push( $x ) { + $x = (float) $x; + + $this->min = min( $this->min, $x ); + $this->max = max( $this->max, $x ); + + $n1 = $this->n; + $this->n += 1; + $delta = $x - $this->m1; + $delta_n = $delta / $this->n; + $this->m1 += $delta_n; + $this->m2 += $delta * $delta_n * $n1; + } + + /** + * Get the mean, or expected value. + * + * The arithmetic mean is the sum of all measurements divided by the number + * of observations in the data set. + * + * @return float Mean + */ + public function getMean() { + return $this->m1; + } + + /** + * Get the estimated variance. + * + * Variance measures how far a set of numbers is spread out. A small + * variance indicates that the data points tend to be very close to the + * mean (and hence to each other), while a high variance indicates that the + * data points are very spread out from the mean and from each other. + * + * @return float Estimated variance + */ + public function getVariance() { + if ( $this->n === 0 ) { + // The variance of the empty set is undefined. + return NAN; + } elseif ( $this->n === 1 ) { + return 0.0; + } else { + return $this->m2 / ( $this->n - 1.0 ); + } + } + + /** + * Get the estimated stanard deviation. + * + * The standard deviation of a statistical population is the square root of + * its variance. It shows shows how much variation from the mean exists. In + * addition to expressing the variability of a population, the standard + * deviation is commonly used to measure confidence in statistical conclusions. + * + * @return float Estimated standard deviation + */ + public function getStdDev() { + return sqrt( $this->getVariance() ); + } + + /** + * Merge another RunningStat instance into this instance. + * + * This instance then has the state it would have had if all the data had + * been accumulated by it alone. + * + * @param RunningStat RunningStat instance to merge into this one + */ + public function merge( RunningStat $other ) { + // If the other RunningStat is empty, there's nothing to do. + if ( $other->n === 0 ) { + return; + } + + // If this RunningStat is empty, copy values from other RunningStat. + if ( $this->n === 0 ) { + $this->n = $other->n; + $this->m1 = $other->m1; + $this->m2 = $other->m2; + $this->min = $other->min; + $this->max = $other->max; + return; + } + + $n = $this->n + $other->n; + $delta = $other->m1 - $this->m1; + $delta2 = $delta * $delta; + + $this->m1 = ( ( $this->n * $this->m1 ) + ( $other->n * $other->m1 ) ) / $n; + $this->m2 = $this->m2 + $other->m2 + ( $delta2 * $this->n * $other->n / $n ); + $this->min = min( $this->min, $other->min ); + $this->max = max( $this->max, $other->max ); + $this->n = $n; + } +} diff --git a/tests/phpunit/includes/RunningStatTest.php b/tests/phpunit/includes/RunningStatTest.php new file mode 100644 index 0000000000..e24c08877b --- /dev/null +++ b/tests/phpunit/includes/RunningStatTest.php @@ -0,0 +1,81 @@ +points as $point ) { + $rstat->push( $point ); + } + + $mean = array_sum( $this->points ) / count( $this->points ); + $variance = array_sum( array_map( function ( $x ) use ( $mean ) { + return pow( $mean - $x, 2 ); + }, $this->points ) ) / ( count( $rstat ) - 1 ); + $stddev = sqrt( $variance ); + $min = min( $this->points ); + $max = max( $this->points ); + + $this->assertEquals( count( $rstat ), count( $this->points ) ); + $this->assertEquals( $rstat->min, min( $this->points ) ); + $this->assertEquals( $rstat->max, max( $this->points ) ); + $this->assertEquals( $rstat->getMean(), $mean ); + $this->assertEquals( $rstat->getVariance(), $variance ); + $this->assertEquals( $rstat->getStdDev(), $stddev ); + } + + /** + * When one RunningStat instance is merged into another, the state of the + * target RunningInstance should have the state that it would have had if + * all the data had been accumulated by it alone. + * @covers RunningStat::merge + * @covers RunningStat::count + */ + public function testRunningStatMerge() { + $expected = new RunningStat(); + + foreach( $this->points as $point ) { + $expected->push( $point ); + } + + // Split the data into two sets + $sets = array_chunk( $this->points, floor( count( $this->points ) / 2 ) ); + + // Accumulate the first half into one RunningStat object + $first = new RunningStat(); + foreach( $sets[0] as $point ) { + $first->push( $point ); + } + + // Accumulate the second half into another RunningStat object + $second = new RunningStat(); + foreach( $sets[1] as $point ) { + $second->push( $point ); + } + + // Merge the second RunningStat object into the first + $first->merge( $second ); + + $this->assertEquals( count( $first ), count( $this->points ) ); + $this->assertEquals( $first, $expected ); + } +} -- 2.20.1