From 593442683dc471c09e48c36503004da3add54e65 Mon Sep 17 00:00:00 2001
From: Timo Tijhof <krinklemail@gmail.com>
Date: Sun, 28 Jan 2018 13:46:32 -0800
Subject: [PATCH] benchmarks: Add rate per second and standard deviation to
 output

* Convert to using RunningStat to ensure mostly constant memory
  usage, instead of storing each result.
* Remove 'median' (no longer possible with this setup).
* Remove 'min' from output.

Change-Id: I6efbc500181eb502c1800165870eee81dbc418a7
---
 includes/libs/XhprofData.php           |  4 +--
 maintenance/benchmarks/Benchmarker.php | 42 +++++++++++---------------
 2 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/includes/libs/XhprofData.php b/includes/libs/XhprofData.php
index 8762446a03..5af22ed5b2 100644
--- a/includes/libs/XhprofData.php
+++ b/includes/libs/XhprofData.php
@@ -209,14 +209,14 @@ class XhprofData {
 			foreach ( $this->inclusive as $func => $stats ) {
 				foreach ( $stats as $name => $value ) {
 					if ( $value instanceof RunningStat ) {
-						$total = $value->m1 * $value->n;
+						$total = $value->getMean() * $value->getCount();
 						$percent = ( isset( $main[$name] ) && $main[$name] )
 							? 100 * $total / $main[$name]
 							: 0;
 						$this->inclusive[$func][$name] = [
 							'total' => $total,
 							'min' => $value->min,
-							'mean' => $value->m1,
+							'mean' => $value->getMean(),
 							'max' => $value->max,
 							'variance' => $value->m2,
 							'percent' => $percent,
diff --git a/maintenance/benchmarks/Benchmarker.php b/maintenance/benchmarks/Benchmarker.php
index ffb8cb353f..f1e7dbbdee 100644
--- a/maintenance/benchmarks/Benchmarker.php
+++ b/maintenance/benchmarks/Benchmarker.php
@@ -26,6 +26,8 @@
  * @ingroup Benchmark
  */
 
+use Wikimedia\RunningStat;
+
 // @codeCoverageIgnoreStart
 require_once __DIR__ . '/../Maintenance.php';
 // @codeCoverageIgnoreEnd
@@ -68,7 +70,7 @@ abstract class Benchmarker extends Maintenance {
 			}
 
 			// Run benchmarks
-			$times = [];
+			$stat = new RunningStat();
 			for ( $i = 0; $i < $count; $i++ ) {
 				$t = microtime( true );
 				call_user_func_array( $bench['function'], $bench['args'] );
@@ -76,20 +78,8 @@ abstract class Benchmarker extends Maintenance {
 				if ( $verbose ) {
 					$this->verboseRun( $i );
 				}
-				$times[] = $t;
-			}
-
-			// Collect metrics
-			sort( $times, SORT_NUMERIC );
-			$min = $times[0];
-			$max = end( $times );
-			if ( $count % 2 ) {
-				$median = $times[ ( $count - 1 ) / 2 ];
-			} else {
-				$median = ( $times[$count / 2] + $times[$count / 2 - 1] ) / 2;
+				$stat->addObservation( $t );
 			}
-			$total = array_sum( $times );
-			$mean = $total / $count;
 
 			// Name defaults to name of called function
 			if ( is_string( $key ) ) {
@@ -108,12 +98,13 @@ abstract class Benchmarker extends Maintenance {
 
 			$this->addResult( [
 				'name' => $name,
-				'count' => $count,
-				'total' => $total,
-				'min' => $min,
-				'median' => $median,
-				'mean' => $mean,
-				'max' => $max,
+				'count' => $stat->getCount(),
+				// Get rate per second from mean (in ms)
+				'rate' => 1.0 / ( $stat->getMean() / 1000.0 ),
+				'total' => $stat->getMean() * $stat->getCount(),
+				'mean' => $stat->getMean(),
+				'max' => $stat->max,
+				'stddev' => $stat->getStdDev(),
 				'usage' => [
 					'mem' => memory_get_usage( true ),
 					'mempeak' => memory_get_peak_usage( true ),
@@ -137,12 +128,15 @@ abstract class Benchmarker extends Maintenance {
 	public function addResult( $res ) {
 		$ret = sprintf( "%s\n  %' 6s: %d\n",
 			$res['name'],
-			'times',
+			'count',
 			$res['count']
 		);
-
-		foreach ( [ 'total', 'min', 'median', 'mean', 'max' ] as $metric ) {
-			$ret .= sprintf( "  %' 6s: %6.2fms\n",
+		$ret .= sprintf( "  %' 6s: %8.1f/s\n",
+			'rate',
+			$res['rate']
+		);
+		foreach ( [ 'total', 'mean', 'max', 'stddev' ] as $metric ) {
+			$ret .= sprintf( "  %' 6s: %8.2fms\n",
 				$metric,
 				$res[$metric]
 			);
-- 
2.20.1