From de9f9bda7db9d607767968ddaf1a55a50916a4b1 Mon Sep 17 00:00:00 2001
From: Kunal Mehta <legoktm@member.fsf.org>
Date: Sun, 9 Apr 2017 23:54:01 -0700
Subject: [PATCH] API: Optionally include in job queue size in maxlag

maxlag is the default mechanism most bots and libraries use in
determining when to back off due to wiki overload. However these days,
there are other things that should be considered when asking bots to
back off, one of those is job queue size.

For compatibility and simplicity of use, the number of jobs is converted
into something resembling seconds using a configurable factor. We also
output the total number of jobs in the API error output so more
sophisticated clients can do a more advanced back off.

Bug: T160003
Change-Id: Iedae2344a3d93202efbdd1bf807cef6165b6257a
---
 includes/DefaultSettings.php | 13 +++++++++++
 includes/api/ApiMain.php     | 43 +++++++++++++++++++++++++++++++-----
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index d158c1e31d..53a147b524 100644
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -7394,6 +7394,19 @@ $wgJobQueueAggregator = [
 	'class' => 'JobQueueAggregatorNull'
 ];
 
+/**
+ * Whether to include the number of jobs that are queued
+ * for the API's maxlag parameter.
+ * The total number of jobs will be divided by this to get an
+ * estimated second of maxlag. Typically bots backoff at maxlag=5,
+ * so setting this to the max number of jobs that should be in your
+ * queue divided by 5 should have the effect of stopping bots once
+ * that limit is hit.
+ *
+ * @since 1.29
+ */
+$wgJobQueueIncludeInMaxLagFactor = false;
+
 /**
  * Additional functions to be performed with updateSpecialPages.
  * Expensive Querypages are already updated.
diff --git a/includes/api/ApiMain.php b/includes/api/ApiMain.php
index b5eff7056e..4068a50bb6 100644
--- a/includes/api/ApiMain.php
+++ b/includes/api/ApiMain.php
@@ -1230,6 +1230,35 @@ class ApiMain extends ApiBase {
 		return $module;
 	}
 
+	/**
+	 * @return array
+	 */
+	private function getMaxLag() {
+		$dbLag = MediaWikiServices::getInstance()->getDBLoadBalancer()->getMaxLag();
+		$lagInfo = [
+			'host' => $dbLag[0],
+			'lag' => $dbLag[1],
+			'type' => 'db'
+		];
+
+		$jobQueueLagFactor = $this->getConfig()->get( 'JobQueueIncludeInMaxLagFactor' );
+		if ( $jobQueueLagFactor ) {
+			// Turn total number of jobs into seconds by using the configured value
+			$totalJobs = array_sum( JobQueueGroup::singleton()->getQueueSizes() );
+			$jobQueueLag = $totalJobs / (float)$jobQueueLagFactor;
+			if ( $jobQueueLag > $lagInfo['lag'] ) {
+				$lagInfo = [
+					'host' => wfHostname(), // XXX: Is there a better value that could be used?
+					'lag' => $jobQueueLag,
+					'type' => 'jobqueue',
+					'jobs' => $totalJobs,
+				];
+			}
+		}
+
+		return $lagInfo;
+	}
+
 	/**
 	 * Check the max lag if necessary
 	 * @param ApiBase $module Api module being used
@@ -1239,18 +1268,22 @@ class ApiMain extends ApiBase {
 	protected function checkMaxLag( $module, $params ) {
 		if ( $module->shouldCheckMaxlag() && isset( $params['maxlag'] ) ) {
 			$maxLag = $params['maxlag'];
-			list( $host, $lag ) = wfGetLB()->getMaxLag();
-			if ( $lag > $maxLag ) {
+			$lagInfo = $this->getMaxLag();
+			if ( $lagInfo['lag'] > $maxLag ) {
 				$response = $this->getRequest()->response();
 
 				$response->header( 'Retry-After: ' . max( intval( $maxLag ), 5 ) );
-				$response->header( 'X-Database-Lag: ' . intval( $lag ) );
+				$response->header( 'X-Database-Lag: ' . intval( $lagInfo['lag'] ) );
 
 				if ( $this->getConfig()->get( 'ShowHostnames' ) ) {
-					$this->dieWithError( [ 'apierror-maxlag', $lag, $host ] );
+					$this->dieWithError(
+						[ 'apierror-maxlag', $lagInfo['lag'], $lagInfo['host'] ],
+						'maxlag',
+						$lagInfo
+					);
 				}
 
-				$this->dieWithError( [ 'apierror-maxlag-generic', $lag ], 'maxlag' );
+				$this->dieWithError( [ 'apierror-maxlag-generic', $lagInfo['lag'] ], 'maxlag', $lagInfo );
 			}
 		}
 
-- 
2.20.1