From 04c610200fec46870b839ab956ca60266a4792b2 Mon Sep 17 00:00:00 2001
From: Aaron Schulz <aschulz@wikimedia.org>
Date: Fri, 1 Mar 2013 14:37:51 -0800
Subject: [PATCH] [JobQueue] Throttle refreshLinks2 jobs based on finishing the
 refreshLinks jobs.

* This should lower the rate of queue activity when major templates change.
* Also fixed a broken array_diff() call in nextJobDB.php.
* Additionally removed checkJob from nextJobDB.php. This is redundant to
  runJobs.php de-listing queues via JobQueueGroup::pop() when the queue
  is empty.

Change-Id: I1518a0de9e7ada22350d9993dd7ffe5f2ce23745
---
 includes/DefaultSettings.php   |  2 ++
 includes/job/JobQueueGroup.php | 17 +++++++++++++++++
 maintenance/nextJobDB.php      | 18 +++---------------
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index 73c097efd0..7b80715b9d 100644
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -5546,6 +5546,8 @@ $wgJobTypesExcludedFromDefaultQueue = array( 'AssembleUploadChunks', 'PublishSta
 
 /**
  * Map of job types to configuration arrays.
+ * This determines which queue class and storage system is used for each job type.
+ * Job types that do not have explicit configuration will use the 'default' config.
  * These settings should be global to all wikis.
  */
 $wgJobTypeConf = array(
diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php
index 23a54949a9..dae026c5bd 100644
--- a/includes/job/JobQueueGroup.php
+++ b/includes/job/JobQueueGroup.php
@@ -236,6 +236,23 @@ class JobQueueGroup {
 		return $types;
 	}
 
+	/**
+	 * Check if jobs should not be popped of a queue right now.
+	 * This is only used for performance, such as to avoid spamming
+	 * the queue with many sub-jobs before they actually get run.
+	 *
+	 * @param $type string
+	 * @return bool
+	 */
+	public function isQueueDeprioritized( $type ) {
+		if ( $type === 'refreshLinks2' ) {
+			// Don't keep converting refreshLinks2 => refreshLinks jobs if the
+			// later jobs have not been done yet. This helps throttle queue spam.
+			return !$this->get( 'refreshLinks' )->isEmpty();
+		}
+		return false;
+	}
+
 	/**
 	 * Execute any due periodic queue maintenance tasks for all queues.
 	 *
diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php
index f5bf4da287..1be5146eb2 100644
--- a/maintenance/nextJobDB.php
+++ b/maintenance/nextJobDB.php
@@ -77,10 +77,9 @@ class nextJobDB extends Maintenance {
 				return; // no jobs for this type
 			}
 
-			list( $type, $db ) = $candidates[ mt_rand( 0, count( $candidates ) - 1 ) ];
-			if ( !$this->checkJob( $type, $db ) ) { // queue is actually empty?
-				$pendingDBs[$type] = array_diff( $pendingDBs[$type], $db );
-				JobQueueAggregator::singleton()->notifyQueueEmpty( $db, $type );
+			list( $type, $db ) = $candidates[mt_rand( 0, count( $candidates ) - 1 )];
+			if ( JobQueueGroup::singleton( $db )->isQueueDeprioritized( $type ) ) {
+				$pendingDBs[$type] = array_diff( $pendingDBs[$type], array( $db ) );
 				$again = true;
 			}
 		} while ( $again );
@@ -92,17 +91,6 @@ class nextJobDB extends Maintenance {
 		}
 	}
 
-	/**
-	 * Check if the specified database has a job of the specified type in it.
-	 * The type may be false to indicate "all".
-	 * @param $type string
-	 * @param $dbName string
-	 * @return bool
-	 */
-	private function checkJob( $type, $dbName ) {
-		return !JobQueueGroup::singleton( $dbName )->get( $type )->isEmpty();
-	}
-
 	/**
 	 * Do all ready periodic jobs for all databases every 5 minutes (and .1% of the time)
 	 * @return integer
-- 
2.20.1