[JobQueue] Throttle refreshLinks2 jobs based on finishing the refreshLinks jobs.

* This should lower the rate of queue activity when major templates change.
* Also fixed a broken array_diff() call in nextJobDB.php.
* Additionally removed checkJob from nextJobDB.php. This is redundant to
  runJobs.php de-listing queues via JobQueueGroup::pop() when the queue
  is empty.

Change-Id: I1518a0de9e7ada22350d9993dd7ffe5f2ce23745

diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index 73c097e..7b80715 100644 (file)
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -5546,6 +5546,8 @@ $wgJobTypesExcludedFromDefaultQueue = array( 'AssembleUploadChunks', 'PublishSta
 
 /**
  * Map of job types to configuration arrays.
+ * This determines which queue class and storage system is used for each job type.
+ * Job types that do not have explicit configuration will use the 'default' config.
  * These settings should be global to all wikis.
  */
 $wgJobTypeConf = array(

diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php
index 23a5494..dae026c 100644 (file)
--- a/includes/job/JobQueueGroup.php
+++ b/includes/job/JobQueueGroup.php
@@ -236,6 +236,23 @@ class JobQueueGroup {
                return $types;
        }
 
+       /**
+        * Check if jobs should not be popped of a queue right now.
+        * This is only used for performance, such as to avoid spamming
+        * the queue with many sub-jobs before they actually get run.
+        *
+        * @param $type string
+        * @return bool
+        */
+       public function isQueueDeprioritized( $type ) {
+               if ( $type === 'refreshLinks2' ) {
+                       // Don't keep converting refreshLinks2 => refreshLinks jobs if the
+                       // later jobs have not been done yet. This helps throttle queue spam.
+                       return !$this->get( 'refreshLinks' )->isEmpty();
+               }
+               return false;
+       }
+
        /**
         * Execute any due periodic queue maintenance tasks for all queues.
         *

diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php
index f5bf4da..1be5146 100644 (file)
--- a/maintenance/nextJobDB.php
+++ b/maintenance/nextJobDB.php
@@ -77,10 +77,9 @@ class nextJobDB extends Maintenance {
                                return; // no jobs for this type
                        }
 
-                       list( $type, $db ) = $candidates[ mt_rand( 0, count( $candidates ) - 1 ) ];
-                       if ( !$this->checkJob( $type, $db ) ) { // queue is actually empty?
-                               $pendingDBs[$type] = array_diff( $pendingDBs[$type], $db );
-                               JobQueueAggregator::singleton()->notifyQueueEmpty( $db, $type );
+                       list( $type, $db ) = $candidates[mt_rand( 0, count( $candidates ) - 1 )];
+                       if ( JobQueueGroup::singleton( $db )->isQueueDeprioritized( $type ) ) {
+                               $pendingDBs[$type] = array_diff( $pendingDBs[$type], array( $db ) );
                                $again = true;
                        }
                } while ( $again );
@@ -92,17 +91,6 @@ class nextJobDB extends Maintenance {
                }
        }
 
-       /**
-        * Check if the specified database has a job of the specified type in it.
-        * The type may be false to indicate "all".
-        * @param $type string
-        * @param $dbName string
-        * @return bool
-        */
-       private function checkJob( $type, $dbName ) {
-               return !JobQueueGroup::singleton( $dbName )->get( $type )->isEmpty();
-       }
-
        /**
         * Do all ready periodic jobs for all databases every 5 minutes (and .1% of the time)
         * @return integer