From 8d5af36eb00080eea1b245881fe38ced6462c199 Mon Sep 17 00:00:00 2001
From: Aaron Schulz <aschulz@wikimedia.org>
Date: Mon, 11 Mar 2013 18:56:16 -0700
Subject: [PATCH] [JobQueue] Try to cut down on waitForBackups() calls in
 runJobs.php.

Change-Id: I2fc97ef8dbc02d1184959ba962dcafdae9fae808
---
 includes/job/JobQueueGroup.php | 19 +++++++++++++++++++
 maintenance/runJobs.php        | 11 +++++++----
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php
index 50b5471bd9..351c71a370 100644
--- a/includes/job/JobQueueGroup.php
+++ b/includes/job/JobQueueGroup.php
@@ -201,6 +201,25 @@ class JobQueueGroup {
 		return $this->get( $job->getType() )->deduplicateRootJob( $job );
 	}
 
+	/**
+	 * Wait for any slaves or backup queue servers to catch up.
+	 *
+	 * This does nothing for certain queue classes.
+	 *
+	 * @return void
+	 * @throws MWException
+	 */
+	public function waitForBackups() {
+		global $wgJobTypeConf;
+
+		wfProfileIn( __METHOD__ );
+		// Try to avoid doing this more than once per queue storage medium
+		foreach ( $wgJobTypeConf as $type => $conf ) {
+			$this->get( $type )->waitForBackups();
+		}
+		wfProfileOut( __METHOD__ );
+	}
+
 	/**
 	 * Get the list of queue types
 	 *
diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php
index 714bb84845..b1be714c14 100644
--- a/maintenance/runJobs.php
+++ b/maintenance/runJobs.php
@@ -73,7 +73,7 @@ class RunJobs extends Maintenance {
 		$type = $this->getOption( 'type', false );
 		$wgTitle = Title::newFromText( 'RunJobs.php' );
 		$dbw = wfGetDB( DB_MASTER );
-		$n = 0;
+		$jobsRun = 0; // counter
 
 		$group = JobQueueGroup::singleton();
 		// Handle any required periodic queue maintenance
@@ -88,6 +88,7 @@ class RunJobs extends Maintenance {
 				? $group->pop( JobQueueGroup::TYPE_DEFAULT, JobQueueGroup::USE_CACHE )
 				: $group->pop( $type ); // job from a single queue
 			if ( $job ) { // found a job
+				++$jobsRun;
 				$this->runJobsLog( $job->toString() . " STARTING" );
 
 				// Run the job...
@@ -113,19 +114,21 @@ class RunJobs extends Maintenance {
 				}
 
 				// Break out if we hit the job count or wall time limits...
-				if ( $maxJobs && ++$n >= $maxJobs ) {
+				if ( $maxJobs && $jobsRun >= $maxJobs ) {
 					break;
 				} elseif ( $maxTime && ( time() - $startTime ) > $maxTime ) {
 					break;
 				}
 
-				// Don't let any queue slaves/backups fall behind
-				$group->get( $job->getType() )->waitForBackups();
 				// Don't let any of the main DB slaves get backed up
 				$timePassed = time() - $lastTime;
 				if ( $timePassed >= 5 || $timePassed < 0 ) {
 					wfWaitForSlaves();
 				}
+				// Don't let any queue slaves/backups fall behind
+				if ( $jobsRun > 0 && ( $jobsRun % 100 ) == 0 ) {
+					$group->waitForBackups();
+				}
 			}
 		} while ( $job ); // stop when there are no jobs
 	}
-- 
2.20.1