[JobQueue] Try to cut down on waitForBackups() calls in runJobs.php.

Change-Id: I2fc97ef8dbc02d1184959ba962dcafdae9fae808

diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php
index 50b5471..351c71a 100644 (file)
--- a/includes/job/JobQueueGroup.php
+++ b/includes/job/JobQueueGroup.php
@@ -201,6 +201,25 @@ class JobQueueGroup {
                return $this->get( $job->getType() )->deduplicateRootJob( $job );
        }
 
+       /**
+        * Wait for any slaves or backup queue servers to catch up.
+        *
+        * This does nothing for certain queue classes.
+        *
+        * @return void
+        * @throws MWException
+        */
+       public function waitForBackups() {
+               global $wgJobTypeConf;
+
+               wfProfileIn( __METHOD__ );
+               // Try to avoid doing this more than once per queue storage medium
+               foreach ( $wgJobTypeConf as $type => $conf ) {
+                       $this->get( $type )->waitForBackups();
+               }
+               wfProfileOut( __METHOD__ );
+       }
+
        /**
         * Get the list of queue types
         *

diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php
index 714bb84..b1be714 100644 (file)
--- a/maintenance/runJobs.php
+++ b/maintenance/runJobs.php
@@ -73,7 +73,7 @@ class RunJobs extends Maintenance {
                $type = $this->getOption( 'type', false );
                $wgTitle = Title::newFromText( 'RunJobs.php' );
                $dbw = wfGetDB( DB_MASTER );
-               $n = 0;
+               $jobsRun = 0; // counter
 
                $group = JobQueueGroup::singleton();
                // Handle any required periodic queue maintenance
@@ -88,6 +88,7 @@ class RunJobs extends Maintenance {
                                ? $group->pop( JobQueueGroup::TYPE_DEFAULT, JobQueueGroup::USE_CACHE )
                                : $group->pop( $type ); // job from a single queue
                        if ( $job ) { // found a job
+                               ++$jobsRun;
                                $this->runJobsLog( $job->toString() . " STARTING" );
 
                                // Run the job...
@@ -113,19 +114,21 @@ class RunJobs extends Maintenance {
                                }
 
                                // Break out if we hit the job count or wall time limits...
-                               if ( $maxJobs && ++$n >= $maxJobs ) {
+                               if ( $maxJobs && $jobsRun >= $maxJobs ) {
                                        break;
                                } elseif ( $maxTime && ( time() - $startTime ) > $maxTime ) {
                                        break;
                                }
 
-                               // Don't let any queue slaves/backups fall behind
-                               $group->get( $job->getType() )->waitForBackups();
                                // Don't let any of the main DB slaves get backed up
                                $timePassed = time() - $lastTime;
                                if ( $timePassed >= 5 || $timePassed < 0 ) {
                                        wfWaitForSlaves();
                                }
+                               // Don't let any queue slaves/backups fall behind
+                               if ( $jobsRun > 0 && ( $jobsRun % 100 ) == 0 ) {
+                                       $group->waitForBackups();
+                               }
                        }
                } while ( $job ); // stop when there are no jobs
        }