5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @defgroup JobQueue JobQueue
25 * Class to both describe a background job and handle jobs.
42 /*-------------------------------------------------------------------------
44 *------------------------------------------------------------------------*/
48 * @return boolean success
50 abstract function run();
52 /*-------------------------------------------------------------------------
54 *------------------------------------------------------------------------*/
57 * Pop a job of a certain type. This tries less hard than pop() to
58 * actually find a job; it may be adversely affected by concurrent job
65 static function pop_type( $type ) {
66 wfProfilein( __METHOD__
);
68 $dbw = wfGetDB( DB_MASTER
);
70 $dbw->begin( __METHOD__
);
72 $row = $dbw->selectRow(
75 array( 'job_cmd' => $type ),
77 array( 'LIMIT' => 1, 'FOR UPDATE' )
80 if ( $row === false ) {
81 $dbw->commit( __METHOD__
);
82 wfProfileOut( __METHOD__
);
86 /* Ensure we "own" this row */
87 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
88 $affected = $dbw->affectedRows();
89 $dbw->commit( __METHOD__
);
91 if ( $affected == 0 ) {
92 wfProfileOut( __METHOD__
);
96 wfIncrStats( 'job-pop' );
97 $namespace = $row->job_namespace
;
98 $dbkey = $row->job_title
;
99 $title = Title
::makeTitleSafe( $namespace, $dbkey );
100 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
),
103 $job->removeDuplicates();
105 wfProfileOut( __METHOD__
);
110 * Pop a job off the front of the queue
112 * @param $offset Integer: Number of jobs to skip
113 * @return Job or false if there's no jobs
115 static function pop( $offset = 0 ) {
116 wfProfileIn( __METHOD__
);
118 $dbr = wfGetDB( DB_SLAVE
);
120 /* Get a job from the slave, start with an offset,
121 scan full set afterwards, avoid hitting purged rows
123 NB: If random fetch previously was used, offset
124 will always be ahead of few entries
127 $conditions = self
::defaultQueueConditions();
129 $offset = intval( $offset );
130 $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
132 $row = $dbr->selectRow( 'job', '*',
133 array_merge( $conditions, array( "job_id >= $offset" ) ),
138 // Refetching without offset is needed as some of job IDs could have had delayed commits
139 // and have lower IDs than jobs already executed, blame concurrency :)
141 if ( $row === false ) {
142 if ( $offset != 0 ) {
143 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__
, $options );
146 if ( $row === false ) {
147 wfProfileOut( __METHOD__
);
152 // Try to delete it from the master
153 $dbw = wfGetDB( DB_MASTER
);
154 $dbw->begin( __METHOD__
);
155 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
156 $affected = $dbw->affectedRows();
157 $dbw->commit( __METHOD__
);
160 $dbw->begin( __METHOD__
);
162 // Failed, someone else beat us to it
163 // Try getting a random row
164 $row = $dbw->selectRow( 'job', array( 'minjob' => 'MIN(job_id)',
165 'maxjob' => 'MAX(job_id)' ), '1=1', __METHOD__
);
166 if ( $row === false ||
is_null( $row->minjob
) ||
is_null( $row->maxjob
) ) {
168 $dbw->rollback( __METHOD__
);
169 wfProfileOut( __METHOD__
);
172 // Get the random row
173 $row = $dbw->selectRow( 'job', '*',
174 'job_id >= ' . mt_rand( $row->minjob
, $row->maxjob
), __METHOD__
);
175 if ( $row === false ) {
176 // Random job gone before we got the chance to select it
178 $dbw->rollback( __METHOD__
);
179 wfProfileOut( __METHOD__
);
182 // Delete the random row
183 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
184 $affected = $dbw->affectedRows();
185 $dbw->commit( __METHOD__
);
188 // Random job gone before we exclusively deleted it
190 wfProfileOut( __METHOD__
);
195 // If execution got to here, there's a row in $row that has been deleted from the database
196 // by this thread. Hence the concurrent pop was successful.
197 wfIncrStats( 'job-pop' );
198 $namespace = $row->job_namespace
;
199 $dbkey = $row->job_title
;
200 $title = Title
::makeTitleSafe( $namespace, $dbkey );
202 if ( is_null( $title ) ) {
203 wfProfileOut( __METHOD__
);
207 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
), $row->job_id
);
209 // Remove any duplicates it may have later in the queue
210 $job->removeDuplicates();
212 wfProfileOut( __METHOD__
);
217 * Create the appropriate object to handle a specific job
219 * @param $command String: Job command
220 * @param $title Title: Associated title
221 * @param $params Array|bool: Job parameters
222 * @param $id Int: Job identifier
223 * @throws MWException
226 static function factory( $command, Title
$title, $params = false, $id = 0 ) {
227 global $wgJobClasses;
228 if( isset( $wgJobClasses[$command] ) ) {
229 $class = $wgJobClasses[$command];
230 return new $class( $title, $params, $id );
232 throw new MWException( "Invalid job command `{$command}`" );
239 static function makeBlob( $params ) {
240 if ( $params !== false ) {
241 return serialize( $params );
251 static function extractBlob( $blob ) {
252 if ( (string)$blob !== '' ) {
253 return unserialize( $blob );
260 * Batch-insert a group of jobs into the queue.
261 * This will be wrapped in a transaction with a forced commit.
263 * This may add duplicate at insert time, but they will be
264 * removed later on, when the first one is popped.
266 * @param $jobs array of Job objects
268 static function batchInsert( $jobs ) {
269 if ( !count( $jobs ) ) {
272 $dbw = wfGetDB( DB_MASTER
);
278 foreach ( $jobs as $job ) {
279 $rows[] = $job->insertFields();
280 if ( count( $rows ) >= 50 ) {
281 # Do a small transaction to avoid slave lag
282 $dbw->begin( __METHOD__
);
283 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
284 $dbw->commit( __METHOD__
);
288 if ( $rows ) { // last chunk
289 $dbw->begin( __METHOD__
);
290 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
291 $dbw->commit( __METHOD__
);
293 wfIncrStats( 'job-insert', count( $jobs ) );
297 * Insert a group of jobs into the queue.
299 * Same as batchInsert() but does not commit and can thus
300 * be rolled-back as part of a larger transaction. However,
301 * large batches of jobs can cause slave lag.
303 * @param $jobs array of Job objects
305 static function safeBatchInsert( $jobs ) {
306 if ( !count( $jobs ) ) {
309 $dbw = wfGetDB( DB_MASTER
);
311 foreach ( $jobs as $job ) {
312 $rows[] = $job->insertFields();
313 if ( count( $rows ) >= 500 ) {
314 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
318 if ( $rows ) { // last chunk
319 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
321 wfIncrStats( 'job-insert', count( $jobs ) );
326 * SQL conditions to apply on most JobQueue queries
328 * Whenever we exclude jobs types from the default queue, we want to make
329 * sure that queries to the job queue actually ignore them.
331 * @return array SQL conditions suitable for Database:: methods
333 static function defaultQueueConditions( ) {
334 global $wgJobTypesExcludedFromDefaultQueue;
335 $conditions = array();
336 if ( count( $wgJobTypesExcludedFromDefaultQueue ) > 0 ) {
337 $dbr = wfGetDB( DB_SLAVE
);
338 foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
339 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
345 /*-------------------------------------------------------------------------
346 * Non-static functions
347 *------------------------------------------------------------------------*/
352 * @param $params array|bool
355 function __construct( $command, $title, $params = false, $id = 0 ) {
356 $this->command
= $command;
357 $this->title
= $title;
358 $this->params
= $params;
361 // A bit of premature generalisation
362 // Oh well, the whole class is premature generalisation really
363 $this->removeDuplicates
= true;
367 * Insert a single job into the queue.
368 * @return bool true on success
371 $fields = $this->insertFields();
373 $dbw = wfGetDB( DB_MASTER
);
375 if ( $this->removeDuplicates
) {
376 $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__
);
377 if ( $dbw->numRows( $res ) ) {
381 wfIncrStats( 'job-insert' );
382 return $dbw->insert( 'job', $fields, __METHOD__
);
388 protected function insertFields() {
389 $dbw = wfGetDB( DB_MASTER
);
391 'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
392 'job_cmd' => $this->command
,
393 'job_namespace' => $this->title
->getNamespace(),
394 'job_title' => $this->title
->getDBkey(),
395 'job_timestamp' => $dbw->timestamp(),
396 'job_params' => Job
::makeBlob( $this->params
)
401 * Remove jobs in the job queue which are duplicates of this job.
402 * This is deadlock-prone and so starts its own transaction.
404 function removeDuplicates() {
405 if ( !$this->removeDuplicates
) {
409 $fields = $this->insertFields();
410 unset( $fields['job_id'] );
411 $dbw = wfGetDB( DB_MASTER
);
412 $dbw->begin( __METHOD__
);
413 $dbw->delete( 'job', $fields, __METHOD__
);
414 $affected = $dbw->affectedRows();
415 $dbw->commit( __METHOD__
);
417 wfIncrStats( 'job-dup-delete', $affected );
424 function toString() {
426 if ( $this->params
) {
427 foreach ( $this->params
as $key => $value ) {
428 if ( $paramString != '' ) {
431 $paramString .= "$key=$value";
435 if ( is_object( $this->title
) ) {
436 $s = "{$this->command} " . $this->title
->getPrefixedDBkey();
437 if ( $paramString !== '' ) {
438 $s .= ' ' . $paramString;
442 return "{$this->command} $paramString";
446 protected function setLastError( $error ) {
447 $this->error
= $error;
450 function getLastError() {