<?php
/**
+ * Job queue base code.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
* @defgroup JobQueue JobQueue
*/
-if ( !defined( 'MEDIAWIKI' ) ) {
- die( "This file is part of MediaWiki, it is not a valid entry point\n" );
-}
-
/**
* Class to both describe a background job and handle jobs.
*
* @ingroup JobQueue
*/
abstract class Job {
+
+ /**
+ * @var Title
+ */
+ var $title;
+
var $command,
- $title,
$params,
$id,
$removeDuplicates,
* Static functions
*------------------------------------------------------------------------*/
- /**
- * @deprecated use LinksUpdate::queueRecursiveJobs()
- */
- /**
- * static function queueLinksJobs( $titles ) {}
- */
-
/**
* Pop a job of a certain type. This tries less hard than pop() to
* actually find a job; it may be adversely affected by concurrent job
* runners.
+ *
+ * @param $type string
+ *
+ * @return Job
*/
static function pop_type( $type ) {
wfProfilein( __METHOD__ );
$dbw = wfGetDB( DB_MASTER );
+ $dbw->begin( __METHOD__ );
+
$row = $dbw->selectRow(
'job',
'*',
array( 'job_cmd' => $type ),
__METHOD__,
- array( 'LIMIT' => 1 )
+ array( 'LIMIT' => 1, 'FOR UPDATE' )
);
if ( $row === false ) {
+ $dbw->commit( __METHOD__ );
wfProfileOut( __METHOD__ );
return false;
}
/* Ensure we "own" this row */
$dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
$affected = $dbw->affectedRows();
+ $dbw->commit( __METHOD__ );
if ( $affected == 0 ) {
wfProfileOut( __METHOD__ );
return false;
}
+ wfIncrStats( 'job-pop' );
$namespace = $row->job_namespace;
$dbkey = $row->job_title;
$title = Title::makeTitleSafe( $namespace, $dbkey );
$job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
$row->job_id );
- $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
- $dbw->commit();
+ $job->removeDuplicates();
wfProfileOut( __METHOD__ );
return $job;
will always be ahead of few entries
*/
- $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__,
- array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
+ $conditions = self::defaultQueueConditions();
+
+ $offset = intval( $offset );
+ $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
+
+ $row = $dbr->selectRow( 'job', '*',
+ array_merge( $conditions, array( "job_id >= $offset" ) ),
+ __METHOD__,
+ $options
+ );
// Refetching without offset is needed as some of job IDs could have had delayed commits
// and have lower IDs than jobs already executed, blame concurrency :)
//
if ( $row === false ) {
if ( $offset != 0 ) {
- $row = $dbr->selectRow( 'job', '*', '', __METHOD__,
- array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
+ $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options );
}
if ( $row === false ) {
return false;
}
}
- $offset = $row->job_id;
// Try to delete it from the master
$dbw = wfGetDB( DB_MASTER );
$dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
$affected = $dbw->affectedRows();
- $dbw->commit();
+ $dbw->commit( __METHOD__ );
if ( !$affected ) {
// Failed, someone else beat us to it
// Delete the random row
$dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
$affected = $dbw->affectedRows();
- $dbw->commit();
+ $dbw->commit( __METHOD__ );
if ( !$affected ) {
// Random job gone before we exclusively deleted it
// If execution got to here, there's a row in $row that has been deleted from the database
// by this thread. Hence the concurrent pop was successful.
+ wfIncrStats( 'job-pop' );
$namespace = $row->job_namespace;
$dbkey = $row->job_title;
$title = Title::makeTitleSafe( $namespace, $dbkey );
+
+ if ( is_null( $title ) ) {
+ wfProfileOut( __METHOD__ );
+ return false;
+ }
+
$job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
// Remove any duplicates it may have later in the queue
- // Deadlock prone section
- $dbw->begin();
- $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
- $dbw->commit();
+ $job->removeDuplicates();
wfProfileOut( __METHOD__ );
return $job;
* @param $id Int: Job identifier
* @return Job
*/
- static function factory( $command, $title, $params = false, $id = 0 ) {
+ static function factory( $command, Title $title, $params = false, $id = 0 ) {
global $wgJobClasses;
if( isset( $wgJobClasses[$command] ) ) {
$class = $wgJobClasses[$command];
throw new MWException( "Invalid job command `{$command}`" );
}
+ /**
+ * @param $params
+ * @return string
+ */
static function makeBlob( $params ) {
if ( $params !== false ) {
return serialize( $params );
}
}
+ /**
+ * @param $blob
+ * @return bool|mixed
+ */
static function extractBlob( $blob ) {
if ( (string)$blob !== '' ) {
return unserialize( $blob );
* @param $jobs array of Job objects
*/
static function batchInsert( $jobs ) {
- if( !count( $jobs ) ) {
+ if ( !count( $jobs ) ) {
return;
}
$dbw = wfGetDB( DB_MASTER );
$rows = array();
- foreach( $jobs as $job ) {
+
+ /**
+ * @var $job Job
+ */
+ foreach ( $jobs as $job ) {
$rows[] = $job->insertFields();
if ( count( $rows ) >= 50 ) {
# Do a small transaction to avoid slave lag
- $dbw->begin();
+ $dbw->begin( __METHOD__ );
$dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
- $dbw->commit();
+ $dbw->commit( __METHOD__ );
$rows = array();
}
}
- if ( $rows ) {
- $dbw->begin();
+ if ( $rows ) { // last chunk
+ $dbw->begin( __METHOD__ );
$dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
- $dbw->commit();
+ $dbw->commit( __METHOD__ );
}
+ wfIncrStats( 'job-insert', count( $jobs ) );
+ }
+
+ /**
+ * Insert a group of jobs into the queue.
+ *
+ * Same as batchInsert() but does not commit and can thus
+ * be rolled-back as part of a larger transaction. However,
+ * large batches of jobs can cause slave lag.
+ *
+ * @param $jobs array of Job objects
+ */
+ static function safeBatchInsert( $jobs ) {
+ if ( !count( $jobs ) ) {
+ return;
+ }
+ $dbw = wfGetDB( DB_MASTER );
+ $rows = array();
+ foreach ( $jobs as $job ) {
+ $rows[] = $job->insertFields();
+ if ( count( $rows ) >= 500 ) {
+ $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
+ $rows = array();
+ }
+ }
+ if ( $rows ) { // last chunk
+ $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
+ }
+ wfIncrStats( 'job-insert', count( $jobs ) );
+ }
+
+
+ /**
+ * SQL conditions to apply on most JobQueue queries
+ *
+ * Whenever we exclude jobs types from the default queue, we want to make
+ * sure that queries to the job queue actually ignore them.
+ *
+ * @return array SQL conditions suitable for Database:: methods
+ */
+ static function defaultQueueConditions( ) {
+ global $wgJobTypesExcludedFromDefaultQueue;
+ $conditions = array();
+ if ( count( $wgJobTypesExcludedFromDefaultQueue ) > 0 ) {
+ $dbr = wfGetDB( DB_SLAVE );
+ foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
+ $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
+ }
+ }
+ return $conditions;
}
/*-------------------------------------------------------------------------
* Non-static functions
*------------------------------------------------------------------------*/
+ /**
+ * @param $command
+ * @param $title
+ * @param $params array
+ * @param int $id
+ */
function __construct( $command, $title, $params = false, $id = 0 ) {
$this->command = $command;
$this->title = $title;
if ( $this->removeDuplicates ) {
$res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
if ( $dbw->numRows( $res ) ) {
- return;
+ return true;
}
}
+ wfIncrStats( 'job-insert' );
return $dbw->insert( 'job', $fields, __METHOD__ );
}
+ /**
+ * @return array
+ */
protected function insertFields() {
$dbw = wfGetDB( DB_MASTER );
return array(
'job_cmd' => $this->command,
'job_namespace' => $this->title->getNamespace(),
'job_title' => $this->title->getDBkey(),
+ 'job_timestamp' => $dbw->timestamp(),
'job_params' => Job::makeBlob( $this->params )
);
}
+ /**
+ * Remove jobs in the job queue which are duplicates of this job.
+ * This is deadlock-prone and so starts its own transaction.
+ */
+ function removeDuplicates() {
+ if ( !$this->removeDuplicates ) {
+ return;
+ }
+
+ $fields = $this->insertFields();
+ unset( $fields['job_id'] );
+ $dbw = wfGetDB( DB_MASTER );
+ $dbw->begin( __METHOD__ );
+ $dbw->delete( 'job', $fields, __METHOD__ );
+ $affected = $dbw->affectedRows();
+ $dbw->commit( __METHOD__ );
+ if ( $affected ) {
+ wfIncrStats( 'job-dup-delete', $affected );
+ }
+ }
+
+ /**
+ * @return string
+ */
function toString() {
$paramString = '';
if ( $this->params ) {
dépôts / lhc / web / wiklou.git / blobdiff