<?php
/**
+ * Job queue base code
+ *
+ * @file
* @defgroup JobQueue JobQueue
*/
* @ingroup JobQueue
*/
abstract class Job {
+
+ /**
+ * @var Title
+ */
+ var $title;
+
var $command,
- $title,
$params,
$id,
$removeDuplicates,
* Static functions
*------------------------------------------------------------------------*/
- /**
- * @deprecated use LinksUpdate::queueRecursiveJobs()
- */
- /**
- * static function queueLinksJobs( $titles ) {}
- */
-
/**
* Pop a job of a certain type. This tries less hard than pop() to
* actually find a job; it may be adversely affected by concurrent job
* runners.
+ *
+ * @param $type string
+ *
+ * @return Job
*/
static function pop_type( $type ) {
wfProfilein( __METHOD__ );
$dbw = wfGetDB( DB_MASTER );
+ $dbw->begin();
+
$row = $dbw->selectRow(
'job',
'*',
array( 'job_cmd' => $type ),
__METHOD__,
- array( 'LIMIT' => 1 )
+ array( 'LIMIT' => 1, 'FOR UPDATE' )
);
if ( $row === false ) {
+ $dbw->commit();
wfProfileOut( __METHOD__ );
return false;
}
/* Ensure we "own" this row */
$dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
$affected = $dbw->affectedRows();
+ $dbw->commit();
if ( $affected == 0 ) {
wfProfileOut( __METHOD__ );
return false;
}
+ wfIncrStats( 'job-pop' );
$namespace = $row->job_namespace;
$dbkey = $row->job_title;
$title = Title::makeTitleSafe( $namespace, $dbkey );
$job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
$row->job_id );
- $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
- $dbw->commit();
+ $job->removeDuplicates();
wfProfileOut( __METHOD__ );
return $job;
* @return Job or false if there's no jobs
*/
static function pop( $offset = 0 ) {
+ global $wgJobTypesExcludedFromDefaultQueue;
wfProfileIn( __METHOD__ );
$dbr = wfGetDB( DB_SLAVE );
NB: If random fetch previously was used, offset
will always be ahead of few entries
*/
+ $conditions = array();
+ if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
+ foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
+ $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
+ }
+ }
+ $offset = intval( $offset );
+ $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
- $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__,
- array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
+ $row = $dbr->selectRow( 'job', '*',
+ array_merge( $conditions, array( "job_id >= $offset" ) ),
+ __METHOD__,
+ $options
+ );
// Refetching without offset is needed as some of job IDs could have had delayed commits
// and have lower IDs than jobs already executed, blame concurrency :)
//
if ( $row === false ) {
if ( $offset != 0 ) {
- $row = $dbr->selectRow( 'job', '*', '', __METHOD__,
- array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
+ $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options );
}
if ( $row === false ) {
return false;
}
}
- $offset = $row->job_id;
// Try to delete it from the master
$dbw = wfGetDB( DB_MASTER );
// If execution got to here, there's a row in $row that has been deleted from the database
// by this thread. Hence the concurrent pop was successful.
+ wfIncrStats( 'job-pop' );
$namespace = $row->job_namespace;
$dbkey = $row->job_title;
$title = Title::makeTitleSafe( $namespace, $dbkey );
$job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
// Remove any duplicates it may have later in the queue
- // Deadlock prone section
- $dbw->begin();
- $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
- $dbw->commit();
+ $job->removeDuplicates();
wfProfileOut( __METHOD__ );
return $job;
* @param $jobs array of Job objects
*/
static function batchInsert( $jobs ) {
- if( !count( $jobs ) ) {
+ if ( !count( $jobs ) ) {
return;
}
$dbw = wfGetDB( DB_MASTER );
$rows = array();
- foreach( $jobs as $job ) {
+ foreach ( $jobs as $job ) {
$rows[] = $job->insertFields();
if ( count( $rows ) >= 50 ) {
# Do a small transaction to avoid slave lag
$rows = array();
}
}
- if ( $rows ) {
+ if ( $rows ) { // last chunk
$dbw->begin();
$dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
$dbw->commit();
}
+ wfIncrStats( 'job-insert', count( $jobs ) );
+ }
+
+ /**
+ * Insert a group of jobs into the queue.
+ *
+ * Same as batchInsert() but does not commit and can thus
+ * be rolled-back as part of a larger transaction. However,
+ * large batches of jobs can cause slave lag.
+ *
+ * @param $jobs array of Job objects
+ */
+ static function safeBatchInsert( $jobs ) {
+ if ( !count( $jobs ) ) {
+ return;
+ }
+ $dbw = wfGetDB( DB_MASTER );
+ $rows = array();
+ foreach ( $jobs as $job ) {
+ $rows[] = $job->insertFields();
+ if ( count( $rows ) >= 500 ) {
+ $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
+ $rows = array();
+ }
+ }
+ if ( $rows ) { // last chunk
+ $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
+ }
+ wfIncrStats( 'job-insert', count( $jobs ) );
}
/*-------------------------------------------------------------------------
* Non-static functions
*------------------------------------------------------------------------*/
+ /**
+ * @param $command
+ * @param $title
+ * @param $params array
+ * @param int $id
+ */
function __construct( $command, $title, $params = false, $id = 0 ) {
$this->command = $command;
$this->title = $title;
return;
}
}
+ wfIncrStats( 'job-insert' );
return $dbw->insert( 'job', $fields, __METHOD__ );
}
);
}
+ /**
+ * Remove jobs in the job queue which are duplicates of this job.
+ * This is deadlock-prone and so starts its own transaction.
+ */
+ function removeDuplicates() {
+ if ( !$this->removeDuplicates ) {
+ return;
+ }
+
+ $fields = $this->insertFields();
+ unset( $fields['job_id'] );
+ $dbw = wfGetDB( DB_MASTER );
+ $dbw->begin();
+ $dbw->delete( 'job', $fields, __METHOD__ );
+ $affected = $dbw->affectedRows();
+ $dbw->commit();
+ if ( $affected ) {
+ wfIncrStats( 'job-dup-delete', $affected );
+ }
+ }
+
function toString() {
$paramString = '';
if ( $this->params ) {
dépôts / lhc / web / wiklou.git / blobdiff