From: Aaron Date: Fri, 22 Jun 2012 22:15:21 +0000 (-0700) Subject: Moved JobQueue.php file to Job.php X-Git-Tag: 1.31.0-rc.0~23252^2 X-Git-Url: http://git.cyclocoop.org/%27%20.generer_url_ecrire%28_request%28%27exec%27%29%2C%27cmd=switch&outil=rss_couteau_suisse%27%29.%27?a=commitdiff_plain;h=0c187a776c90fdce647bf5299774f7952763ba30;p=lhc%2Fweb%2Fwiklou.git Moved JobQueue.php file to Job.php Change-Id: I0c2fd3e4095a61db96b7165d1f9bf79fff51ea72 --- diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index d7927de7f5..d92aa99f52 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -630,7 +630,7 @@ $wgAutoloadLocalClasses = array( 'DoubleRedirectJob' => 'includes/job/DoubleRedirectJob.php', 'EmaillingJob' => 'includes/job/EmaillingJob.php', 'EnotifNotifyJob' => 'includes/job/EnotifNotifyJob.php', - 'Job' => 'includes/job/JobQueue.php', + 'Job' => 'includes/job/Job.php', 'RefreshLinksJob' => 'includes/job/RefreshLinksJob.php', 'RefreshLinksJob2' => 'includes/job/RefreshLinksJob.php', 'UploadFromUrlJob' => 'includes/job/UploadFromUrlJob.php', diff --git a/includes/job/Job.php b/includes/job/Job.php new file mode 100644 index 0000000000..7b7ec0c22a --- /dev/null +++ b/includes/job/Job.php @@ -0,0 +1,447 @@ +begin( __METHOD__ ); + + $row = $dbw->selectRow( + 'job', + '*', + array( 'job_cmd' => $type ), + __METHOD__, + array( 'LIMIT' => 1, 'FOR UPDATE' ) + ); + + if ( $row === false ) { + $dbw->commit( __METHOD__ ); + wfProfileOut( __METHOD__ ); + return false; + } + + /* Ensure we "own" this row */ + $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); + $affected = $dbw->affectedRows(); + $dbw->commit( __METHOD__ ); + + if ( $affected == 0 ) { + wfProfileOut( __METHOD__ ); + return false; + } + + wfIncrStats( 'job-pop' ); + $namespace = $row->job_namespace; + $dbkey = $row->job_title; + $title = Title::makeTitleSafe( $namespace, $dbkey ); + $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), + $row->job_id ); + + $job->removeDuplicates(); + + wfProfileOut( __METHOD__ ); + return $job; + } + + /** + * Pop a job off the front of the queue + * + * @param $offset Integer: Number of jobs to skip + * @return Job or false if there's no jobs + */ + static function pop( $offset = 0 ) { + wfProfileIn( __METHOD__ ); + + $dbr = wfGetDB( DB_SLAVE ); + + /* Get a job from the slave, start with an offset, + scan full set afterwards, avoid hitting purged rows + + NB: If random fetch previously was used, offset + will always be ahead of few entries + */ + + $conditions = self::defaultQueueConditions(); + + $offset = intval( $offset ); + $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' ); + + $row = $dbr->selectRow( 'job', '*', + array_merge( $conditions, array( "job_id >= $offset" ) ), + __METHOD__, + $options + ); + + // Refetching without offset is needed as some of job IDs could have had delayed commits + // and have lower IDs than jobs already executed, blame concurrency :) + // + if ( $row === false ) { + if ( $offset != 0 ) { + $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options ); + } + + if ( $row === false ) { + wfProfileOut( __METHOD__ ); + return false; + } + } + + // Try to delete it from the master + $dbw = wfGetDB( DB_MASTER ); + $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); + $affected = $dbw->affectedRows(); + $dbw->commit( __METHOD__ ); + + if ( !$affected ) { + // Failed, someone else beat us to it + // Try getting a random row + $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob', + 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ ); + if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) { + // No jobs to get + wfProfileOut( __METHOD__ ); + return false; + } + // Get the random row + $row = $dbw->selectRow( 'job', '*', + 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ ); + if ( $row === false ) { + // Random job gone before we got the chance to select it + // Give up + wfProfileOut( __METHOD__ ); + return false; + } + // Delete the random row + $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); + $affected = $dbw->affectedRows(); + $dbw->commit( __METHOD__ ); + + if ( !$affected ) { + // Random job gone before we exclusively deleted it + // Give up + wfProfileOut( __METHOD__ ); + return false; + } + } + + // If execution got to here, there's a row in $row that has been deleted from the database + // by this thread. Hence the concurrent pop was successful. + wfIncrStats( 'job-pop' ); + $namespace = $row->job_namespace; + $dbkey = $row->job_title; + $title = Title::makeTitleSafe( $namespace, $dbkey ); + + if ( is_null( $title ) ) { + wfProfileOut( __METHOD__ ); + return false; + } + + $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id ); + + // Remove any duplicates it may have later in the queue + $job->removeDuplicates(); + + wfProfileOut( __METHOD__ ); + return $job; + } + + /** + * Create the appropriate object to handle a specific job + * + * @param $command String: Job command + * @param $title Title: Associated title + * @param $params Array: Job parameters + * @param $id Int: Job identifier + * @return Job + */ + static function factory( $command, Title $title, $params = false, $id = 0 ) { + global $wgJobClasses; + if( isset( $wgJobClasses[$command] ) ) { + $class = $wgJobClasses[$command]; + return new $class( $title, $params, $id ); + } + throw new MWException( "Invalid job command `{$command}`" ); + } + + /** + * @param $params + * @return string + */ + static function makeBlob( $params ) { + if ( $params !== false ) { + return serialize( $params ); + } else { + return ''; + } + } + + /** + * @param $blob + * @return bool|mixed + */ + static function extractBlob( $blob ) { + if ( (string)$blob !== '' ) { + return unserialize( $blob ); + } else { + return false; + } + } + + /** + * Batch-insert a group of jobs into the queue. + * This will be wrapped in a transaction with a forced commit. + * + * This may add duplicate at insert time, but they will be + * removed later on, when the first one is popped. + * + * @param $jobs array of Job objects + */ + static function batchInsert( $jobs ) { + if ( !count( $jobs ) ) { + return; + } + $dbw = wfGetDB( DB_MASTER ); + $rows = array(); + + /** + * @var $job Job + */ + foreach ( $jobs as $job ) { + $rows[] = $job->insertFields(); + if ( count( $rows ) >= 50 ) { + # Do a small transaction to avoid slave lag + $dbw->begin( __METHOD__ ); + $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); + $dbw->commit( __METHOD__ ); + $rows = array(); + } + } + if ( $rows ) { // last chunk + $dbw->begin( __METHOD__ ); + $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); + $dbw->commit( __METHOD__ ); + } + wfIncrStats( 'job-insert', count( $jobs ) ); + } + + /** + * Insert a group of jobs into the queue. + * + * Same as batchInsert() but does not commit and can thus + * be rolled-back as part of a larger transaction. However, + * large batches of jobs can cause slave lag. + * + * @param $jobs array of Job objects + */ + static function safeBatchInsert( $jobs ) { + if ( !count( $jobs ) ) { + return; + } + $dbw = wfGetDB( DB_MASTER ); + $rows = array(); + foreach ( $jobs as $job ) { + $rows[] = $job->insertFields(); + if ( count( $rows ) >= 500 ) { + $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); + $rows = array(); + } + } + if ( $rows ) { // last chunk + $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); + } + wfIncrStats( 'job-insert', count( $jobs ) ); + } + + + /** + * SQL conditions to apply on most JobQueue queries + * + * Whenever we exclude jobs types from the default queue, we want to make + * sure that queries to the job queue actually ignore them. + * + * @return array SQL conditions suitable for Database:: methods + */ + static function defaultQueueConditions( ) { + global $wgJobTypesExcludedFromDefaultQueue; + $conditions = array(); + if ( count( $wgJobTypesExcludedFromDefaultQueue ) > 0 ) { + $dbr = wfGetDB( DB_SLAVE ); + foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) { + $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType ); + } + } + return $conditions; + } + + /*------------------------------------------------------------------------- + * Non-static functions + *------------------------------------------------------------------------*/ + + /** + * @param $command + * @param $title + * @param $params array + * @param int $id + */ + function __construct( $command, $title, $params = false, $id = 0 ) { + $this->command = $command; + $this->title = $title; + $this->params = $params; + $this->id = $id; + + // A bit of premature generalisation + // Oh well, the whole class is premature generalisation really + $this->removeDuplicates = true; + } + + /** + * Insert a single job into the queue. + * @return bool true on success + */ + function insert() { + $fields = $this->insertFields(); + + $dbw = wfGetDB( DB_MASTER ); + + if ( $this->removeDuplicates ) { + $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ ); + if ( $dbw->numRows( $res ) ) { + return true; + } + } + wfIncrStats( 'job-insert' ); + return $dbw->insert( 'job', $fields, __METHOD__ ); + } + + /** + * @return array + */ + protected function insertFields() { + $dbw = wfGetDB( DB_MASTER ); + return array( + 'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ), + 'job_cmd' => $this->command, + 'job_namespace' => $this->title->getNamespace(), + 'job_title' => $this->title->getDBkey(), + 'job_timestamp' => $dbw->timestamp(), + 'job_params' => Job::makeBlob( $this->params ) + ); + } + + /** + * Remove jobs in the job queue which are duplicates of this job. + * This is deadlock-prone and so starts its own transaction. + */ + function removeDuplicates() { + if ( !$this->removeDuplicates ) { + return; + } + + $fields = $this->insertFields(); + unset( $fields['job_id'] ); + $dbw = wfGetDB( DB_MASTER ); + $dbw->begin( __METHOD__ ); + $dbw->delete( 'job', $fields, __METHOD__ ); + $affected = $dbw->affectedRows(); + $dbw->commit( __METHOD__ ); + if ( $affected ) { + wfIncrStats( 'job-dup-delete', $affected ); + } + } + + /** + * @return string + */ + function toString() { + $paramString = ''; + if ( $this->params ) { + foreach ( $this->params as $key => $value ) { + if ( $paramString != '' ) { + $paramString .= ' '; + } + $paramString .= "$key=$value"; + } + } + + if ( is_object( $this->title ) ) { + $s = "{$this->command} " . $this->title->getPrefixedDBkey(); + if ( $paramString !== '' ) { + $s .= ' ' . $paramString; + } + return $s; + } else { + return "{$this->command} $paramString"; + } + } + + protected function setLastError( $error ) { + $this->error = $error; + } + + function getLastError() { + return $this->error; + } +} diff --git a/includes/job/JobQueue.php b/includes/job/JobQueue.php deleted file mode 100644 index 7b7ec0c22a..0000000000 --- a/includes/job/JobQueue.php +++ /dev/null @@ -1,447 +0,0 @@ -begin( __METHOD__ ); - - $row = $dbw->selectRow( - 'job', - '*', - array( 'job_cmd' => $type ), - __METHOD__, - array( 'LIMIT' => 1, 'FOR UPDATE' ) - ); - - if ( $row === false ) { - $dbw->commit( __METHOD__ ); - wfProfileOut( __METHOD__ ); - return false; - } - - /* Ensure we "own" this row */ - $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); - $affected = $dbw->affectedRows(); - $dbw->commit( __METHOD__ ); - - if ( $affected == 0 ) { - wfProfileOut( __METHOD__ ); - return false; - } - - wfIncrStats( 'job-pop' ); - $namespace = $row->job_namespace; - $dbkey = $row->job_title; - $title = Title::makeTitleSafe( $namespace, $dbkey ); - $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), - $row->job_id ); - - $job->removeDuplicates(); - - wfProfileOut( __METHOD__ ); - return $job; - } - - /** - * Pop a job off the front of the queue - * - * @param $offset Integer: Number of jobs to skip - * @return Job or false if there's no jobs - */ - static function pop( $offset = 0 ) { - wfProfileIn( __METHOD__ ); - - $dbr = wfGetDB( DB_SLAVE ); - - /* Get a job from the slave, start with an offset, - scan full set afterwards, avoid hitting purged rows - - NB: If random fetch previously was used, offset - will always be ahead of few entries - */ - - $conditions = self::defaultQueueConditions(); - - $offset = intval( $offset ); - $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' ); - - $row = $dbr->selectRow( 'job', '*', - array_merge( $conditions, array( "job_id >= $offset" ) ), - __METHOD__, - $options - ); - - // Refetching without offset is needed as some of job IDs could have had delayed commits - // and have lower IDs than jobs already executed, blame concurrency :) - // - if ( $row === false ) { - if ( $offset != 0 ) { - $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options ); - } - - if ( $row === false ) { - wfProfileOut( __METHOD__ ); - return false; - } - } - - // Try to delete it from the master - $dbw = wfGetDB( DB_MASTER ); - $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); - $affected = $dbw->affectedRows(); - $dbw->commit( __METHOD__ ); - - if ( !$affected ) { - // Failed, someone else beat us to it - // Try getting a random row - $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob', - 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ ); - if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) { - // No jobs to get - wfProfileOut( __METHOD__ ); - return false; - } - // Get the random row - $row = $dbw->selectRow( 'job', '*', - 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ ); - if ( $row === false ) { - // Random job gone before we got the chance to select it - // Give up - wfProfileOut( __METHOD__ ); - return false; - } - // Delete the random row - $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); - $affected = $dbw->affectedRows(); - $dbw->commit( __METHOD__ ); - - if ( !$affected ) { - // Random job gone before we exclusively deleted it - // Give up - wfProfileOut( __METHOD__ ); - return false; - } - } - - // If execution got to here, there's a row in $row that has been deleted from the database - // by this thread. Hence the concurrent pop was successful. - wfIncrStats( 'job-pop' ); - $namespace = $row->job_namespace; - $dbkey = $row->job_title; - $title = Title::makeTitleSafe( $namespace, $dbkey ); - - if ( is_null( $title ) ) { - wfProfileOut( __METHOD__ ); - return false; - } - - $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id ); - - // Remove any duplicates it may have later in the queue - $job->removeDuplicates(); - - wfProfileOut( __METHOD__ ); - return $job; - } - - /** - * Create the appropriate object to handle a specific job - * - * @param $command String: Job command - * @param $title Title: Associated title - * @param $params Array: Job parameters - * @param $id Int: Job identifier - * @return Job - */ - static function factory( $command, Title $title, $params = false, $id = 0 ) { - global $wgJobClasses; - if( isset( $wgJobClasses[$command] ) ) { - $class = $wgJobClasses[$command]; - return new $class( $title, $params, $id ); - } - throw new MWException( "Invalid job command `{$command}`" ); - } - - /** - * @param $params - * @return string - */ - static function makeBlob( $params ) { - if ( $params !== false ) { - return serialize( $params ); - } else { - return ''; - } - } - - /** - * @param $blob - * @return bool|mixed - */ - static function extractBlob( $blob ) { - if ( (string)$blob !== '' ) { - return unserialize( $blob ); - } else { - return false; - } - } - - /** - * Batch-insert a group of jobs into the queue. - * This will be wrapped in a transaction with a forced commit. - * - * This may add duplicate at insert time, but they will be - * removed later on, when the first one is popped. - * - * @param $jobs array of Job objects - */ - static function batchInsert( $jobs ) { - if ( !count( $jobs ) ) { - return; - } - $dbw = wfGetDB( DB_MASTER ); - $rows = array(); - - /** - * @var $job Job - */ - foreach ( $jobs as $job ) { - $rows[] = $job->insertFields(); - if ( count( $rows ) >= 50 ) { - # Do a small transaction to avoid slave lag - $dbw->begin( __METHOD__ ); - $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - $dbw->commit( __METHOD__ ); - $rows = array(); - } - } - if ( $rows ) { // last chunk - $dbw->begin( __METHOD__ ); - $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - $dbw->commit( __METHOD__ ); - } - wfIncrStats( 'job-insert', count( $jobs ) ); - } - - /** - * Insert a group of jobs into the queue. - * - * Same as batchInsert() but does not commit and can thus - * be rolled-back as part of a larger transaction. However, - * large batches of jobs can cause slave lag. - * - * @param $jobs array of Job objects - */ - static function safeBatchInsert( $jobs ) { - if ( !count( $jobs ) ) { - return; - } - $dbw = wfGetDB( DB_MASTER ); - $rows = array(); - foreach ( $jobs as $job ) { - $rows[] = $job->insertFields(); - if ( count( $rows ) >= 500 ) { - $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - $rows = array(); - } - } - if ( $rows ) { // last chunk - $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - } - wfIncrStats( 'job-insert', count( $jobs ) ); - } - - - /** - * SQL conditions to apply on most JobQueue queries - * - * Whenever we exclude jobs types from the default queue, we want to make - * sure that queries to the job queue actually ignore them. - * - * @return array SQL conditions suitable for Database:: methods - */ - static function defaultQueueConditions( ) { - global $wgJobTypesExcludedFromDefaultQueue; - $conditions = array(); - if ( count( $wgJobTypesExcludedFromDefaultQueue ) > 0 ) { - $dbr = wfGetDB( DB_SLAVE ); - foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) { - $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType ); - } - } - return $conditions; - } - - /*------------------------------------------------------------------------- - * Non-static functions - *------------------------------------------------------------------------*/ - - /** - * @param $command - * @param $title - * @param $params array - * @param int $id - */ - function __construct( $command, $title, $params = false, $id = 0 ) { - $this->command = $command; - $this->title = $title; - $this->params = $params; - $this->id = $id; - - // A bit of premature generalisation - // Oh well, the whole class is premature generalisation really - $this->removeDuplicates = true; - } - - /** - * Insert a single job into the queue. - * @return bool true on success - */ - function insert() { - $fields = $this->insertFields(); - - $dbw = wfGetDB( DB_MASTER ); - - if ( $this->removeDuplicates ) { - $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ ); - if ( $dbw->numRows( $res ) ) { - return true; - } - } - wfIncrStats( 'job-insert' ); - return $dbw->insert( 'job', $fields, __METHOD__ ); - } - - /** - * @return array - */ - protected function insertFields() { - $dbw = wfGetDB( DB_MASTER ); - return array( - 'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ), - 'job_cmd' => $this->command, - 'job_namespace' => $this->title->getNamespace(), - 'job_title' => $this->title->getDBkey(), - 'job_timestamp' => $dbw->timestamp(), - 'job_params' => Job::makeBlob( $this->params ) - ); - } - - /** - * Remove jobs in the job queue which are duplicates of this job. - * This is deadlock-prone and so starts its own transaction. - */ - function removeDuplicates() { - if ( !$this->removeDuplicates ) { - return; - } - - $fields = $this->insertFields(); - unset( $fields['job_id'] ); - $dbw = wfGetDB( DB_MASTER ); - $dbw->begin( __METHOD__ ); - $dbw->delete( 'job', $fields, __METHOD__ ); - $affected = $dbw->affectedRows(); - $dbw->commit( __METHOD__ ); - if ( $affected ) { - wfIncrStats( 'job-dup-delete', $affected ); - } - } - - /** - * @return string - */ - function toString() { - $paramString = ''; - if ( $this->params ) { - foreach ( $this->params as $key => $value ) { - if ( $paramString != '' ) { - $paramString .= ' '; - } - $paramString .= "$key=$value"; - } - } - - if ( is_object( $this->title ) ) { - $s = "{$this->command} " . $this->title->getPrefixedDBkey(); - if ( $paramString !== '' ) { - $s .= ' ' . $paramString; - } - return $s; - } else { - return "{$this->command} $paramString"; - } - } - - protected function setLastError( $error ) { - $this->error = $error; - } - - function getLastError() { - return $this->error; - } -}