includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * Job queue base code
   4  *
   5  * @file
   6  * @defgroup JobQueue JobQueue
   7  */
   8
   9 if ( !defined( 'MEDIAWIKI' ) ) {
  10         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
  11 }
  12
  13 /**
  14  * Class to both describe a background job and handle jobs.
  15  *
  16  * @ingroup JobQueue
  17  */
  18 abstract class Job {
  19
  20         /**
  21          * @var Title
  22          */
  23         var $title;
  24
  25         var $command,
  26                 $params,
  27                 $id,
  28                 $removeDuplicates,
  29                 $error;
  30
  31         /*-------------------------------------------------------------------------
  32          * Abstract functions
  33          *------------------------------------------------------------------------*/
  34
  35         /**
  36          * Run the job
  37          * @return boolean success
  38          */
  39         abstract function run();
  40
  41         /*-------------------------------------------------------------------------
  42          * Static functions
  43          *------------------------------------------------------------------------*/
  44
  45         /**
  46          * Pop a job of a certain type.  This tries less hard than pop() to
  47          * actually find a job; it may be adversely affected by concurrent job
  48          * runners.
  49          *
  50          * @param $type string
  51          *
  52          * @return Job
  53          */
  54         static function pop_type( $type ) {
  55                 wfProfilein( __METHOD__ );
  56
  57                 $dbw = wfGetDB( DB_MASTER );
  58
  59                 $row = $dbw->selectRow(
  60                         'job',
  61                         '*',
  62                         array( 'job_cmd' => $type ),
  63                         __METHOD__,
  64                         array( 'LIMIT' => 1 )
  65                 );
  66
  67                 if ( $row === false ) {
  68                         wfProfileOut( __METHOD__ );
  69                         return false;
  70                 }
  71
  72                 /* Ensure we "own" this row */
  73                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  74                 $affected = $dbw->affectedRows();
  75                 $dbw->commit();
  76
  77                 if ( $affected == 0 ) {
  78                         wfProfileOut( __METHOD__ );
  79                         return false;
  80                 }
  81
  82                 wfIncrStats( 'job-pop' );
  83                 $namespace = $row->job_namespace;
  84                 $dbkey = $row->job_title;
  85                 $title = Title::makeTitleSafe( $namespace, $dbkey );
  86                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
  87                         $row->job_id );
  88
  89                 $job->removeDuplicates();
  90
  91                 wfProfileOut( __METHOD__ );
  92                 return $job;
  93         }
  94
  95         /**
  96          * Pop a job off the front of the queue
  97          *
  98          * @param $offset Integer: Number of jobs to skip
  99          * @return Job or false if there's no jobs
 100          */
 101         static function pop( $offset = 0 ) {
 102                 global $wgJobTypesExcludedFromDefaultQueue;
 103                 wfProfileIn( __METHOD__ );
 104
 105                 $dbr = wfGetDB( DB_SLAVE );
 106
 107                 /* Get a job from the slave, start with an offset,
 108                         scan full set afterwards, avoid hitting purged rows
 109
 110                         NB: If random fetch previously was used, offset
 111                                 will always be ahead of few entries
 112                 */
 113                 $conditions = array();
 114                 if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
 115                         foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
 116                                 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
 117                         }
 118                 }
 119                 $offset = intval( $offset );
 120                 $row = $dbr->selectRow( 'job', '*', array_merge( $conditions, array( "job_id >= $offset" ) ) , __METHOD__,
 121                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 )
 122                 );
 123
 124                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 125                 // and have lower IDs than jobs already executed, blame concurrency :)
 126                 //
 127                 if ( $row === false ) {
 128                         if ( $offset != 0 ) {
 129                                 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__,
 130                                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 131                         }
 132
 133                         if ( $row === false ) {
 134                                 wfProfileOut( __METHOD__ );
 135                                 return false;
 136                         }
 137                 }
 138
 139                 // Try to delete it from the master
 140                 $dbw = wfGetDB( DB_MASTER );
 141                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 142                 $affected = $dbw->affectedRows();
 143                 $dbw->commit();
 144
 145                 if ( !$affected ) {
 146                         // Failed, someone else beat us to it
 147                         // Try getting a random row
 148                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 149                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 150                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 151                                 // No jobs to get
 152                                 wfProfileOut( __METHOD__ );
 153                                 return false;
 154                         }
 155                         // Get the random row
 156                         $row = $dbw->selectRow( 'job', '*',
 157                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 158                         if ( $row === false ) {
 159                                 // Random job gone before we got the chance to select it
 160                                 // Give up
 161                                 wfProfileOut( __METHOD__ );
 162                                 return false;
 163                         }
 164                         // Delete the random row
 165                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 166                         $affected = $dbw->affectedRows();
 167                         $dbw->commit();
 168
 169                         if ( !$affected ) {
 170                                 // Random job gone before we exclusively deleted it
 171                                 // Give up
 172                                 wfProfileOut( __METHOD__ );
 173                                 return false;
 174                         }
 175                 }
 176
 177                 // If execution got to here, there's a row in $row that has been deleted from the database
 178                 // by this thread. Hence the concurrent pop was successful.
 179                 wfIncrStats( 'job-pop' );
 180                 $namespace = $row->job_namespace;
 181                 $dbkey = $row->job_title;
 182                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 183                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 184
 185                 // Remove any duplicates it may have later in the queue
 186                 $job->removeDuplicates();
 187
 188                 wfProfileOut( __METHOD__ );
 189                 return $job;
 190         }
 191
 192         /**
 193          * Create the appropriate object to handle a specific job
 194          *
 195          * @param $command String: Job command
 196          * @param $title Title: Associated title
 197          * @param $params Array: Job parameters
 198          * @param $id Int: Job identifier
 199          * @return Job
 200          */
 201         static function factory( $command, $title, $params = false, $id = 0 ) {
 202                 global $wgJobClasses;
 203                 if( isset( $wgJobClasses[$command] ) ) {
 204                         $class = $wgJobClasses[$command];
 205                         return new $class( $title, $params, $id );
 206                 }
 207                 throw new MWException( "Invalid job command `{$command}`" );
 208         }
 209
 210         static function makeBlob( $params ) {
 211                 if ( $params !== false ) {
 212                         return serialize( $params );
 213                 } else {
 214                         return '';
 215                 }
 216         }
 217
 218         static function extractBlob( $blob ) {
 219                 if ( (string)$blob !== '' ) {
 220                         return unserialize( $blob );
 221                 } else {
 222                         return false;
 223                 }
 224         }
 225
 226         /**
 227          * Batch-insert a group of jobs into the queue.
 228          * This will be wrapped in a transaction with a forced commit.
 229          *
 230          * This may add duplicate at insert time, but they will be
 231          * removed later on, when the first one is popped.
 232          *
 233          * @param $jobs array of Job objects
 234          */
 235         static function batchInsert( $jobs ) {
 236                 if ( !count( $jobs ) ) {
 237                         return;
 238                 }
 239                 $dbw = wfGetDB( DB_MASTER );
 240                 $rows = array();
 241                 foreach ( $jobs as $job ) {
 242                         $rows[] = $job->insertFields();
 243                         if ( count( $rows ) >= 50 ) {
 244                                 # Do a small transaction to avoid slave lag
 245                                 $dbw->begin();
 246                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 247                                 $dbw->commit();
 248                                 $rows = array();
 249                         }
 250                 }
 251                 if ( $rows ) { // last chunk
 252                         $dbw->begin();
 253                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 254                         $dbw->commit();
 255                 }
 256                 wfIncrStats( 'job-insert', count( $jobs ) );
 257         }
 258
 259         /**
 260          * Insert a group of jobs into the queue.
 261          *
 262          * Same as batchInsert() but does not commit and can thus
 263          * be rolled-back as part of a larger transaction. However,
 264          * large batches of jobs can cause slave lag.
 265          *
 266          * @param $jobs array of Job objects
 267          */
 268         static function safeBatchInsert( $jobs ) {
 269                 if ( !count( $jobs ) ) {
 270                         return;
 271                 }
 272                 $dbw = wfGetDB( DB_MASTER );
 273                 $rows = array();
 274                 foreach ( $jobs as $job ) {
 275                         $rows[] = $job->insertFields();
 276                         if ( count( $rows ) >= 500 ) {
 277                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 278                                 $rows = array();
 279                         }
 280                 }
 281                 if ( $rows ) { // last chunk
 282                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 283                 }
 284                 wfIncrStats( 'job-insert', count( $jobs ) );
 285         }
 286
 287         /*-------------------------------------------------------------------------
 288          * Non-static functions
 289          *------------------------------------------------------------------------*/
 290
 291         /**
 292          * @param $command
 293          * @param $title
 294          * @param $params array
 295          * @param int $id
 296          */
 297         function __construct( $command, $title, $params = false, $id = 0 ) {
 298                 $this->command = $command;
 299                 $this->title = $title;
 300                 $this->params = $params;
 301                 $this->id = $id;
 302
 303                 // A bit of premature generalisation
 304                 // Oh well, the whole class is premature generalisation really
 305                 $this->removeDuplicates = true;
 306         }
 307
 308         /**
 309          * Insert a single job into the queue.
 310          * @return bool true on success
 311          */
 312         function insert() {
 313                 $fields = $this->insertFields();
 314
 315                 $dbw = wfGetDB( DB_MASTER );
 316
 317                 if ( $this->removeDuplicates ) {
 318                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 319                         if ( $dbw->numRows( $res ) ) {
 320                                 return;
 321                         }
 322                 }
 323                 wfIncrStats( 'job-insert' );
 324                 return $dbw->insert( 'job', $fields, __METHOD__ );
 325         }
 326
 327         protected function insertFields() {
 328                 $dbw = wfGetDB( DB_MASTER );
 329                 return array(
 330                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 331                         'job_cmd' => $this->command,
 332                         'job_namespace' => $this->title->getNamespace(),
 333                         'job_title' => $this->title->getDBkey(),
 334                         'job_params' => Job::makeBlob( $this->params )
 335                 );
 336         }
 337
 338         /**
 339          * Remove jobs in the job queue which are duplicates of this job.
 340          * This is deadlock-prone and so starts its own transaction.
 341          */
 342         function removeDuplicates() {
 343                 if ( !$this->removeDuplicates ) {
 344                         return;
 345                 }
 346
 347                 $fields = $this->insertFields();
 348                 unset( $fields['job_id'] );
 349                 $dbw = wfGetDB( DB_MASTER );
 350                 $dbw->begin();
 351                 $dbw->delete( 'job', $fields, __METHOD__ );
 352                 $affected = $dbw->affectedRows();
 353                 $dbw->commit();
 354                 if ( $affected ) {
 355                         wfIncrStats( 'job-dup-delete', $affected );
 356                 }
 357         }
 358
 359         function toString() {
 360                 $paramString = '';
 361                 if ( $this->params ) {
 362                         foreach ( $this->params as $key => $value ) {
 363                                 if ( $paramString != '' ) {
 364                                         $paramString .= ' ';
 365                                 }
 366                                 $paramString .= "$key=$value";
 367                         }
 368                 }
 369
 370                 if ( is_object( $this->title ) ) {
 371                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 372                         if ( $paramString !== '' ) {
 373                                 $s .= ' ' . $paramString;
 374                         }
 375                         return $s;
 376                 } else {
 377                         return "{$this->command} $paramString";
 378                 }
 379         }
 380
 381         protected function setLastError( $error ) {
 382                 $this->error = $error;
 383         }
 384
 385         function getLastError() {
 386                 return $this->error;
 387         }
 388 }