includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * Job queue base code
   4  *
   5  * @file
   6  * @defgroup JobQueue JobQueue
   7  */
   8
   9 if ( !defined( 'MEDIAWIKI' ) ) {
  10         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
  11 }
  12
  13 /**
  14  * Class to both describe a background job and handle jobs.
  15  *
  16  * @ingroup JobQueue
  17  */
  18 abstract class Job {
  19
  20         /**
  21          * @var Title
  22          */
  23         var $title;
  24
  25         var $command,
  26                 $params,
  27                 $id,
  28                 $removeDuplicates,
  29                 $error;
  30
  31         /*-------------------------------------------------------------------------
  32          * Abstract functions
  33          *------------------------------------------------------------------------*/
  34
  35         /**
  36          * Run the job
  37          * @return boolean success
  38          */
  39         abstract function run();
  40
  41         /*-------------------------------------------------------------------------
  42          * Static functions
  43          *------------------------------------------------------------------------*/
  44
  45         /**
  46          * Pop a job of a certain type.  This tries less hard than pop() to
  47          * actually find a job; it may be adversely affected by concurrent job
  48          * runners.
  49          *
  50          * @param $type string
  51          *
  52          * @return Job
  53          */
  54         static function pop_type( $type ) {
  55                 wfProfilein( __METHOD__ );
  56
  57                 $dbw = wfGetDB( DB_MASTER );
  58
  59                 $dbw->begin( __METHOD__ );
  60
  61                 $row = $dbw->selectRow(
  62                         'job',
  63                         '*',
  64                         array( 'job_cmd' => $type ),
  65                         __METHOD__,
  66                         array( 'LIMIT' => 1, 'FOR UPDATE' )
  67                 );
  68
  69                 if ( $row === false ) {
  70                         $dbw->commit( __METHOD__ );
  71                         wfProfileOut( __METHOD__ );
  72                         return false;
  73                 }
  74
  75                 /* Ensure we "own" this row */
  76                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  77                 $affected = $dbw->affectedRows();
  78                 $dbw->commit( __METHOD__ );
  79
  80                 if ( $affected == 0 ) {
  81                         wfProfileOut( __METHOD__ );
  82                         return false;
  83                 }
  84
  85                 wfIncrStats( 'job-pop' );
  86                 $namespace = $row->job_namespace;
  87                 $dbkey = $row->job_title;
  88                 $title = Title::makeTitleSafe( $namespace, $dbkey );
  89                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
  90                         $row->job_id );
  91
  92                 $job->removeDuplicates();
  93
  94                 wfProfileOut( __METHOD__ );
  95                 return $job;
  96         }
  97
  98         /**
  99          * Pop a job off the front of the queue
 100          *
 101          * @param $offset Integer: Number of jobs to skip
 102          * @return Job or false if there's no jobs
 103          */
 104         static function pop( $offset = 0 ) {
 105                 global $wgJobTypesExcludedFromDefaultQueue;
 106                 wfProfileIn( __METHOD__ );
 107
 108                 $dbr = wfGetDB( DB_SLAVE );
 109
 110                 /* Get a job from the slave, start with an offset,
 111                         scan full set afterwards, avoid hitting purged rows
 112
 113                         NB: If random fetch previously was used, offset
 114                                 will always be ahead of few entries
 115                 */
 116                 $conditions = array();
 117                 if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
 118                         foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
 119                                 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
 120                         }
 121                 }
 122                 $offset = intval( $offset );
 123                 $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
 124
 125                 $row = $dbr->selectRow( 'job', '*',
 126                         array_merge( $conditions, array( "job_id >= $offset" ) ),
 127                         __METHOD__,
 128                         $options
 129                 );
 130
 131                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 132                 // and have lower IDs than jobs already executed, blame concurrency :)
 133                 //
 134                 if ( $row === false ) {
 135                         if ( $offset != 0 ) {
 136                                 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options );
 137                         }
 138
 139                         if ( $row === false ) {
 140                                 wfProfileOut( __METHOD__ );
 141                                 return false;
 142                         }
 143                 }
 144
 145                 // Try to delete it from the master
 146                 $dbw = wfGetDB( DB_MASTER );
 147                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 148                 $affected = $dbw->affectedRows();
 149                 $dbw->commit( __METHOD__ );
 150
 151                 if ( !$affected ) {
 152                         // Failed, someone else beat us to it
 153                         // Try getting a random row
 154                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 155                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 156                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 157                                 // No jobs to get
 158                                 wfProfileOut( __METHOD__ );
 159                                 return false;
 160                         }
 161                         // Get the random row
 162                         $row = $dbw->selectRow( 'job', '*',
 163                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 164                         if ( $row === false ) {
 165                                 // Random job gone before we got the chance to select it
 166                                 // Give up
 167                                 wfProfileOut( __METHOD__ );
 168                                 return false;
 169                         }
 170                         // Delete the random row
 171                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 172                         $affected = $dbw->affectedRows();
 173                         $dbw->commit( __METHOD__ );
 174
 175                         if ( !$affected ) {
 176                                 // Random job gone before we exclusively deleted it
 177                                 // Give up
 178                                 wfProfileOut( __METHOD__ );
 179                                 return false;
 180                         }
 181                 }
 182
 183                 // If execution got to here, there's a row in $row that has been deleted from the database
 184                 // by this thread. Hence the concurrent pop was successful.
 185                 wfIncrStats( 'job-pop' );
 186                 $namespace = $row->job_namespace;
 187                 $dbkey = $row->job_title;
 188                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 189
 190                 if ( is_null( $title ) ) {
 191                         return false;
 192                 }
 193
 194                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 195
 196                 // Remove any duplicates it may have later in the queue
 197                 $job->removeDuplicates();
 198
 199                 wfProfileOut( __METHOD__ );
 200                 return $job;
 201         }
 202
 203         /**
 204          * Create the appropriate object to handle a specific job
 205          *
 206          * @param $command String: Job command
 207          * @param $title Title: Associated title
 208          * @param $params Array: Job parameters
 209          * @param $id Int: Job identifier
 210          * @return Job
 211          */
 212         static function factory( $command, Title $title, $params = false, $id = 0 ) {
 213                 global $wgJobClasses;
 214                 if( isset( $wgJobClasses[$command] ) ) {
 215                         $class = $wgJobClasses[$command];
 216                         return new $class( $title, $params, $id );
 217                 }
 218                 throw new MWException( "Invalid job command `{$command}`" );
 219         }
 220
 221         /**
 222          * @param $params
 223          * @return string
 224          */
 225         static function makeBlob( $params ) {
 226                 if ( $params !== false ) {
 227                         return serialize( $params );
 228                 } else {
 229                         return '';
 230                 }
 231         }
 232
 233         /**
 234          * @param $blob
 235          * @return bool|mixed
 236          */
 237         static function extractBlob( $blob ) {
 238                 if ( (string)$blob !== '' ) {
 239                         return unserialize( $blob );
 240                 } else {
 241                         return false;
 242                 }
 243         }
 244
 245         /**
 246          * Batch-insert a group of jobs into the queue.
 247          * This will be wrapped in a transaction with a forced commit.
 248          *
 249          * This may add duplicate at insert time, but they will be
 250          * removed later on, when the first one is popped.
 251          *
 252          * @param $jobs array of Job objects
 253          */
 254         static function batchInsert( $jobs ) {
 255                 if ( !count( $jobs ) ) {
 256                         return;
 257                 }
 258                 $dbw = wfGetDB( DB_MASTER );
 259                 $rows = array();
 260
 261                 /**
 262                  * @var $job Job
 263                  */
 264                 foreach ( $jobs as $job ) {
 265                         $rows[] = $job->insertFields();
 266                         if ( count( $rows ) >= 50 ) {
 267                                 # Do a small transaction to avoid slave lag
 268                                 $dbw->begin( __METHOD__ );
 269                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 270                                 $dbw->commit( __METHOD__ );
 271                                 $rows = array();
 272                         }
 273                 }
 274                 if ( $rows ) { // last chunk
 275                         $dbw->begin( __METHOD__ );
 276                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 277                         $dbw->commit( __METHOD__ );
 278                 }
 279                 wfIncrStats( 'job-insert', count( $jobs ) );
 280         }
 281
 282         /**
 283          * Insert a group of jobs into the queue.
 284          *
 285          * Same as batchInsert() but does not commit and can thus
 286          * be rolled-back as part of a larger transaction. However,
 287          * large batches of jobs can cause slave lag.
 288          *
 289          * @param $jobs array of Job objects
 290          */
 291         static function safeBatchInsert( $jobs ) {
 292                 if ( !count( $jobs ) ) {
 293                         return;
 294                 }
 295                 $dbw = wfGetDB( DB_MASTER );
 296                 $rows = array();
 297                 foreach ( $jobs as $job ) {
 298                         $rows[] = $job->insertFields();
 299                         if ( count( $rows ) >= 500 ) {
 300                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 301                                 $rows = array();
 302                         }
 303                 }
 304                 if ( $rows ) { // last chunk
 305                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 306                 }
 307                 wfIncrStats( 'job-insert', count( $jobs ) );
 308         }
 309
 310         /*-------------------------------------------------------------------------
 311          * Non-static functions
 312          *------------------------------------------------------------------------*/
 313
 314         /**
 315          * @param $command
 316          * @param $title
 317          * @param $params array
 318          * @param int $id
 319          */
 320         function __construct( $command, $title, $params = false, $id = 0 ) {
 321                 $this->command = $command;
 322                 $this->title = $title;
 323                 $this->params = $params;
 324                 $this->id = $id;
 325
 326                 // A bit of premature generalisation
 327                 // Oh well, the whole class is premature generalisation really
 328                 $this->removeDuplicates = true;
 329         }
 330
 331         /**
 332          * Insert a single job into the queue.
 333          * @return bool true on success
 334          */
 335         function insert() {
 336                 $fields = $this->insertFields();
 337
 338                 $dbw = wfGetDB( DB_MASTER );
 339
 340                 if ( $this->removeDuplicates ) {
 341                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 342                         if ( $dbw->numRows( $res ) ) {
 343                                 return true;
 344                         }
 345                 }
 346                 wfIncrStats( 'job-insert' );
 347                 return $dbw->insert( 'job', $fields, __METHOD__ );
 348         }
 349
 350         /**
 351          * @return array
 352          */
 353         protected function insertFields() {
 354                 $dbw = wfGetDB( DB_MASTER );
 355                 return array(
 356                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 357                         'job_cmd' => $this->command,
 358                         'job_namespace' => $this->title->getNamespace(),
 359                         'job_title' => $this->title->getDBkey(),
 360                         'job_timestamp' => $dbw->timestamp(),
 361                         'job_params' => Job::makeBlob( $this->params )
 362                 );
 363         }
 364
 365         /**
 366          * Remove jobs in the job queue which are duplicates of this job.
 367          * This is deadlock-prone and so starts its own transaction.
 368          */
 369         function removeDuplicates() {
 370                 if ( !$this->removeDuplicates ) {
 371                         return;
 372                 }
 373
 374                 $fields = $this->insertFields();
 375                 unset( $fields['job_id'] );
 376                 $dbw = wfGetDB( DB_MASTER );
 377                 $dbw->begin( __METHOD__ );
 378                 $dbw->delete( 'job', $fields, __METHOD__ );
 379                 $affected = $dbw->affectedRows();
 380                 $dbw->commit( __METHOD__ );
 381                 if ( $affected ) {
 382                         wfIncrStats( 'job-dup-delete', $affected );
 383                 }
 384         }
 385
 386         /**
 387          * @return string
 388          */
 389         function toString() {
 390                 $paramString = '';
 391                 if ( $this->params ) {
 392                         foreach ( $this->params as $key => $value ) {
 393                                 if ( $paramString != '' ) {
 394                                         $paramString .= ' ';
 395                                 }
 396                                 $paramString .= "$key=$value";
 397                         }
 398                 }
 399
 400                 if ( is_object( $this->title ) ) {
 401                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 402                         if ( $paramString !== '' ) {
 403                                 $s .= ' ' . $paramString;
 404                         }
 405                         return $s;
 406                 } else {
 407                         return "{$this->command} $paramString";
 408                 }
 409         }
 410
 411         protected function setLastError( $error ) {
 412                 $this->error = $error;
 413         }
 414
 415         function getLastError() {
 416                 return $this->error;
 417         }
 418 }