includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * Job queue base code
   4  *
   5  * @file
   6  * @defgroup JobQueue JobQueue
   7  */
   8
   9 if ( !defined( 'MEDIAWIKI' ) ) {
  10         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
  11 }
  12
  13 /**
  14  * Class to both describe a background job and handle jobs.
  15  *
  16  * @ingroup JobQueue
  17  */
  18 abstract class Job {
  19         var $command,
  20                 $title,
  21                 $params,
  22                 $id,
  23                 $removeDuplicates,
  24                 $error;
  25
  26         /*-------------------------------------------------------------------------
  27          * Abstract functions
  28          *------------------------------------------------------------------------*/
  29
  30         /**
  31          * Run the job
  32          * @return boolean success
  33          */
  34         abstract function run();
  35
  36         /*-------------------------------------------------------------------------
  37          * Static functions
  38          *------------------------------------------------------------------------*/
  39
  40         /**
  41          * Pop a job of a certain type.  This tries less hard than pop() to
  42          * actually find a job; it may be adversely affected by concurrent job
  43          * runners.
  44          */
  45         static function pop_type( $type ) {
  46                 wfProfilein( __METHOD__ );
  47
  48                 $dbw = wfGetDB( DB_MASTER );
  49
  50                 $row = $dbw->selectRow(
  51                         'job',
  52                         '*',
  53                         array( 'job_cmd' => $type ),
  54                         __METHOD__,
  55                         array( 'LIMIT' => 1 )
  56                 );
  57
  58                 if ( $row === false ) {
  59                         wfProfileOut( __METHOD__ );
  60                         return false;
  61                 }
  62
  63                 /* Ensure we "own" this row */
  64                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  65                 $affected = $dbw->affectedRows();
  66
  67                 if ( $affected == 0 ) {
  68                         wfProfileOut( __METHOD__ );
  69                         return false;
  70                 }
  71
  72                 $namespace = $row->job_namespace;
  73                 $dbkey = $row->job_title;
  74                 $title = Title::makeTitleSafe( $namespace, $dbkey );
  75                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
  76                         $row->job_id );
  77
  78                 $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
  79                 $dbw->commit();
  80
  81                 wfProfileOut( __METHOD__ );
  82                 return $job;
  83         }
  84
  85         /**
  86          * Pop a job off the front of the queue
  87          *
  88          * @param $offset Integer: Number of jobs to skip
  89          * @return Job or false if there's no jobs
  90          */
  91         static function pop( $offset = 0 ) {
  92                 wfProfileIn( __METHOD__ );
  93
  94                 $dbr = wfGetDB( DB_SLAVE );
  95
  96                 /* Get a job from the slave, start with an offset,
  97                         scan full set afterwards, avoid hitting purged rows
  98
  99                         NB: If random fetch previously was used, offset
 100                                 will always be ahead of few entries
 101                 */
 102
 103                 $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__,
 104                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 105
 106                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 107                 // and have lower IDs than jobs already executed, blame concurrency :)
 108                 //
 109                 if ( $row === false ) {
 110                         if ( $offset != 0 ) {
 111                                 $row = $dbr->selectRow( 'job', '*', '', __METHOD__,
 112                                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 113                         }
 114
 115                         if ( $row === false ) {
 116                                 wfProfileOut( __METHOD__ );
 117                                 return false;
 118                         }
 119                 }
 120                 $offset = $row->job_id;
 121
 122                 // Try to delete it from the master
 123                 $dbw = wfGetDB( DB_MASTER );
 124                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 125                 $affected = $dbw->affectedRows();
 126                 $dbw->commit();
 127
 128                 if ( !$affected ) {
 129                         // Failed, someone else beat us to it
 130                         // Try getting a random row
 131                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 132                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 133                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 134                                 // No jobs to get
 135                                 wfProfileOut( __METHOD__ );
 136                                 return false;
 137                         }
 138                         // Get the random row
 139                         $row = $dbw->selectRow( 'job', '*',
 140                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 141                         if ( $row === false ) {
 142                                 // Random job gone before we got the chance to select it
 143                                 // Give up
 144                                 wfProfileOut( __METHOD__ );
 145                                 return false;
 146                         }
 147                         // Delete the random row
 148                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 149                         $affected = $dbw->affectedRows();
 150                         $dbw->commit();
 151
 152                         if ( !$affected ) {
 153                                 // Random job gone before we exclusively deleted it
 154                                 // Give up
 155                                 wfProfileOut( __METHOD__ );
 156                                 return false;
 157                         }
 158                 }
 159
 160                 // If execution got to here, there's a row in $row that has been deleted from the database
 161                 // by this thread. Hence the concurrent pop was successful.
 162                 $namespace = $row->job_namespace;
 163                 $dbkey = $row->job_title;
 164                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 165                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 166
 167                 // Remove any duplicates it may have later in the queue
 168                 // Deadlock prone section
 169                 $dbw->begin();
 170                 $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
 171                 $dbw->commit();
 172
 173                 wfProfileOut( __METHOD__ );
 174                 return $job;
 175         }
 176
 177         /**
 178          * Create the appropriate object to handle a specific job
 179          *
 180          * @param $command String: Job command
 181          * @param $title Title: Associated title
 182          * @param $params Array: Job parameters
 183          * @param $id Int: Job identifier
 184          * @return Job
 185          */
 186         static function factory( $command, $title, $params = false, $id = 0 ) {
 187                 global $wgJobClasses;
 188                 if( isset( $wgJobClasses[$command] ) ) {
 189                         $class = $wgJobClasses[$command];
 190                         return new $class( $title, $params, $id );
 191                 }
 192                 throw new MWException( "Invalid job command `{$command}`" );
 193         }
 194
 195         static function makeBlob( $params ) {
 196                 if ( $params !== false ) {
 197                         return serialize( $params );
 198                 } else {
 199                         return '';
 200                 }
 201         }
 202
 203         static function extractBlob( $blob ) {
 204                 if ( (string)$blob !== '' ) {
 205                         return unserialize( $blob );
 206                 } else {
 207                         return false;
 208                 }
 209         }
 210
 211         /**
 212          * Batch-insert a group of jobs into the queue.
 213          * This will be wrapped in a transaction with a forced commit.
 214          *
 215          * This may add duplicate at insert time, but they will be
 216          * removed later on, when the first one is popped.
 217          *
 218          * @param $jobs array of Job objects
 219          */
 220         static function batchInsert( $jobs ) {
 221                 if( !count( $jobs ) ) {
 222                         return;
 223                 }
 224                 $dbw = wfGetDB( DB_MASTER );
 225                 $rows = array();
 226                 foreach( $jobs as $job ) {
 227                         $rows[] = $job->insertFields();
 228                         if ( count( $rows ) >= 50 ) {
 229                                 # Do a small transaction to avoid slave lag
 230                                 $dbw->begin();
 231                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 232                                 $dbw->commit();
 233                                 $rows = array();
 234                         }
 235                 }
 236                 if ( $rows ) {
 237                         $dbw->begin();
 238                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 239                         $dbw->commit();
 240                 }
 241         }
 242
 243         /*-------------------------------------------------------------------------
 244          * Non-static functions
 245          *------------------------------------------------------------------------*/
 246
 247         function __construct( $command, $title, $params = false, $id = 0 ) {
 248                 $this->command = $command;
 249                 $this->title = $title;
 250                 $this->params = $params;
 251                 $this->id = $id;
 252
 253                 // A bit of premature generalisation
 254                 // Oh well, the whole class is premature generalisation really
 255                 $this->removeDuplicates = true;
 256         }
 257
 258         /**
 259          * Insert a single job into the queue.
 260          * @return bool true on success
 261          */
 262         function insert() {
 263                 $fields = $this->insertFields();
 264
 265                 $dbw = wfGetDB( DB_MASTER );
 266
 267                 if ( $this->removeDuplicates ) {
 268                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 269                         if ( $dbw->numRows( $res ) ) {
 270                                 return;
 271                         }
 272                 }
 273                 return $dbw->insert( 'job', $fields, __METHOD__ );
 274         }
 275
 276         protected function insertFields() {
 277                 $dbw = wfGetDB( DB_MASTER );
 278                 return array(
 279                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 280                         'job_cmd' => $this->command,
 281                         'job_namespace' => $this->title->getNamespace(),
 282                         'job_title' => $this->title->getDBkey(),
 283                         'job_params' => Job::makeBlob( $this->params )
 284                 );
 285         }
 286
 287         function toString() {
 288                 $paramString = '';
 289                 if ( $this->params ) {
 290                         foreach ( $this->params as $key => $value ) {
 291                                 if ( $paramString != '' ) {
 292                                         $paramString .= ' ';
 293                                 }
 294                                 $paramString .= "$key=$value";
 295                         }
 296                 }
 297
 298                 if ( is_object( $this->title ) ) {
 299                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 300                         if ( $paramString !== '' ) {
 301                                 $s .= ' ' . $paramString;
 302                         }
 303                         return $s;
 304                 } else {
 305                         return "{$this->command} $paramString";
 306                 }
 307         }
 308
 309         protected function setLastError( $error ) {
 310                 $this->error = $error;
 311         }
 312
 313         function getLastError() {
 314                 return $this->error;
 315         }
 316 }