includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * Job queue base code.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @defgroup JobQueue JobQueue
  22  */
  23
  24 if ( !defined( 'MEDIAWIKI' ) ) {
  25         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
  26 }
  27
  28 /**
  29  * Class to both describe a background job and handle jobs.
  30  *
  31  * @ingroup JobQueue
  32  */
  33 abstract class Job {
  34
  35         /**
  36          * @var Title
  37          */
  38         var $title;
  39
  40         var $command,
  41                 $params,
  42                 $id,
  43                 $removeDuplicates,
  44                 $error;
  45
  46         /*-------------------------------------------------------------------------
  47          * Abstract functions
  48          *------------------------------------------------------------------------*/
  49
  50         /**
  51          * Run the job
  52          * @return boolean success
  53          */
  54         abstract function run();
  55
  56         /*-------------------------------------------------------------------------
  57          * Static functions
  58          *------------------------------------------------------------------------*/
  59
  60         /**
  61          * Pop a job of a certain type.  This tries less hard than pop() to
  62          * actually find a job; it may be adversely affected by concurrent job
  63          * runners.
  64          *
  65          * @param $type string
  66          *
  67          * @return Job
  68          */
  69         static function pop_type( $type ) {
  70                 wfProfilein( __METHOD__ );
  71
  72                 $dbw = wfGetDB( DB_MASTER );
  73
  74                 $dbw->begin( __METHOD__ );
  75
  76                 $row = $dbw->selectRow(
  77                         'job',
  78                         '*',
  79                         array( 'job_cmd' => $type ),
  80                         __METHOD__,
  81                         array( 'LIMIT' => 1, 'FOR UPDATE' )
  82                 );
  83
  84                 if ( $row === false ) {
  85                         $dbw->commit( __METHOD__ );
  86                         wfProfileOut( __METHOD__ );
  87                         return false;
  88                 }
  89
  90                 /* Ensure we "own" this row */
  91                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  92                 $affected = $dbw->affectedRows();
  93                 $dbw->commit( __METHOD__ );
  94
  95                 if ( $affected == 0 ) {
  96                         wfProfileOut( __METHOD__ );
  97                         return false;
  98                 }
  99
 100                 wfIncrStats( 'job-pop' );
 101                 $namespace = $row->job_namespace;
 102                 $dbkey = $row->job_title;
 103                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 104                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
 105                         $row->job_id );
 106
 107                 $job->removeDuplicates();
 108
 109                 wfProfileOut( __METHOD__ );
 110                 return $job;
 111         }
 112
 113         /**
 114          * Pop a job off the front of the queue
 115          *
 116          * @param $offset Integer: Number of jobs to skip
 117          * @return Job or false if there's no jobs
 118          */
 119         static function pop( $offset = 0 ) {
 120                 global $wgJobTypesExcludedFromDefaultQueue;
 121                 wfProfileIn( __METHOD__ );
 122
 123                 $dbr = wfGetDB( DB_SLAVE );
 124
 125                 /* Get a job from the slave, start with an offset,
 126                         scan full set afterwards, avoid hitting purged rows
 127
 128                         NB: If random fetch previously was used, offset
 129                                 will always be ahead of few entries
 130                 */
 131                 $conditions = array();
 132                 if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
 133                         foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
 134                                 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
 135                         }
 136                 }
 137                 $offset = intval( $offset );
 138                 $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
 139
 140                 $row = $dbr->selectRow( 'job', '*',
 141                         array_merge( $conditions, array( "job_id >= $offset" ) ),
 142                         __METHOD__,
 143                         $options
 144                 );
 145
 146                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 147                 // and have lower IDs than jobs already executed, blame concurrency :)
 148                 //
 149                 if ( $row === false ) {
 150                         if ( $offset != 0 ) {
 151                                 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options );
 152                         }
 153
 154                         if ( $row === false ) {
 155                                 wfProfileOut( __METHOD__ );
 156                                 return false;
 157                         }
 158                 }
 159
 160                 // Try to delete it from the master
 161                 $dbw = wfGetDB( DB_MASTER );
 162                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 163                 $affected = $dbw->affectedRows();
 164                 $dbw->commit( __METHOD__ );
 165
 166                 if ( !$affected ) {
 167                         // Failed, someone else beat us to it
 168                         // Try getting a random row
 169                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 170                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 171                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 172                                 // No jobs to get
 173                                 wfProfileOut( __METHOD__ );
 174                                 return false;
 175                         }
 176                         // Get the random row
 177                         $row = $dbw->selectRow( 'job', '*',
 178                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 179                         if ( $row === false ) {
 180                                 // Random job gone before we got the chance to select it
 181                                 // Give up
 182                                 wfProfileOut( __METHOD__ );
 183                                 return false;
 184                         }
 185                         // Delete the random row
 186                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 187                         $affected = $dbw->affectedRows();
 188                         $dbw->commit( __METHOD__ );
 189
 190                         if ( !$affected ) {
 191                                 // Random job gone before we exclusively deleted it
 192                                 // Give up
 193                                 wfProfileOut( __METHOD__ );
 194                                 return false;
 195                         }
 196                 }
 197
 198                 // If execution got to here, there's a row in $row that has been deleted from the database
 199                 // by this thread. Hence the concurrent pop was successful.
 200                 wfIncrStats( 'job-pop' );
 201                 $namespace = $row->job_namespace;
 202                 $dbkey = $row->job_title;
 203                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 204
 205                 if ( is_null( $title ) ) {
 206                         return false;
 207                 }
 208
 209                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 210
 211                 // Remove any duplicates it may have later in the queue
 212                 $job->removeDuplicates();
 213
 214                 wfProfileOut( __METHOD__ );
 215                 return $job;
 216         }
 217
 218         /**
 219          * Create the appropriate object to handle a specific job
 220          *
 221          * @param $command String: Job command
 222          * @param $title Title: Associated title
 223          * @param $params Array: Job parameters
 224          * @param $id Int: Job identifier
 225          * @return Job
 226          */
 227         static function factory( $command, Title $title, $params = false, $id = 0 ) {
 228                 global $wgJobClasses;
 229                 if( isset( $wgJobClasses[$command] ) ) {
 230                         $class = $wgJobClasses[$command];
 231                         return new $class( $title, $params, $id );
 232                 }
 233                 throw new MWException( "Invalid job command `{$command}`" );
 234         }
 235
 236         /**
 237          * @param $params
 238          * @return string
 239          */
 240         static function makeBlob( $params ) {
 241                 if ( $params !== false ) {
 242                         return serialize( $params );
 243                 } else {
 244                         return '';
 245                 }
 246         }
 247
 248         /**
 249          * @param $blob
 250          * @return bool|mixed
 251          */
 252         static function extractBlob( $blob ) {
 253                 if ( (string)$blob !== '' ) {
 254                         return unserialize( $blob );
 255                 } else {
 256                         return false;
 257                 }
 258         }
 259
 260         /**
 261          * Batch-insert a group of jobs into the queue.
 262          * This will be wrapped in a transaction with a forced commit.
 263          *
 264          * This may add duplicate at insert time, but they will be
 265          * removed later on, when the first one is popped.
 266          *
 267          * @param $jobs array of Job objects
 268          */
 269         static function batchInsert( $jobs ) {
 270                 if ( !count( $jobs ) ) {
 271                         return;
 272                 }
 273                 $dbw = wfGetDB( DB_MASTER );
 274                 $rows = array();
 275
 276                 /**
 277                  * @var $job Job
 278                  */
 279                 foreach ( $jobs as $job ) {
 280                         $rows[] = $job->insertFields();
 281                         if ( count( $rows ) >= 50 ) {
 282                                 # Do a small transaction to avoid slave lag
 283                                 $dbw->begin( __METHOD__ );
 284                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 285                                 $dbw->commit( __METHOD__ );
 286                                 $rows = array();
 287                         }
 288                 }
 289                 if ( $rows ) { // last chunk
 290                         $dbw->begin( __METHOD__ );
 291                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 292                         $dbw->commit( __METHOD__ );
 293                 }
 294                 wfIncrStats( 'job-insert', count( $jobs ) );
 295         }
 296
 297         /**
 298          * Insert a group of jobs into the queue.
 299          *
 300          * Same as batchInsert() but does not commit and can thus
 301          * be rolled-back as part of a larger transaction. However,
 302          * large batches of jobs can cause slave lag.
 303          *
 304          * @param $jobs array of Job objects
 305          */
 306         static function safeBatchInsert( $jobs ) {
 307                 if ( !count( $jobs ) ) {
 308                         return;
 309                 }
 310                 $dbw = wfGetDB( DB_MASTER );
 311                 $rows = array();
 312                 foreach ( $jobs as $job ) {
 313                         $rows[] = $job->insertFields();
 314                         if ( count( $rows ) >= 500 ) {
 315                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 316                                 $rows = array();
 317                         }
 318                 }
 319                 if ( $rows ) { // last chunk
 320                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 321                 }
 322                 wfIncrStats( 'job-insert', count( $jobs ) );
 323         }
 324
 325         /*-------------------------------------------------------------------------
 326          * Non-static functions
 327          *------------------------------------------------------------------------*/
 328
 329         /**
 330          * @param $command
 331          * @param $title
 332          * @param $params array
 333          * @param int $id
 334          */
 335         function __construct( $command, $title, $params = false, $id = 0 ) {
 336                 $this->command = $command;
 337                 $this->title = $title;
 338                 $this->params = $params;
 339                 $this->id = $id;
 340
 341                 // A bit of premature generalisation
 342                 // Oh well, the whole class is premature generalisation really
 343                 $this->removeDuplicates = true;
 344         }
 345
 346         /**
 347          * Insert a single job into the queue.
 348          * @return bool true on success
 349          */
 350         function insert() {
 351                 $fields = $this->insertFields();
 352
 353                 $dbw = wfGetDB( DB_MASTER );
 354
 355                 if ( $this->removeDuplicates ) {
 356                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 357                         if ( $dbw->numRows( $res ) ) {
 358                                 return true;
 359                         }
 360                 }
 361                 wfIncrStats( 'job-insert' );
 362                 return $dbw->insert( 'job', $fields, __METHOD__ );
 363         }
 364
 365         /**
 366          * @return array
 367          */
 368         protected function insertFields() {
 369                 $dbw = wfGetDB( DB_MASTER );
 370                 return array(
 371                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 372                         'job_cmd' => $this->command,
 373                         'job_namespace' => $this->title->getNamespace(),
 374                         'job_title' => $this->title->getDBkey(),
 375                         'job_timestamp' => $dbw->timestamp(),
 376                         'job_params' => Job::makeBlob( $this->params )
 377                 );
 378         }
 379
 380         /**
 381          * Remove jobs in the job queue which are duplicates of this job.
 382          * This is deadlock-prone and so starts its own transaction.
 383          */
 384         function removeDuplicates() {
 385                 if ( !$this->removeDuplicates ) {
 386                         return;
 387                 }
 388
 389                 $fields = $this->insertFields();
 390                 unset( $fields['job_id'] );
 391                 $dbw = wfGetDB( DB_MASTER );
 392                 $dbw->begin( __METHOD__ );
 393                 $dbw->delete( 'job', $fields, __METHOD__ );
 394                 $affected = $dbw->affectedRows();
 395                 $dbw->commit( __METHOD__ );
 396                 if ( $affected ) {
 397                         wfIncrStats( 'job-dup-delete', $affected );
 398                 }
 399         }
 400
 401         /**
 402          * @return string
 403          */
 404         function toString() {
 405                 $paramString = '';
 406                 if ( $this->params ) {
 407                         foreach ( $this->params as $key => $value ) {
 408                                 if ( $paramString != '' ) {
 409                                         $paramString .= ' ';
 410                                 }
 411                                 $paramString .= "$key=$value";
 412                         }
 413                 }
 414
 415                 if ( is_object( $this->title ) ) {
 416                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 417                         if ( $paramString !== '' ) {
 418                                 $s .= ' ' . $paramString;
 419                         }
 420                         return $s;
 421                 } else {
 422                         return "{$this->command} $paramString";
 423                 }
 424         }
 425
 426         protected function setLastError( $error ) {
 427                 $this->error = $error;
 428         }
 429
 430         function getLastError() {
 431                 return $this->error;
 432         }
 433 }