includes/job/jobs/HTMLCacheUpdateJob.php

   1 <?php
   2 /**
   3  * HTML cache invalidation of all pages linking to a given title.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Cache
  22  */
  23
  24 /**
  25  * Job wrapper for HTMLCacheUpdate. Gets run whenever a related
  26  * job gets called from the queue.
  27  *
  28  * This class is designed to work efficiently with small numbers of links, and
  29  * to work reasonably well with up to ~10^5 links. Above ~10^6 links, the memory
  30  * and time requirements of loading all backlinked IDs in doUpdate() might become
  31  * prohibitive. The requirements measured at Wikimedia are approximately:
  32  *
  33  *   memory: 48 bytes per row
  34  *   time: 16us per row for the query plus processing
  35  *
  36  * The reason this query is done is to support partitioning of the job
  37  * by backlinked ID. The memory issue could be allieviated by doing this query in
  38  * batches, but of course LIMIT with an offset is inefficient on the DB side.
  39  *
  40  * The class is nevertheless a vast improvement on the previous method of using
  41  * File::getLinksTo() and Title::touchArray(), which uses about 2KB of memory per
  42  * link.
  43  *
  44  * @ingroup JobQueue
  45  */
  46 class HTMLCacheUpdateJob extends Job {
  47         /** @var BacklinkCache */
  48         protected $blCache;
  49
  50         protected $rowsPerJob, $rowsPerQuery;
  51
  52         /**
  53          * Construct a job
  54          * @param $title Title: the title linked to
  55          * @param array $params job parameters (table, start and end page_ids)
  56          * @param $id Integer: job id
  57          */
  58         function __construct( $title, $params, $id = 0 ) {
  59                 global $wgUpdateRowsPerJob, $wgUpdateRowsPerQuery;
  60
  61                 parent::__construct( 'htmlCacheUpdate', $title, $params, $id );
  62
  63                 $this->rowsPerJob = $wgUpdateRowsPerJob;
  64                 $this->rowsPerQuery = $wgUpdateRowsPerQuery;
  65                 $this->blCache = $title->getBacklinkCache();
  66         }
  67
  68         public function run() {
  69                 if ( isset( $this->params['start'] ) && isset( $this->params['end'] ) ) {
  70                         # This is hit when a job is actually performed
  71                         return $this->doPartialUpdate();
  72                 } else {
  73                         # This is hit when the jobs have to be inserted
  74                         return $this->doFullUpdate();
  75                 }
  76         }
  77
  78         /**
  79          * Update all of the backlinks
  80          */
  81         protected function doFullUpdate() {
  82                 global $wgMaxBacklinksInvalidate;
  83
  84                 # Get an estimate of the number of rows from the BacklinkCache
  85                 $numRows = $this->blCache->getNumLinks( $this->params['table'] );
  86                 if ( $wgMaxBacklinksInvalidate !== false && $numRows > $wgMaxBacklinksInvalidate ) {
  87                         wfDebug( "Skipped HTML cache invalidation of {$this->title->getPrefixedText()}." );
  88                         return true;
  89                 }
  90
  91                 if ( $numRows > $this->rowsPerJob * 2 ) {
  92                         # Do fast cached partition
  93                         $this->insertPartitionJobs();
  94                 } else {
  95                         # Get the links from the DB
  96                         $titleArray = $this->blCache->getLinks( $this->params['table'] );
  97                         # Check if the row count estimate was correct
  98                         if ( $titleArray->count() > $this->rowsPerJob * 2 ) {
  99                                 # Not correct, do accurate partition
 100                                 wfDebug( __METHOD__ . ": row count estimate was incorrect, repartitioning\n" );
 101                                 $this->insertJobsFromTitles( $titleArray );
 102                         } else {
 103                                 $this->invalidateTitles( $titleArray ); // just do the query
 104                         }
 105                 }
 106
 107                 return true;
 108         }
 109
 110         /**
 111          * Update some of the backlinks, defined by a page ID range
 112          */
 113         protected function doPartialUpdate() {
 114                 $titleArray = $this->blCache->getLinks(
 115                         $this->params['table'], $this->params['start'], $this->params['end'] );
 116                 if ( $titleArray->count() <= $this->rowsPerJob * 2 ) {
 117                         # This partition is small enough, do the update
 118                         $this->invalidateTitles( $titleArray );
 119                 } else {
 120                         # Partitioning was excessively inaccurate. Divide the job further.
 121                         # This can occur when a large number of links are added in a short
 122                         # period of time, say by updating a heavily-used template.
 123                         $this->insertJobsFromTitles( $titleArray );
 124                 }
 125                 return true;
 126         }
 127
 128         /**
 129          * Partition the current range given by $this->params['start'] and $this->params['end'],
 130          * using a pre-calculated title array which gives the links in that range.
 131          * Queue the resulting jobs.
 132          *
 133          * @param $titleArray array
 134          * @param $rootJobParams array
 135          * @return void
 136          */
 137         protected function insertJobsFromTitles( $titleArray, $rootJobParams = array() ) {
 138                 // Carry over any "root job" information
 139                 $rootJobParams = $this->getRootJobParams();
 140                 # We make subpartitions in the sense that the start of the first job
 141                 # will be the start of the parent partition, and the end of the last
 142                 # job will be the end of the parent partition.
 143                 $jobs = array();
 144                 $start = $this->params['start']; # start of the current job
 145                 $numTitles = 0;
 146                 foreach ( $titleArray as $title ) {
 147                         $id = $title->getArticleID();
 148                         # $numTitles is now the number of titles in the current job not
 149                         # including the current ID
 150                         if ( $numTitles >= $this->rowsPerJob ) {
 151                                 # Add a job up to but not including the current ID
 152                                 $jobs[] = new HTMLCacheUpdateJob( $this->title,
 153                                         array(
 154                                                 'table' => $this->params['table'],
 155                                                 'start' => $start,
 156                                                 'end'   => $id - 1
 157                                         ) + $rootJobParams // carry over information for de-duplication
 158                                 );
 159                                 $start = $id;
 160                                 $numTitles = 0;
 161                         }
 162                         $numTitles++;
 163                 }
 164                 # Last job
 165                 $jobs[] = new HTMLCacheUpdateJob( $this->title,
 166                         array(
 167                                 'table' => $this->params['table'],
 168                                 'start' => $start,
 169                                 'end'   => $this->params['end']
 170                         ) + $rootJobParams // carry over information for de-duplication
 171                 );
 172                 wfDebug( __METHOD__ . ": repartitioning into " . count( $jobs ) . " jobs\n" );
 173
 174                 if ( count( $jobs ) < 2 ) {
 175                         # I don't think this is possible at present, but handling this case
 176                         # makes the code a bit more robust against future code updates and
 177                         # avoids a potential infinite loop of repartitioning
 178                         wfDebug( __METHOD__ . ": repartitioning failed!\n" );
 179                         $this->invalidateTitles( $titleArray );
 180                 } else {
 181                         JobQueueGroup::singleton()->push( $jobs );
 182                 }
 183         }
 184
 185         /**
 186          * @param $rootJobParams array
 187          * @return void
 188          */
 189         protected function insertPartitionJobs( $rootJobParams = array() ) {
 190                 // Carry over any "root job" information
 191                 $rootJobParams = $this->getRootJobParams();
 192
 193                 $batches = $this->blCache->partition( $this->params['table'], $this->rowsPerJob );
 194                 if ( !count( $batches ) ) {
 195                         return; // no jobs to insert
 196                 }
 197
 198                 $jobs = array();
 199                 foreach ( $batches as $batch ) {
 200                         list( $start, $end ) = $batch;
 201                         $jobs[] = new HTMLCacheUpdateJob( $this->title,
 202                                 array(
 203                                         'table' => $this->params['table'],
 204                                         'start' => $start,
 205                                         'end'   => $end,
 206                                 ) + $rootJobParams // carry over information for de-duplication
 207                         );
 208                 }
 209
 210                 JobQueueGroup::singleton()->push( $jobs );
 211         }
 212
 213         /**
 214          * Invalidate an array (or iterator) of Title objects, right now
 215          * @param $titleArray array
 216          */
 217         protected function invalidateTitles( $titleArray ) {
 218                 global $wgUseFileCache, $wgUseSquid;
 219
 220                 $dbw = wfGetDB( DB_MASTER );
 221                 $timestamp = $dbw->timestamp();
 222
 223                 # Get all IDs in this query into an array
 224                 $ids = array();
 225                 foreach ( $titleArray as $title ) {
 226                         $ids[] = $title->getArticleID();
 227                 }
 228
 229                 if ( !$ids ) {
 230                         return;
 231                 }
 232
 233                 # Don't invalidated pages that were already invalidated
 234                 $touchedCond = isset( $this->params['rootJobTimestamp'] )
 235                         ? array( "page_touched < " .
 236                                 $dbw->addQuotes( $dbw->timestamp( $this->params['rootJobTimestamp'] ) ) )
 237                         : array();
 238
 239                 # Update page_touched
 240                 $batches = array_chunk( $ids, $this->rowsPerQuery );
 241                 foreach ( $batches as $batch ) {
 242                         $dbw->update( 'page',
 243                                 array( 'page_touched' => $timestamp ),
 244                                 array( 'page_id' => $batch ) + $touchedCond,
 245                                 __METHOD__
 246                         );
 247                 }
 248
 249                 # Update squid
 250                 if ( $wgUseSquid ) {
 251                         $u = SquidUpdate::newFromTitles( $titleArray );
 252                         $u->doUpdate();
 253                 }
 254
 255                 # Update file cache
 256                 if  ( $wgUseFileCache ) {
 257                         foreach ( $titleArray as $title ) {
 258                                 HTMLFileCache::clearFileCache( $title );
 259                         }
 260                 }
 261         }
 262 }