3 * HTML cache invalidation of all pages linking to a given title.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Job wrapper for HTMLCacheUpdate. Gets run whenever a related
26 * job gets called from the queue.
28 * This class is designed to work efficiently with small numbers of links, and
29 * to work reasonably well with up to ~10^5 links. Above ~10^6 links, the memory
30 * and time requirements of loading all backlinked IDs in doUpdate() might become
31 * prohibitive. The requirements measured at Wikimedia are approximately:
33 * memory: 48 bytes per row
34 * time: 16us per row for the query plus processing
36 * The reason this query is done is to support partitioning of the job
37 * by backlinked ID. The memory issue could be allieviated by doing this query in
38 * batches, but of course LIMIT with an offset is inefficient on the DB side.
40 * The class is nevertheless a vast improvement on the previous method of using
41 * File::getLinksTo() and Title::touchArray(), which uses about 2KB of memory per
46 class HTMLCacheUpdateJob
extends Job
{
47 /** @var BacklinkCache */
50 protected $rowsPerJob, $rowsPerQuery;
54 * @param $title Title: the title linked to
55 * @param array $params job parameters (table, start and end page_ids)
56 * @param $id Integer: job id
58 function __construct( $title, $params, $id = 0 ) {
59 global $wgUpdateRowsPerJob, $wgUpdateRowsPerQuery;
61 parent
::__construct( 'htmlCacheUpdate', $title, $params, $id );
63 $this->rowsPerJob
= $wgUpdateRowsPerJob;
64 $this->rowsPerQuery
= $wgUpdateRowsPerQuery;
65 $this->blCache
= $title->getBacklinkCache();
68 public function run() {
69 if ( isset( $this->params
['start'] ) && isset( $this->params
['end'] ) ) {
70 # This is hit when a job is actually performed
71 return $this->doPartialUpdate();
73 # This is hit when the jobs have to be inserted
74 return $this->doFullUpdate();
79 * Update all of the backlinks
81 protected function doFullUpdate() {
82 global $wgMaxBacklinksInvalidate;
84 # Get an estimate of the number of rows from the BacklinkCache
85 $numRows = $this->blCache
->getNumLinks( $this->params
['table'] );
86 if ( $wgMaxBacklinksInvalidate !== false && $numRows > $wgMaxBacklinksInvalidate ) {
87 wfDebug( "Skipped HTML cache invalidation of {$this->title->getPrefixedText()}." );
91 if ( $numRows > $this->rowsPerJob
* 2 ) {
92 # Do fast cached partition
93 $this->insertPartitionJobs();
95 # Get the links from the DB
96 $titleArray = $this->blCache
->getLinks( $this->params
['table'] );
97 # Check if the row count estimate was correct
98 if ( $titleArray->count() > $this->rowsPerJob
* 2 ) {
99 # Not correct, do accurate partition
100 wfDebug( __METHOD__
.": row count estimate was incorrect, repartitioning\n" );
101 $this->insertJobsFromTitles( $titleArray );
103 $this->invalidateTitles( $titleArray ); // just do the query
111 * Update some of the backlinks, defined by a page ID range
113 protected function doPartialUpdate() {
114 $titleArray = $this->blCache
->getLinks(
115 $this->params
['table'], $this->params
['start'], $this->params
['end'] );
116 if ( $titleArray->count() <= $this->rowsPerJob
* 2 ) {
117 # This partition is small enough, do the update
118 $this->invalidateTitles( $titleArray );
120 # Partitioning was excessively inaccurate. Divide the job further.
121 # This can occur when a large number of links are added in a short
122 # period of time, say by updating a heavily-used template.
123 $this->insertJobsFromTitles( $titleArray );
129 * Partition the current range given by $this->params['start'] and $this->params['end'],
130 * using a pre-calculated title array which gives the links in that range.
131 * Queue the resulting jobs.
133 * @param $titleArray array
134 * @param $rootJobParams array
137 protected function insertJobsFromTitles( $titleArray, $rootJobParams = array() ) {
138 // Carry over any "root job" information
139 $rootJobParams = $this->getRootJobParams();
140 # We make subpartitions in the sense that the start of the first job
141 # will be the start of the parent partition, and the end of the last
142 # job will be the end of the parent partition.
144 $start = $this->params
['start']; # start of the current job
146 foreach ( $titleArray as $title ) {
147 $id = $title->getArticleID();
148 # $numTitles is now the number of titles in the current job not
149 # including the current ID
150 if ( $numTitles >= $this->rowsPerJob
) {
151 # Add a job up to but not including the current ID
152 $jobs[] = new HTMLCacheUpdateJob( $this->title
,
154 'table' => $this->params
['table'],
157 ) +
$rootJobParams // carry over information for de-duplication
165 $jobs[] = new HTMLCacheUpdateJob( $this->title
,
167 'table' => $this->params
['table'],
169 'end' => $this->params
['end']
170 ) +
$rootJobParams // carry over information for de-duplication
172 wfDebug( __METHOD__
.": repartitioning into " . count( $jobs ) . " jobs\n" );
174 if ( count( $jobs ) < 2 ) {
175 # I don't think this is possible at present, but handling this case
176 # makes the code a bit more robust against future code updates and
177 # avoids a potential infinite loop of repartitioning
178 wfDebug( __METHOD__
.": repartitioning failed!\n" );
179 $this->invalidateTitles( $titleArray );
181 JobQueueGroup
::singleton()->push( $jobs );
186 * @param $rootJobParams array
189 protected function insertPartitionJobs( $rootJobParams = array() ) {
190 // Carry over any "root job" information
191 $rootJobParams = $this->getRootJobParams();
193 $batches = $this->blCache
->partition( $this->params
['table'], $this->rowsPerJob
);
194 if ( !count( $batches ) ) {
195 return; // no jobs to insert
199 foreach ( $batches as $batch ) {
200 list( $start, $end ) = $batch;
201 $jobs[] = new HTMLCacheUpdateJob( $this->title
,
203 'table' => $this->params
['table'],
206 ) +
$rootJobParams // carry over information for de-duplication
210 JobQueueGroup
::singleton()->push( $jobs );
214 * Invalidate an array (or iterator) of Title objects, right now
215 * @param $titleArray array
217 protected function invalidateTitles( $titleArray ) {
218 global $wgUseFileCache, $wgUseSquid;
220 $dbw = wfGetDB( DB_MASTER
);
221 $timestamp = $dbw->timestamp();
223 # Get all IDs in this query into an array
225 foreach ( $titleArray as $title ) {
226 $ids[] = $title->getArticleID();
233 # Don't invalidated pages that were already invalidated
234 $touchedCond = isset( $this->params
['rootJobTimestamp'] )
235 ?
array( "page_touched < " .
236 $dbw->addQuotes( $dbw->timestamp( $this->params
['rootJobTimestamp'] ) ) )
239 # Update page_touched
240 $batches = array_chunk( $ids, $this->rowsPerQuery
);
241 foreach ( $batches as $batch ) {
242 $dbw->update( 'page',
243 array( 'page_touched' => $timestamp ),
244 array( 'page_id' => $batch ) +
$touchedCond,
251 $u = SquidUpdate
::newFromTitles( $titleArray );
256 if ( $wgUseFileCache ) {
257 foreach ( $titleArray as $title ) {
258 HTMLFileCache
::clearFileCache( $title );