3 * Job to update link tables for pages
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Job to update link tables for pages
27 * This job comes in a few variants:
28 * - a) Recursive jobs to update links for backlink pages for a given title.
29 * These jobs have (recursive:true,table:<table>) set.
30 * - b) Jobs to update links for a set of pages (the job title is ignored).
31 * These jobs have (pages:(<page ID>:(<namespace>,<title>),...) set.
32 * - c) Jobs to update links for a single page (the job title)
33 * These jobs need no extra fields set.
37 class RefreshLinksJob
extends Job
{
38 const PARSE_THRESHOLD_SEC
= 1.0;
40 const CLOCK_FUDGE
= 10;
42 function __construct( Title
$title, array $params ) {
43 parent
::__construct( 'refreshLinks', $title, $params );
44 // Base backlink update jobs and per-title update jobs can be de-duplicated.
45 // If template A changes twice before any jobs run, a clean queue will have:
47 // The second job is ignored by the queue on insertion.
48 // Suppose, many pages use template A, and that template itself uses template B.
49 // An edit to both will first create two base jobs. A clean FIFO queue will have:
51 // When these jobs run, the queue will have per-title and remnant partition jobs:
52 // (titleX,titleY,titleZ,...,A remnant,titleM,titleN,titleO,...,B remnant)
53 // Some these jobs will be the same, and will automatically be ignored by
54 // the queue upon insertion. Some title jobs will run before the duplicate is
55 // inserted, so the work will still be done twice in those cases. More titles
56 // can be de-duplicated as the remnant jobs continue to be broken down. This
57 // works best when $wgUpdateRowsPerJob, and either the pages have few backlinks
58 // and/or the backlink sets for pages A and B are almost identical.
59 $this->removeDuplicates
= !isset( $params['range'] )
60 && ( !isset( $params['pages'] ) ||
count( $params['pages'] ) == 1 );
65 * @param array $params
66 * @return RefreshLinksJob
68 public static function newPrioritized( Title
$title, array $params ) {
69 $job = new self( $title, $params );
70 $job->command
= 'refreshLinksPrioritized';
77 * @param array $params
78 * @return RefreshLinksJob
80 public static function newDynamic( Title
$title, array $params ) {
81 $job = new self( $title, $params );
82 $job->command
= 'refreshLinksDynamic';
88 global $wgUpdateRowsPerJob;
90 // Job to update all (or a range of) backlink pages for a page
91 if ( !empty( $this->params
['recursive'] ) ) {
92 // Carry over information for de-duplication
93 $extraParams = $this->getRootJobParams();
94 // Avoid slave lag when fetching templates.
95 // When the outermost job is run, we know that the caller that enqueued it must have
96 // committed the relevant changes to the DB by now. At that point, record the master
97 // position and pass it along as the job recursively breaks into smaller range jobs.
98 // Hopefully, when leaf jobs are popped, the slaves will have reached that position.
99 if ( isset( $this->params
['masterPos'] ) ) {
100 $extraParams['masterPos'] = $this->params
['masterPos'];
101 } elseif ( wfGetLB()->getServerCount() > 1 ) {
102 $extraParams['masterPos'] = wfGetLB()->getMasterPos();
104 $extraParams['masterPos'] = false;
106 $extraParams['triggeredRecursive'] = true;
107 // Convert this into no more than $wgUpdateRowsPerJob RefreshLinks per-title
108 // jobs and possibly a recursive RefreshLinks job for the rest of the backlinks
109 $jobs = BacklinkJobUtils
::partitionBacklinkJob(
113 array( 'params' => $extraParams )
115 JobQueueGroup
::singleton()->push( $jobs );
116 // Job to update link tables for a set of titles
117 } elseif ( isset( $this->params
['pages'] ) ) {
118 $this->waitForMasterPosition();
119 foreach ( $this->params
['pages'] as $pageId => $nsAndKey ) {
120 list( $ns, $dbKey ) = $nsAndKey;
121 $this->runForTitle( Title
::makeTitleSafe( $ns, $dbKey ) );
123 // Job to update link tables for a given title
125 $this->waitForMasterPosition();
126 $this->runForTitle( $this->title
);
132 protected function waitForMasterPosition() {
133 if ( !empty( $this->params
['masterPos'] ) && wfGetLB()->getServerCount() > 1 ) {
134 // Wait for the current/next slave DB handle to catch up to the master.
135 // This way, we get the correct page_latest for templates or files that just
136 // changed milliseconds ago, having triggered this job to begin with.
137 wfGetLB()->waitFor( $this->params
['masterPos'] );
142 * @param Title $title
145 protected function runForTitle( Title
$title ) {
146 $page = WikiPage
::factory( $title );
147 if ( !empty( $this->params
['triggeringRevisionId'] ) ) {
148 // Fetch the specified revision; lockAndGetLatest() below detects if the page
149 // was edited since and aborts in order to avoid corrupting the link tables
150 $revision = Revision
::newFromId(
151 $this->params
['triggeringRevisionId'],
152 Revision
::READ_LATEST
155 // Fetch current revision; READ_LATEST reduces lockAndGetLatest() check failures
156 $revision = Revision
::newFromTitle( $title, false, Revision
::READ_LATEST
);
160 $this->setLastError( "Revision not found for {$title->getPrefixedDBkey()}" );
161 return false; // just deleted?
164 $content = $revision->getContent( Revision
::RAW
);
166 // If there is no content, pretend the content is empty
167 $content = $revision->getContentHandler()->makeEmptyContent();
170 $parserOutput = false;
171 $parserOptions = $page->makeParserOptions( 'canonical' );
172 // If page_touched changed after this root job, then it is likely that
173 // any views of the pages already resulted in re-parses which are now in
174 // cache. The cache can be reused to avoid expensive parsing in some cases.
175 if ( isset( $this->params
['rootJobTimestamp'] ) ) {
176 $opportunistic = !empty( $this->params
['isOpportunistic'] );
178 $skewedTimestamp = $this->params
['rootJobTimestamp'];
179 if ( $opportunistic ) {
180 // Neither clock skew nor DB snapshot/slave lag matter much for such
181 // updates; focus on reusing the (often recently updated) cache
183 // For transclusion updates, the template changes must be reflected
184 $skewedTimestamp = wfTimestamp( TS_MW
,
185 wfTimestamp( TS_UNIX
, $skewedTimestamp ) + self
::CLOCK_FUDGE
189 if ( $page->getLinksTimestamp() > $skewedTimestamp ) {
190 // Something already updated the backlinks since this job was made
194 if ( $page->getTouched() >= $skewedTimestamp ||
$opportunistic ) {
195 // Something bumped page_touched since this job was made
196 // or the cache is otherwise suspected to be up-to-date
197 $parserOutput = ParserCache
::singleton()->getDirty( $page, $parserOptions );
198 if ( $parserOutput && $parserOutput->getCacheTime() < $skewedTimestamp ) {
199 $parserOutput = false; // too stale
204 // Fetch the current revision and parse it if necessary...
205 if ( $parserOutput == false ) {
206 $start = microtime( true );
207 // Revision ID must be passed to the parser output to get revision variables correct
208 $parserOutput = $content->getParserOutput(
209 $title, $revision->getId(), $parserOptions, false );
210 $elapsed = microtime( true ) - $start;
211 // If it took a long time to render, then save this back to the cache to avoid
212 // wasted CPU by other apaches or job runners. We don't want to always save to
213 // cache as this can cause high cache I/O and LRU churn when a template changes.
214 if ( $elapsed >= self
::PARSE_THRESHOLD_SEC
215 && $page->shouldCheckParserCache( $parserOptions, $revision->getId() )
216 && $parserOutput->isCacheable()
218 $ctime = wfTimestamp( TS_MW
, (int)$start ); // cache time
219 ParserCache
::singleton()->save(
220 $parserOutput, $page, $parserOptions, $ctime, $revision->getId()
225 $updates = $content->getSecondaryDataUpdates(
228 !empty( $this->params
['useRecursiveLinksUpdate'] ),
232 foreach ( $updates as $key => $update ) {
233 // FIXME: move category change RC stuff to a separate update.
234 // RC entry addition aborts if edits where since made, which is not necessary.
235 // It's also an SoC violation for links update code to care about RC.
236 if ( $update instanceof LinksUpdate
) {
237 if ( !empty( $this->params
['triggeredRecursive'] ) ) {
238 $update->setTriggeredRecursive();
240 if ( !empty( $this->params
['triggeringUser'] ) ) {
241 $userInfo = $this->params
['triggeringUser'];
242 if ( $userInfo['userId'] ) {
243 $user = User
::newFromId( $userInfo['userId'] );
245 // Anonymous, use the username
246 $user = User
::newFromName( $userInfo['userName'], false );
248 $update->setTriggeringUser( $user );
250 if ( !empty( $this->params
['triggeringRevisionId'] ) ) {
251 $update->setRevision( $revision );
256 $latestNow = $page->lockAndGetLatest();
257 if ( !$latestNow ||
$revision->getId() != $latestNow ) {
258 // Do not clobber over newer updates with older ones. If all jobs where FIFO and
259 // serialized, it would be OK to update links based on older revisions since it
260 // would eventually get to the latest. Since that is not the case (by design),
261 // only update the link tables to a state matching the current revision's output.
262 $this->setLastError( "page_latest changed from {$revision->getId()} to $latestNow" );
266 DataUpdate
::runUpdates( $updates );
268 InfoAction
::invalidateCache( $title );
273 public function getDeduplicationInfo() {
274 $info = parent
::getDeduplicationInfo();
275 if ( is_array( $info['params'] ) ) {
276 // Don't let highly unique "masterPos" values ruin duplicate detection
277 unset( $info['params']['masterPos'] );
278 // For per-pages jobs, the job title is that of the template that changed
279 // (or similar), so remove that since it ruins duplicate detection
280 if ( isset( $info['pages'] ) ) {
281 unset( $info['namespace'] );
282 unset( $info['title'] );
289 public function workItemCount() {
290 return isset( $this->params
['pages'] ) ?
count( $this->params
['pages'] ) : 1;