6 array_diff_* were computed twice. Compute once, and remember them instead.
[lhc/web/wiklou.git] / includes / LinksUpdate.php
1 <?php
2 /**
3 * See docs/deferred.txt
4 *
5 * @todo document (e.g. one-sentence top-level class description).
6 */
7 class LinksUpdate {
8
9 /**@{{
10 * @private
11 */
12 var $mId, //!< Page ID of the article linked from
13 $mTitle, //!< Title object of the article linked from
14 $mLinks, //!< Map of title strings to IDs for the links in the document
15 $mImages, //!< DB keys of the images used, in the array key only
16 $mTemplates, //!< Map of title strings to IDs for the template references, including broken ones
17 $mExternals, //!< URLs of external links, array key only
18 $mCategories, //!< Map of category names to sort keys
19 $mInterlangs, //!< Map of language codes to titles
20 $mProperties, //!< Map of arbitrary name to value
21 $mDb, //!< Database connection reference
22 $mOptions, //!< SELECT options to be used (array)
23 $mRecursive; //!< Whether to queue jobs for recursive updates
24 /**@}}*/
25
26 /**
27 * Constructor
28 *
29 * @param Title $title Title of the page we're updating
30 * @param ParserOutput $parserOutput Output from a full parse of this page
31 * @param bool $recursive Queue jobs for recursive updates?
32 */
33 function LinksUpdate( $title, $parserOutput, $recursive = true ) {
34 global $wgAntiLockFlags;
35
36 if ( $wgAntiLockFlags & ALF_NO_LINK_LOCK ) {
37 $this->mOptions = array();
38 } else {
39 $this->mOptions = array( 'FOR UPDATE' );
40 }
41 $this->mDb = wfGetDB( DB_MASTER );
42
43 if ( !is_object( $title ) ) {
44 throw new MWException( "The calling convention to LinksUpdate::LinksUpdate() has changed. " .
45 "Please see Article::editUpdates() for an invocation example.\n" );
46 }
47 $this->mTitle = $title;
48 $this->mId = $title->getArticleID();
49
50 $this->mParserOutput = $parserOutput;
51 $this->mLinks = $parserOutput->getLinks();
52 $this->mImages = $parserOutput->getImages();
53 $this->mTemplates = $parserOutput->getTemplates();
54 $this->mExternals = $parserOutput->getExternalLinks();
55 $this->mCategories = $parserOutput->getCategories();
56 $this->mProperties = $parserOutput->getProperties();
57
58 # Convert the format of the interlanguage links
59 # I didn't want to change it in the ParserOutput, because that array is passed all
60 # the way back to the skin, so either a skin API break would be required, or an
61 # inefficient back-conversion.
62 $ill = $parserOutput->getLanguageLinks();
63 $this->mInterlangs = array();
64 foreach ( $ill as $link ) {
65 list( $key, $title ) = explode( ':', $link, 2 );
66 $this->mInterlangs[$key] = $title;
67 }
68
69 $this->mRecursive = $recursive;
70
71 wfRunHooks( 'LinksUpdateConstructed', array( &$this ) );
72 }
73
74 /**
75 * Update link tables with outgoing links from an updated article
76 */
77 function doUpdate() {
78 global $wgUseDumbLinkUpdate;
79
80 wfRunHooks( 'LinksUpdate', array( &$this ) );
81 if ( $wgUseDumbLinkUpdate ) {
82 $this->doDumbUpdate();
83 } else {
84 $this->doIncrementalUpdate();
85 }
86 wfRunHooks( 'LinksUpdateComplete', array( &$this ) );
87
88 }
89
90 function doIncrementalUpdate() {
91 wfProfileIn( __METHOD__ );
92
93 # Page links
94 $existing = $this->getExistingLinks();
95 $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ),
96 $this->getLinkInsertions( $existing ) );
97
98 # Image links
99 $existing = $this->getExistingImages();
100
101 $imageDeletes = $this->getImageDeletions( $existing );
102 $imageInserts = $this->getImageInsertions( $existing );
103 $this->incrTableUpdate( 'imagelinks', 'il', $imageDeletes, $imageInserts );
104
105 # Invalidate all image description pages which had links added or removed
106 $imageUpdates = $imageDeletions + $imageInsertions
107 $this->invalidateImageDescriptions( $imageUpdates );
108
109 # External links
110 $existing = $this->getExistingExternals();
111 $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ),
112 $this->getExternalInsertions( $existing ) );
113
114 # Language links
115 $existing = $this->getExistingInterlangs();
116 $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ),
117 $this->getInterlangInsertions( $existing ) );
118
119 # Template links
120 $existing = $this->getExistingTemplates();
121 $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
122 $this->getTemplateInsertions( $existing ) );
123
124 # Category links
125 $existing = $this->getExistingCategories();
126
127 $categoryDeletes = $this->getCategoryDeletions( $existing );
128 $categoryInserts = $this->getCategoryInsertions( $existing );
129
130 $this->incrTableUpdate( 'categorylinks', 'cl', $categoryDeletes, $categoryInserts );
131
132 # Invalidate all categories which were added, deleted or changed (set symmetric difference)
133 $categoryUpdates = $categoryInserts + $categoryDeletes;
134 $this->invalidateCategories( $categoryUpdates );
135 $this->updateCategoryCounts( $categoryInserts, $categoryDeletes );
136
137 # Page properties
138 $existing = $this->getExistingProperties();
139
140 $propertiesDeletes = $this->getPropertiesDeletions( $existing );
141 $propertiesInserts = $this->getPropertiesInsertions( $existing );
142
143 $this->incrTableUpdate( 'page_props', 'pp', $propertiesDeletes, $propertiesInserts );
144
145 # Invalidate the necessary pages
146 $changed = $propertiesDeletes + $propertiesInserts;
147 $this->invalidateProperties( $changed );
148
149 # Refresh links of all pages including this page
150 # This will be in a separate transaction
151 if ( $this->mRecursive ) {
152 $this->queueRecursiveJobs();
153 }
154
155 wfProfileOut( __METHOD__ );
156 }
157
158 /**
159 * Link update which clears the previous entries and inserts new ones
160 * May be slower or faster depending on level of lock contention and write speed of DB
161 * Also useful where link table corruption needs to be repaired, e.g. in refreshLinks.php
162 */
163 function doDumbUpdate() {
164 wfProfileIn( __METHOD__ );
165
166 # Refresh category pages and image description pages
167 $existing = $this->getExistingCategories();
168 $categoryInserts = array_diff_assoc( $this->mCategories, $existing );
169 $categoryDeletes = array_diff_assoc( $existing, $this->mCategoties );
170 $categoryUpdates = $categoryInserts + $categoryDeletes;
171 $existing = $this->getExistingImages();
172 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
173
174 $this->dumbTableUpdate( 'pagelinks', $this->getLinkInsertions(), 'pl_from' );
175 $this->dumbTableUpdate( 'imagelinks', $this->getImageInsertions(), 'il_from' );
176 $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' );
177 $this->dumbTableUpdate( 'templatelinks', $this->getTemplateInsertions(), 'tl_from' );
178 $this->dumbTableUpdate( 'externallinks', $this->getExternalInsertions(), 'el_from' );
179 $this->dumbTableUpdate( 'langlinks', $this->getInterlangInsertions(),'ll_from' );
180 $this->dumbTableUpdate( 'page_props', $this->getPropertyInsertions(), 'pp_page' );
181
182 # Update the cache of all the category pages and image description
183 # pages which were changed, and fix the category table count
184 $this->invalidateCategories( $categoryUpdates );
185 $this->updateCategoryCounts( $categoryInserts, $categoryDeletes );
186 $this->invalidateImageDescriptions( $imageUpdates );
187
188 # Refresh links of all pages including this page
189 # This will be in a separate transaction
190 if ( $this->mRecursive ) {
191 $this->queueRecursiveJobs();
192 }
193
194 wfProfileOut( __METHOD__ );
195 }
196
197 function queueRecursiveJobs() {
198 wfProfileIn( __METHOD__ );
199
200 $batchSize = 100;
201 $dbr = wfGetDB( DB_SLAVE );
202 $res = $dbr->select( array( 'templatelinks', 'page' ),
203 array( 'page_namespace', 'page_title' ),
204 array(
205 'page_id=tl_from',
206 'tl_namespace' => $this->mTitle->getNamespace(),
207 'tl_title' => $this->mTitle->getDBkey()
208 ), __METHOD__
209 );
210
211 $done = false;
212 while ( !$done ) {
213 $jobs = array();
214 for ( $i = 0; $i < $batchSize; $i++ ) {
215 $row = $dbr->fetchObject( $res );
216 if ( !$row ) {
217 $done = true;
218 break;
219 }
220 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
221 $jobs[] = new RefreshLinksJob( $title, '' );
222 }
223 Job::batchInsert( $jobs );
224 }
225 $dbr->freeResult( $res );
226 wfProfileOut( __METHOD__ );
227 }
228
229 /**
230 * Invalidate the cache of a list of pages from a single namespace
231 *
232 * @param integer $namespace
233 * @param array $dbkeys
234 */
235 function invalidatePages( $namespace, $dbkeys ) {
236 if ( !count( $dbkeys ) ) {
237 return;
238 }
239
240 /**
241 * Determine which pages need to be updated
242 * This is necessary to prevent the job queue from smashing the DB with
243 * large numbers of concurrent invalidations of the same page
244 */
245 $now = $this->mDb->timestamp();
246 $ids = array();
247 $res = $this->mDb->select( 'page', array( 'page_id' ),
248 array(
249 'page_namespace' => $namespace,
250 'page_title IN (' . $this->mDb->makeList( $dbkeys ) . ')',
251 'page_touched < ' . $this->mDb->addQuotes( $now )
252 ), __METHOD__
253 );
254 while ( $row = $this->mDb->fetchObject( $res ) ) {
255 $ids[] = $row->page_id;
256 }
257 if ( !count( $ids ) ) {
258 return;
259 }
260
261 /**
262 * Do the update
263 * We still need the page_touched condition, in case the row has changed since
264 * the non-locking select above.
265 */
266 $this->mDb->update( 'page', array( 'page_touched' => $now ),
267 array(
268 'page_id IN (' . $this->mDb->makeList( $ids ) . ')',
269 'page_touched < ' . $this->mDb->addQuotes( $now )
270 ), __METHOD__
271 );
272 }
273
274 function invalidateCategories( $cats ) {
275 $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) );
276 }
277
278 /**
279 * Update all the appropriate counts in the category table.
280 * @param $added associative array of category name => sort key
281 * @param $deleted associative array of category name => sort key
282 */
283 function updateCategoryCounts( $added, $deleted ) {
284 $a = new Article($this->mTitle);
285 $a->updateCategoryCounts(
286 array_keys( $added ), array_keys( $deleted )
287 );
288 }
289
290 function invalidateImageDescriptions( $images ) {
291 $this->invalidatePages( NS_IMAGE, array_keys( $images ) );
292 }
293
294 function dumbTableUpdate( $table, $insertions, $fromField ) {
295 $this->mDb->delete( $table, array( $fromField => $this->mId ), __METHOD__ );
296 if ( count( $insertions ) ) {
297 # The link array was constructed without FOR UPDATE, so there may
298 # be collisions. This may cause minor link table inconsistencies,
299 # which is better than crippling the site with lock contention.
300 $this->mDb->insert( $table, $insertions, __METHOD__, array( 'IGNORE' ) );
301 }
302 }
303
304 /**
305 * Make a WHERE clause from a 2-d NS/dbkey array
306 *
307 * @param array $arr 2-d array indexed by namespace and DB key
308 * @param string $prefix Field name prefix, without the underscore
309 */
310 function makeWhereFrom2d( &$arr, $prefix ) {
311 $lb = new LinkBatch;
312 $lb->setArray( $arr );
313 return $lb->constructSet( $prefix, $this->mDb );
314 }
315
316 /**
317 * Update a table by doing a delete query then an insert query
318 * @private
319 */
320 function incrTableUpdate( $table, $prefix, $deletions, $insertions ) {
321 if ( $table == 'page_props' ) {
322 $fromField = 'pp_page';
323 } else {
324 $fromField = "{$prefix}_from";
325 }
326 $where = array( $fromField => $this->mId );
327 if ( $table == 'pagelinks' || $table == 'templatelinks' ) {
328 $clause = $this->makeWhereFrom2d( $deletions, $prefix );
329 if ( $clause ) {
330 $where[] = $clause;
331 } else {
332 $where = false;
333 }
334 } else {
335 if ( $table == 'langlinks' ) {
336 $toField = 'll_lang';
337 } elseif ( $table == 'page_props' ) {
338 $toField = 'pp_propname';
339 } else {
340 $toField = $prefix . '_to';
341 }
342 if ( count( $deletions ) ) {
343 $where[] = "$toField IN (" . $this->mDb->makeList( array_keys( $deletions ) ) . ')';
344 } else {
345 $where = false;
346 }
347 }
348 if ( $where ) {
349 $this->mDb->delete( $table, $where, __METHOD__ );
350 }
351 if ( count( $insertions ) ) {
352 $this->mDb->insert( $table, $insertions, __METHOD__, 'IGNORE' );
353 }
354 }
355
356
357 /**
358 * Get an array of pagelinks insertions for passing to the DB
359 * Skips the titles specified by the 2-D array $existing
360 * @private
361 */
362 function getLinkInsertions( $existing = array() ) {
363 $arr = array();
364 foreach( $this->mLinks as $ns => $dbkeys ) {
365 # array_diff_key() was introduced in PHP 5.1, there is a compatibility function
366 # in GlobalFunctions.php
367 $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
368 foreach ( $diffs as $dbk => $id ) {
369 $arr[] = array(
370 'pl_from' => $this->mId,
371 'pl_namespace' => $ns,
372 'pl_title' => $dbk
373 );
374 }
375 }
376 return $arr;
377 }
378
379 /**
380 * Get an array of template insertions. Like getLinkInsertions()
381 * @private
382 */
383 function getTemplateInsertions( $existing = array() ) {
384 $arr = array();
385 foreach( $this->mTemplates as $ns => $dbkeys ) {
386 $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
387 foreach ( $diffs as $dbk => $id ) {
388 $arr[] = array(
389 'tl_from' => $this->mId,
390 'tl_namespace' => $ns,
391 'tl_title' => $dbk
392 );
393 }
394 }
395 return $arr;
396 }
397
398 /**
399 * Get an array of image insertions
400 * Skips the names specified in $existing
401 * @private
402 */
403 function getImageInsertions( $existing = array() ) {
404 $arr = array();
405 $diffs = array_diff_key( $this->mImages, $existing );
406 foreach( $diffs as $iname => $dummy ) {
407 $arr[] = array(
408 'il_from' => $this->mId,
409 'il_to' => $iname
410 );
411 }
412 return $arr;
413 }
414
415 /**
416 * Get an array of externallinks insertions. Skips the names specified in $existing
417 * @private
418 */
419 function getExternalInsertions( $existing = array() ) {
420 $arr = array();
421 $diffs = array_diff_key( $this->mExternals, $existing );
422 foreach( $diffs as $url => $dummy ) {
423 $arr[] = array(
424 'el_from' => $this->mId,
425 'el_to' => $url,
426 'el_index' => wfMakeUrlIndex( $url ),
427 );
428 }
429 return $arr;
430 }
431
432 /**
433 * Get an array of category insertions
434 * @param array $existing Array mapping existing category names to sort keys. If both
435 * match a link in $this, the link will be omitted from the output
436 * @private
437 */
438 function getCategoryInsertions( $existing = array() ) {
439 $diffs = array_diff_assoc( $this->mCategories, $existing );
440 $arr = array();
441 foreach ( $diffs as $name => $sortkey ) {
442 $arr[] = array(
443 'cl_from' => $this->mId,
444 'cl_to' => $name,
445 'cl_sortkey' => $sortkey,
446 'cl_timestamp' => $this->mDb->timestamp()
447 );
448 }
449 return $arr;
450 }
451
452 /**
453 * Get an array of interlanguage link insertions
454 * @param array $existing Array mapping existing language codes to titles
455 * @private
456 */
457 function getInterlangInsertions( $existing = array() ) {
458 $diffs = array_diff_assoc( $this->mInterlangs, $existing );
459 $arr = array();
460 foreach( $diffs as $lang => $title ) {
461 $arr[] = array(
462 'll_from' => $this->mId,
463 'll_lang' => $lang,
464 'll_title' => $title
465 );
466 }
467 return $arr;
468 }
469
470 /**
471 * Get an array of page property insertions
472 */
473 function getPropertyInsertions( $existing = array() ) {
474 $diffs = array_diff_assoc( $this->mProperties, $existing );
475 $arr = array();
476 foreach ( $diffs as $name => $value ) {
477 $arr[] = array(
478 'pp_page' => $this->mId,
479 'pp_propname' => $name,
480 'pp_value' => $value,
481 );
482 }
483 return $arr;
484 }
485
486
487 /**
488 * Given an array of existing links, returns those links which are not in $this
489 * and thus should be deleted.
490 * @private
491 */
492 function getLinkDeletions( $existing ) {
493 $del = array();
494 foreach ( $existing as $ns => $dbkeys ) {
495 if ( isset( $this->mLinks[$ns] ) ) {
496 $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] );
497 } else {
498 $del[$ns] = $existing[$ns];
499 }
500 }
501 return $del;
502 }
503
504 /**
505 * Given an array of existing templates, returns those templates which are not in $this
506 * and thus should be deleted.
507 * @private
508 */
509 function getTemplateDeletions( $existing ) {
510 $del = array();
511 foreach ( $existing as $ns => $dbkeys ) {
512 if ( isset( $this->mTemplates[$ns] ) ) {
513 $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] );
514 } else {
515 $del[$ns] = $existing[$ns];
516 }
517 }
518 return $del;
519 }
520
521 /**
522 * Given an array of existing images, returns those images which are not in $this
523 * and thus should be deleted.
524 * @private
525 */
526 function getImageDeletions( $existing ) {
527 return array_diff_key( $existing, $this->mImages );
528 }
529
530 /**
531 * Given an array of existing external links, returns those links which are not
532 * in $this and thus should be deleted.
533 * @private
534 */
535 function getExternalDeletions( $existing ) {
536 return array_diff_key( $existing, $this->mExternals );
537 }
538
539 /**
540 * Given an array of existing categories, returns those categories which are not in $this
541 * and thus should be deleted.
542 * @private
543 */
544 function getCategoryDeletions( $existing ) {
545 return array_diff_assoc( $existing, $this->mCategories );
546 }
547
548 /**
549 * Given an array of existing interlanguage links, returns those links which are not
550 * in $this and thus should be deleted.
551 * @private
552 */
553 function getInterlangDeletions( $existing ) {
554 return array_diff_assoc( $existing, $this->mInterlangs );
555 }
556
557 /**
558 * Get array of properties which should be deleted.
559 * @private
560 */
561 function getPropertyDeletions( $existing ) {
562 return array_diff_assoc( $existing, $this->mProperties );
563 }
564
565 /**
566 * Get an array of existing links, as a 2-D array
567 * @private
568 */
569 function getExistingLinks() {
570 $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ),
571 array( 'pl_from' => $this->mId ), __METHOD__, $this->mOptions );
572 $arr = array();
573 while ( $row = $this->mDb->fetchObject( $res ) ) {
574 if ( !isset( $arr[$row->pl_namespace] ) ) {
575 $arr[$row->pl_namespace] = array();
576 }
577 $arr[$row->pl_namespace][$row->pl_title] = 1;
578 }
579 $this->mDb->freeResult( $res );
580 return $arr;
581 }
582
583 /**
584 * Get an array of existing templates, as a 2-D array
585 * @private
586 */
587 function getExistingTemplates() {
588 $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ),
589 array( 'tl_from' => $this->mId ), __METHOD__, $this->mOptions );
590 $arr = array();
591 while ( $row = $this->mDb->fetchObject( $res ) ) {
592 if ( !isset( $arr[$row->tl_namespace] ) ) {
593 $arr[$row->tl_namespace] = array();
594 }
595 $arr[$row->tl_namespace][$row->tl_title] = 1;
596 }
597 $this->mDb->freeResult( $res );
598 return $arr;
599 }
600
601 /**
602 * Get an array of existing images, image names in the keys
603 * @private
604 */
605 function getExistingImages() {
606 $res = $this->mDb->select( 'imagelinks', array( 'il_to' ),
607 array( 'il_from' => $this->mId ), __METHOD__, $this->mOptions );
608 $arr = array();
609 while ( $row = $this->mDb->fetchObject( $res ) ) {
610 $arr[$row->il_to] = 1;
611 }
612 $this->mDb->freeResult( $res );
613 return $arr;
614 }
615
616 /**
617 * Get an array of existing external links, URLs in the keys
618 * @private
619 */
620 function getExistingExternals() {
621 $res = $this->mDb->select( 'externallinks', array( 'el_to' ),
622 array( 'el_from' => $this->mId ), __METHOD__, $this->mOptions );
623 $arr = array();
624 while ( $row = $this->mDb->fetchObject( $res ) ) {
625 $arr[$row->el_to] = 1;
626 }
627 $this->mDb->freeResult( $res );
628 return $arr;
629 }
630
631 /**
632 * Get an array of existing categories, with the name in the key and sort key in the value.
633 * @private
634 */
635 function getExistingCategories() {
636 $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey' ),
637 array( 'cl_from' => $this->mId ), __METHOD__, $this->mOptions );
638 $arr = array();
639 while ( $row = $this->mDb->fetchObject( $res ) ) {
640 $arr[$row->cl_to] = $row->cl_sortkey;
641 }
642 $this->mDb->freeResult( $res );
643 return $arr;
644 }
645
646 /**
647 * Get an array of existing interlanguage links, with the language code in the key and the
648 * title in the value.
649 * @private
650 */
651 function getExistingInterlangs() {
652 $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ),
653 array( 'll_from' => $this->mId ), __METHOD__, $this->mOptions );
654 $arr = array();
655 while ( $row = $this->mDb->fetchObject( $res ) ) {
656 $arr[$row->ll_lang] = $row->ll_title;
657 }
658 return $arr;
659 }
660
661 /**
662 * Get an array of existing categories, with the name in the key and sort key in the value.
663 * @private
664 */
665 function getExistingProperties() {
666 $res = $this->mDb->select( 'page_props', array( 'pp_propname', 'pp_value' ),
667 array( 'pp_page' => $this->mId ), __METHOD__, $this->mOptions );
668 $arr = array();
669 while ( $row = $this->mDb->fetchObject( $res ) ) {
670 $arr[$row->pp_propname] = $row->pp_value;
671 }
672 $this->mDb->freeResult( $res );
673 return $arr;
674 }
675
676
677 /**
678 * Return the title object of the page being updated
679 */
680 function getTitle() {
681 return $this->mTitle;
682 }
683
684 /**
685 * Invalidate any necessary link lists related to page property changes
686 */
687 function invalidateProperties( $changed ) {
688 global $wgPagePropLinkInvalidations;
689
690 foreach ( $changed as $name => $value ) {
691 if ( isset( $wgPagePropLinkInvalidations[$name] ) ) {
692 $inv = $wgPagePropLinkInvalidations[$name];
693 if ( !is_array( $inv ) ) {
694 $inv = array( $inv );
695 }
696 foreach ( $inv as $table ) {
697 $update = new HTMLCacheUpdate( $this->mTitle, $table );
698 $update->doUpdate();
699 }
700 }
701 }
702 }
703 }