3 * See docs/deferred.txt
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
20 * @todo document (e.g. one-sentence top-level class description).
22 class LinksUpdate
extends SecondaryDBDataUpdate
{
27 var $mId, //!< Page ID of the article linked from
28 $mTitle, //!< Title object of the article linked from
29 $mParserOutput, //!< Whether to queue jobs for recursive update
30 $mLinks, //!< Map of title strings to IDs for the links in the document
31 $mImages, //!< DB keys of the images used, in the array key only
32 $mTemplates, //!< Map of title strings to IDs for the template references, including broken ones
33 $mExternals, //!< URLs of external links, array key only
34 $mCategories, //!< Map of category names to sort keys
35 $mInterlangs, //!< Map of language codes to titles
36 $mProperties, //!< Map of arbitrary name to value
37 $mRecursive; //!< Whether to queue jobs for recursive updates
43 * @param $title Title of the page we're updating
44 * @param $parserOutput ParserOutput: output from a full parse of this page
45 * @param $recursive Boolean: queue jobs for recursive updates?
47 function __construct( $title, $parserOutput, $recursive = true ) {
48 parent
::__construct( );
50 if ( !is_object( $title ) ) {
51 throw new MWException( "The calling convention to LinksUpdate::LinksUpdate() has changed. " .
52 "Please see Article::editUpdates() for an invocation example.\n" );
55 $this->mTitle
= $title;
56 $this->mId
= $title->getArticleID();
58 $this->mParserOutput
= $parserOutput;
60 $this->mLinks
= $parserOutput->getLinks();
61 $this->mImages
= $parserOutput->getImages();
62 $this->mTemplates
= $parserOutput->getTemplates();
63 $this->mExternals
= $parserOutput->getExternalLinks();
64 $this->mCategories
= $parserOutput->getCategories();
65 $this->mProperties
= $parserOutput->getProperties();
66 $this->mInterwikis
= $parserOutput->getInterwikiLinks();
68 # Convert the format of the interlanguage links
69 # I didn't want to change it in the ParserOutput, because that array is passed all
70 # the way back to the skin, so either a skin API break would be required, or an
71 # inefficient back-conversion.
72 $ill = $parserOutput->getLanguageLinks();
73 $this->mInterlangs
= array();
74 foreach ( $ill as $link ) {
75 list( $key, $title ) = explode( ':', $link, 2 );
76 $this->mInterlangs
[$key] = $title;
79 foreach ( $this->mCategories
as &$sortkey ) {
80 # If the sortkey is longer then 255 bytes,
81 # it truncated by DB, and then doesn't get
82 # matched when comparing existing vs current
83 # categories, causing bug 25254.
84 # Also. substr behaves weird when given "".
85 if ( $sortkey !== '' ) {
86 $sortkey = substr( $sortkey, 0, 255 );
90 $this->mRecursive
= $recursive;
92 wfRunHooks( 'LinksUpdateConstructed', array( &$this ) );
96 * Update link tables with outgoing links from an updated article
98 public function doUpdate() {
99 global $wgUseDumbLinkUpdate;
101 wfRunHooks( 'LinksUpdate', array( &$this ) );
102 if ( $wgUseDumbLinkUpdate ) {
103 $this->doDumbUpdate();
105 $this->doIncrementalUpdate();
107 wfRunHooks( 'LinksUpdateComplete', array( &$this ) );
110 protected function doIncrementalUpdate() {
111 wfProfileIn( __METHOD__
);
114 $existing = $this->getExistingLinks();
115 $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ),
116 $this->getLinkInsertions( $existing ) );
119 $existing = $this->getExistingImages();
121 $imageDeletes = $this->getImageDeletions( $existing );
122 $this->incrTableUpdate( 'imagelinks', 'il', $imageDeletes,
123 $this->getImageInsertions( $existing ) );
125 # Invalidate all image description pages which had links added or removed
126 $imageUpdates = $imageDeletes +
array_diff_key( $this->mImages
, $existing );
127 $this->invalidateImageDescriptions( $imageUpdates );
130 $existing = $this->getExistingExternals();
131 $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ),
132 $this->getExternalInsertions( $existing ) );
135 $existing = $this->getExistingInterlangs();
136 $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ),
137 $this->getInterlangInsertions( $existing ) );
139 # Inline interwiki links
140 $existing = $this->getExistingInterwikis();
141 $this->incrTableUpdate( 'iwlinks', 'iwl', $this->getInterwikiDeletions( $existing ),
142 $this->getInterwikiInsertions( $existing ) );
145 $existing = $this->getExistingTemplates();
146 $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
147 $this->getTemplateInsertions( $existing ) );
150 $existing = $this->getExistingCategories();
152 $categoryDeletes = $this->getCategoryDeletions( $existing );
154 $this->incrTableUpdate( 'categorylinks', 'cl', $categoryDeletes,
155 $this->getCategoryInsertions( $existing ) );
157 # Invalidate all categories which were added, deleted or changed (set symmetric difference)
158 $categoryInserts = array_diff_assoc( $this->mCategories
, $existing );
159 $categoryUpdates = $categoryInserts +
$categoryDeletes;
160 $this->invalidateCategories( $categoryUpdates );
161 $this->updateCategoryCounts( $categoryInserts, $categoryDeletes );
164 $existing = $this->getExistingProperties();
166 $propertiesDeletes = $this->getPropertyDeletions( $existing );
168 $this->incrTableUpdate( 'page_props', 'pp', $propertiesDeletes,
169 $this->getPropertyInsertions( $existing ) );
171 # Invalidate the necessary pages
172 $changed = $propertiesDeletes +
array_diff_assoc( $this->mProperties
, $existing );
173 $this->invalidateProperties( $changed );
175 # Refresh links of all pages including this page
176 # This will be in a separate transaction
177 if ( $this->mRecursive
) {
178 $this->queueRecursiveJobs();
181 wfProfileOut( __METHOD__
);
185 * Link update which clears the previous entries and inserts new ones
186 * May be slower or faster depending on level of lock contention and write speed of DB
187 * Also useful where link table corruption needs to be repaired, e.g. in refreshLinks.php
189 protected function doDumbUpdate() {
190 wfProfileIn( __METHOD__
);
192 # Refresh category pages and image description pages
193 $existing = $this->getExistingCategories();
194 $categoryInserts = array_diff_assoc( $this->mCategories
, $existing );
195 $categoryDeletes = array_diff_assoc( $existing, $this->mCategories
);
196 $categoryUpdates = $categoryInserts +
$categoryDeletes;
197 $existing = $this->getExistingImages();
198 $imageUpdates = array_diff_key( $existing, $this->mImages
) +
array_diff_key( $this->mImages
, $existing );
200 $this->dumbTableUpdate( 'pagelinks', $this->getLinkInsertions(), 'pl_from' );
201 $this->dumbTableUpdate( 'imagelinks', $this->getImageInsertions(), 'il_from' );
202 $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' );
203 $this->dumbTableUpdate( 'templatelinks', $this->getTemplateInsertions(), 'tl_from' );
204 $this->dumbTableUpdate( 'externallinks', $this->getExternalInsertions(), 'el_from' );
205 $this->dumbTableUpdate( 'langlinks', $this->getInterlangInsertions(),'ll_from' );
206 $this->dumbTableUpdate( 'iwlinks', $this->getInterwikiInsertions(),'iwl_from' );
207 $this->dumbTableUpdate( 'page_props', $this->getPropertyInsertions(), 'pp_page' );
209 # Update the cache of all the category pages and image description
210 # pages which were changed, and fix the category table count
211 $this->invalidateCategories( $categoryUpdates );
212 $this->updateCategoryCounts( $categoryInserts, $categoryDeletes );
213 $this->invalidateImageDescriptions( $imageUpdates );
215 # Refresh links of all pages including this page
216 # This will be in a separate transaction
217 if ( $this->mRecursive
) {
218 $this->queueRecursiveJobs();
221 wfProfileOut( __METHOD__
);
224 function queueRecursiveJobs() {
225 global $wgUpdateRowsPerJob;
226 wfProfileIn( __METHOD__
);
228 $cache = $this->mTitle
->getBacklinkCache();
229 $batches = $cache->partition( 'templatelinks', $wgUpdateRowsPerJob );
231 wfProfileOut( __METHOD__
);
235 foreach ( $batches as $batch ) {
236 list( $start, $end ) = $batch;
238 'table' => 'templatelinks',
242 $jobs[] = new RefreshLinksJob2( $this->mTitle
, $params );
244 Job
::batchInsert( $jobs );
246 wfProfileOut( __METHOD__
);
252 function invalidateCategories( $cats ) {
253 $this->invalidatePages( NS_CATEGORY
, array_keys( $cats ) );
257 * Update all the appropriate counts in the category table.
258 * @param $added array associative array of category name => sort key
259 * @param $deleted array associative array of category name => sort key
261 function updateCategoryCounts( $added, $deleted ) {
262 $a = WikiPage
::factory( $this->mTitle
);
263 $a->updateCategoryCounts(
264 array_keys( $added ), array_keys( $deleted )
271 function invalidateImageDescriptions( $images ) {
272 $this->invalidatePages( NS_FILE
, array_keys( $images ) );
280 private function dumbTableUpdate( $table, $insertions, $fromField ) {
281 $this->mDb
->delete( $table, array( $fromField => $this->mId
), __METHOD__
);
282 if ( count( $insertions ) ) {
283 # The link array was constructed without FOR UPDATE, so there may
284 # be collisions. This may cause minor link table inconsistencies,
285 # which is better than crippling the site with lock contention.
286 $this->mDb
->insert( $table, $insertions, __METHOD__
, array( 'IGNORE' ) );
291 * Update a table by doing a delete query then an insert query
297 function incrTableUpdate( $table, $prefix, $deletions, $insertions ) {
298 if ( $table == 'page_props' ) {
299 $fromField = 'pp_page';
301 $fromField = "{$prefix}_from";
303 $where = array( $fromField => $this->mId
);
304 if ( $table == 'pagelinks' ||
$table == 'templatelinks' ||
$table == 'iwlinks' ) {
305 if ( $table == 'iwlinks' ) {
306 $baseKey = 'iwl_prefix';
308 $baseKey = "{$prefix}_namespace";
310 $clause = $this->mDb
->makeWhereFrom2d( $deletions, $baseKey, "{$prefix}_title" );
317 if ( $table == 'langlinks' ) {
318 $toField = 'll_lang';
319 } elseif ( $table == 'page_props' ) {
320 $toField = 'pp_propname';
322 $toField = $prefix . '_to';
324 if ( count( $deletions ) ) {
325 $where[] = "$toField IN (" . $this->mDb
->makeList( array_keys( $deletions ) ) . ')';
331 $this->mDb
->delete( $table, $where, __METHOD__
);
333 if ( count( $insertions ) ) {
334 $this->mDb
->insert( $table, $insertions, __METHOD__
, 'IGNORE' );
339 * Get an array of pagelinks insertions for passing to the DB
340 * Skips the titles specified by the 2-D array $existing
341 * @param $existing array
344 private function getLinkInsertions( $existing = array() ) {
346 foreach( $this->mLinks
as $ns => $dbkeys ) {
347 $diffs = isset( $existing[$ns] )
348 ?
array_diff_key( $dbkeys, $existing[$ns] )
350 foreach ( $diffs as $dbk => $id ) {
352 'pl_from' => $this->mId
,
353 'pl_namespace' => $ns,
362 * Get an array of template insertions. Like getLinkInsertions()
363 * @param $existing array
366 private function getTemplateInsertions( $existing = array() ) {
368 foreach( $this->mTemplates
as $ns => $dbkeys ) {
369 $diffs = isset( $existing[$ns] ) ?
array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
370 foreach ( $diffs as $dbk => $id ) {
372 'tl_from' => $this->mId
,
373 'tl_namespace' => $ns,
382 * Get an array of image insertions
383 * Skips the names specified in $existing
384 * @param $existing array
387 private function getImageInsertions( $existing = array() ) {
389 $diffs = array_diff_key( $this->mImages
, $existing );
390 foreach( $diffs as $iname => $dummy ) {
392 'il_from' => $this->mId
,
400 * Get an array of externallinks insertions. Skips the names specified in $existing
401 * @param $existing array
404 private function getExternalInsertions( $existing = array() ) {
406 $diffs = array_diff_key( $this->mExternals
, $existing );
407 foreach( $diffs as $url => $dummy ) {
408 foreach( wfMakeUrlIndexes( $url ) as $index ) {
410 'el_from' => $this->mId
,
412 'el_index' => $index,
420 * Get an array of category insertions
422 * @param $existing array mapping existing category names to sort keys. If both
423 * match a link in $this, the link will be omitted from the output
427 private function getCategoryInsertions( $existing = array() ) {
428 global $wgContLang, $wgCategoryCollation;
429 $diffs = array_diff_assoc( $this->mCategories
, $existing );
431 foreach ( $diffs as $name => $prefix ) {
432 $nt = Title
::makeTitleSafe( NS_CATEGORY
, $name );
433 $wgContLang->findVariantLink( $name, $nt, true );
435 if ( $this->mTitle
->getNamespace() == NS_CATEGORY
) {
437 } elseif ( $this->mTitle
->getNamespace() == NS_FILE
) {
443 # Treat custom sortkeys as a prefix, so that if multiple
444 # things are forced to sort as '*' or something, they'll
445 # sort properly in the category rather than in page_id
447 $sortkey = Collation
::singleton()->getSortKey(
448 $this->mTitle
->getCategorySortkey( $prefix ) );
451 'cl_from' => $this->mId
,
453 'cl_sortkey' => $sortkey,
454 'cl_timestamp' => $this->mDb
->timestamp(),
455 'cl_sortkey_prefix' => $prefix,
456 'cl_collation' => $wgCategoryCollation,
464 * Get an array of interlanguage link insertions
466 * @param $existing Array mapping existing language codes to titles
470 private function getInterlangInsertions( $existing = array() ) {
471 $diffs = array_diff_assoc( $this->mInterlangs
, $existing );
473 foreach( $diffs as $lang => $title ) {
475 'll_from' => $this->mId
,
484 * Get an array of page property insertions
485 * @param $existing array
488 function getPropertyInsertions( $existing = array() ) {
489 $diffs = array_diff_assoc( $this->mProperties
, $existing );
491 foreach ( $diffs as $name => $value ) {
493 'pp_page' => $this->mId
,
494 'pp_propname' => $name,
495 'pp_value' => $value,
502 * Get an array of interwiki insertions for passing to the DB
503 * Skips the titles specified by the 2-D array $existing
504 * @param $existing array
507 private function getInterwikiInsertions( $existing = array() ) {
509 foreach( $this->mInterwikis
as $prefix => $dbkeys ) {
510 $diffs = isset( $existing[$prefix] ) ?
array_diff_key( $dbkeys, $existing[$prefix] ) : $dbkeys;
511 foreach ( $diffs as $dbk => $id ) {
513 'iwl_from' => $this->mId
,
514 'iwl_prefix' => $prefix,
523 * Given an array of existing links, returns those links which are not in $this
524 * and thus should be deleted.
525 * @param $existing array
528 private function getLinkDeletions( $existing ) {
530 foreach ( $existing as $ns => $dbkeys ) {
531 if ( isset( $this->mLinks
[$ns] ) ) {
532 $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks
[$ns] );
534 $del[$ns] = $existing[$ns];
541 * Given an array of existing templates, returns those templates which are not in $this
542 * and thus should be deleted.
543 * @param $existing array
546 private function getTemplateDeletions( $existing ) {
548 foreach ( $existing as $ns => $dbkeys ) {
549 if ( isset( $this->mTemplates
[$ns] ) ) {
550 $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates
[$ns] );
552 $del[$ns] = $existing[$ns];
559 * Given an array of existing images, returns those images which are not in $this
560 * and thus should be deleted.
561 * @param $existing array
564 private function getImageDeletions( $existing ) {
565 return array_diff_key( $existing, $this->mImages
);
569 * Given an array of existing external links, returns those links which are not
570 * in $this and thus should be deleted.
571 * @param $existing array
574 private function getExternalDeletions( $existing ) {
575 return array_diff_key( $existing, $this->mExternals
);
579 * Given an array of existing categories, returns those categories which are not in $this
580 * and thus should be deleted.
581 * @param $existing array
584 private function getCategoryDeletions( $existing ) {
585 return array_diff_assoc( $existing, $this->mCategories
);
589 * Given an array of existing interlanguage links, returns those links which are not
590 * in $this and thus should be deleted.
591 * @param $existing array
594 private function getInterlangDeletions( $existing ) {
595 return array_diff_assoc( $existing, $this->mInterlangs
);
599 * Get array of properties which should be deleted.
600 * @param $existing array
603 function getPropertyDeletions( $existing ) {
604 return array_diff_assoc( $existing, $this->mProperties
);
608 * Given an array of existing interwiki links, returns those links which are not in $this
609 * and thus should be deleted.
610 * @param $existing array
613 private function getInterwikiDeletions( $existing ) {
615 foreach ( $existing as $prefix => $dbkeys ) {
616 if ( isset( $this->mInterwikis
[$prefix] ) ) {
617 $del[$prefix] = array_diff_key( $existing[$prefix], $this->mInterwikis
[$prefix] );
619 $del[$prefix] = $existing[$prefix];
626 * Get an array of existing links, as a 2-D array
630 private function getExistingLinks() {
631 $res = $this->mDb
->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ),
632 array( 'pl_from' => $this->mId
), __METHOD__
, $this->mOptions
);
634 foreach ( $res as $row ) {
635 if ( !isset( $arr[$row->pl_namespace
] ) ) {
636 $arr[$row->pl_namespace
] = array();
638 $arr[$row->pl_namespace
][$row->pl_title
] = 1;
644 * Get an array of existing templates, as a 2-D array
648 private function getExistingTemplates() {
649 $res = $this->mDb
->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ),
650 array( 'tl_from' => $this->mId
), __METHOD__
, $this->mOptions
);
652 foreach ( $res as $row ) {
653 if ( !isset( $arr[$row->tl_namespace
] ) ) {
654 $arr[$row->tl_namespace
] = array();
656 $arr[$row->tl_namespace
][$row->tl_title
] = 1;
662 * Get an array of existing images, image names in the keys
666 private function getExistingImages() {
667 $res = $this->mDb
->select( 'imagelinks', array( 'il_to' ),
668 array( 'il_from' => $this->mId
), __METHOD__
, $this->mOptions
);
670 foreach ( $res as $row ) {
671 $arr[$row->il_to
] = 1;
677 * Get an array of existing external links, URLs in the keys
681 private function getExistingExternals() {
682 $res = $this->mDb
->select( 'externallinks', array( 'el_to' ),
683 array( 'el_from' => $this->mId
), __METHOD__
, $this->mOptions
);
685 foreach ( $res as $row ) {
686 $arr[$row->el_to
] = 1;
692 * Get an array of existing categories, with the name in the key and sort key in the value.
696 private function getExistingCategories() {
697 $res = $this->mDb
->select( 'categorylinks', array( 'cl_to', 'cl_sortkey_prefix' ),
698 array( 'cl_from' => $this->mId
), __METHOD__
, $this->mOptions
);
700 foreach ( $res as $row ) {
701 $arr[$row->cl_to
] = $row->cl_sortkey_prefix
;
707 * Get an array of existing interlanguage links, with the language code in the key and the
708 * title in the value.
712 private function getExistingInterlangs() {
713 $res = $this->mDb
->select( 'langlinks', array( 'll_lang', 'll_title' ),
714 array( 'll_from' => $this->mId
), __METHOD__
, $this->mOptions
);
716 foreach ( $res as $row ) {
717 $arr[$row->ll_lang
] = $row->ll_title
;
723 * Get an array of existing inline interwiki links, as a 2-D array
724 * @return array (prefix => array(dbkey => 1))
726 protected function getExistingInterwikis() {
727 $res = $this->mDb
->select( 'iwlinks', array( 'iwl_prefix', 'iwl_title' ),
728 array( 'iwl_from' => $this->mId
), __METHOD__
, $this->mOptions
);
730 foreach ( $res as $row ) {
731 if ( !isset( $arr[$row->iwl_prefix
] ) ) {
732 $arr[$row->iwl_prefix
] = array();
734 $arr[$row->iwl_prefix
][$row->iwl_title
] = 1;
740 * Get an array of existing categories, with the name in the key and sort key in the value.
744 private function getExistingProperties() {
745 $res = $this->mDb
->select( 'page_props', array( 'pp_propname', 'pp_value' ),
746 array( 'pp_page' => $this->mId
), __METHOD__
, $this->mOptions
);
748 foreach ( $res as $row ) {
749 $arr[$row->pp_propname
] = $row->pp_value
;
755 * Return the title object of the page being updated
758 public function getTitle() {
759 return $this->mTitle
;
763 * Returns parser output
765 * @return ParserOutput
767 public function getParserOutput() {
768 return $this->mParserOutput
;
772 * Return the list of images used as generated by the parser
775 public function getImages() {
776 return $this->mImages
;
780 * Invalidate any necessary link lists related to page property changes
783 private function invalidateProperties( $changed ) {
784 global $wgPagePropLinkInvalidations;
786 foreach ( $changed as $name => $value ) {
787 if ( isset( $wgPagePropLinkInvalidations[$name] ) ) {
788 $inv = $wgPagePropLinkInvalidations[$name];
789 if ( !is_array( $inv ) ) {
790 $inv = array( $inv );
792 foreach ( $inv as $table ) {
793 $update = new HTMLCacheUpdate( $this->mTitle
, $table );