3 * Updater for link tracking tables after a page edit.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
22 use MediaWiki\MediaWikiServices
;
23 use Wikimedia\Rdbms\LBFactory
;
26 * Job to add recent change entries mentioning category membership changes
28 * This allows users to easily scan categories for recent page membership changes
32 * - revTimestamp : timestamp of the triggering revision
34 * Category changes will be mentioned for revisions at/after the timestamp for this page
38 class CategoryMembershipChangeJob
extends Job
{
42 const ENQUEUE_FUDGE_SEC
= 60;
50 * @param Title $title The title of the page for which to update category membership.
51 * @param string $revisionTimestamp The timestamp of the new revision that triggered the job.
52 * @return JobSpecification
54 public static function newSpec( Title
$title, $revisionTimestamp ) {
55 return new JobSpecification(
56 'categoryMembershipChange',
58 'pageId' => $title->getArticleID(),
59 'revTimestamp' => $revisionTimestamp,
67 * Constructor for use by the Job Queue infrastructure.
68 * @note Don't call this when queueing a new instance, use newSpec() instead.
69 * @param ParserCache $parserCache Cache outputs of PHP parser.
70 * @param Title $title Title of the categorized page.
71 * @param array $params Such latest revision instance of the categorized page.
73 public function __construct( ParserCache
$parserCache, Title
$title, array $params ) {
74 parent
::__construct( 'categoryMembershipChange', $title, $params );
75 // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
76 // older revision job gets inserted while the newer revision job is de-duplicated.
77 $this->removeDuplicates
= true;
78 $this->parserCache
= $parserCache;
81 public function run() {
82 $lbFactory = MediaWikiServices
::getInstance()->getDBLoadBalancerFactory();
83 $lb = $lbFactory->getMainLB();
84 $dbw = $lb->getConnection( DB_MASTER
);
86 $this->ticket
= $lbFactory->getEmptyTransactionTicket( __METHOD__
);
88 $page = WikiPage
::newFromID( $this->params
['pageId'], WikiPage
::READ_LATEST
);
90 $this->setLastError( "Could not find page #{$this->params['pageId']}" );
91 return false; // deleted?
94 // Cut down on the time spent in safeWaitForMasterPos() in the critical section
95 $dbr = $lb->getConnection( DB_REPLICA
, [ 'recentchanges' ] );
96 if ( !$lb->safeWaitForMasterPos( $dbr ) ) {
97 $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
101 // Use a named lock so that jobs for this page see each others' changes
102 $lockKey = "CategoryMembershipUpdates:{$page->getId()}";
103 $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__
, 3 );
104 if ( !$scopedLock ) {
105 $this->setLastError( "Could not acquire lock '$lockKey'" );
109 // Wait till replica DB is caught up so that jobs for this page see each others' changes
110 if ( !$lb->safeWaitForMasterPos( $dbr ) ) {
111 $this->setLastError( "Timed out while waiting for replica DB to catch up" );
114 // Clear any stale REPEATABLE-READ snapshot
115 $dbr->flushSnapshot( __METHOD__
);
117 $cutoffUnix = wfTimestamp( TS_UNIX
, $this->params
['revTimestamp'] );
118 // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
119 // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
120 $cutoffUnix -= self
::ENQUEUE_FUDGE_SEC
;
122 // Get the newest page revision that has a SRC_CATEGORIZE row.
123 // Assume that category changes before it were already handled.
124 $row = $dbr->selectRow(
126 [ 'rev_timestamp', 'rev_id' ],
128 'rev_page' => $page->getId(),
129 'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
130 'EXISTS (' . $dbr->selectSQLText(
134 'rc_this_oldid = rev_id',
135 'rc_source' => RecentChange
::SRC_CATEGORIZE
,
136 // Allow rc_cur_id or rc_timestamp index usage
137 'rc_cur_id = rev_page',
138 'rc_timestamp = rev_timestamp'
143 [ 'ORDER BY' => 'rev_timestamp DESC, rev_id DESC' ]
145 // Only consider revisions newer than any such revision
147 $cutoffUnix = wfTimestamp( TS_UNIX
, $row->rev_timestamp
);
148 $lastRevId = (int)$row->rev_id
;
153 // Find revisions to this page made around and after this revision which lack category
154 // notifications in recent changes. This lets jobs pick up were the last one left off.
155 $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
156 $revQuery = Revision
::getQueryInfo();
161 'rev_page' => $page->getId(),
162 "rev_timestamp > $encCutoff" .
163 " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
166 [ 'ORDER BY' => 'rev_timestamp ASC, rev_id ASC' ],
170 // Apply all category updates in revision timestamp order
171 foreach ( $res as $row ) {
172 $this->notifyUpdatesForRevision( $lbFactory, $page, Revision
::newFromRow( $row ) );
179 * @param LBFactory $lbFactory
180 * @param WikiPage $page
181 * @param Revision $newRev
182 * @throws MWException
184 protected function notifyUpdatesForRevision(
185 LBFactory
$lbFactory, WikiPage
$page, Revision
$newRev
187 $config = RequestContext
::getMain()->getConfig();
188 $title = $page->getTitle();
190 // Get the new revision
191 if ( !$newRev->getContent() ) {
195 // Get the prior revision (the same for null edits)
196 if ( $newRev->getParentId() ) {
197 $oldRev = Revision
::newFromId( $newRev->getParentId(), Revision
::READ_LATEST
);
198 if ( !$oldRev ||
!$oldRev->getContent() ) {
205 // Parse the new revision and get the categories
206 $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
207 list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
208 if ( !$categoryInserts && !$categoryDeletes ) {
209 return; // nothing to do
212 $catMembChange = new CategoryMembershipChange( $title, $newRev );
213 $catMembChange->checkTemplateLinks();
215 $batchSize = $config->get( 'UpdateRowsPerQuery' );
218 foreach ( $categoryInserts as $categoryName ) {
219 $categoryTitle = Title
::makeTitle( NS_CATEGORY
, $categoryName );
220 $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
221 if ( $insertCount++
&& ( $insertCount %
$batchSize ) == 0 ) {
222 $lbFactory->commitAndWaitForReplication( __METHOD__
, $this->ticket
);
226 foreach ( $categoryDeletes as $categoryName ) {
227 $categoryTitle = Title
::makeTitle( NS_CATEGORY
, $categoryName );
228 $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
229 if ( $insertCount++
&& ( $insertCount++ %
$batchSize ) == 0 ) {
230 $lbFactory->commitAndWaitForReplication( __METHOD__
, $this->ticket
);
235 private function getExplicitCategoriesChanges(
236 WikiPage
$page, Revision
$newRev, Revision
$oldRev = null
238 // Inject the same timestamp for both revision parses to avoid seeing category changes
239 // due to time-based parser functions. Inject the same page title for the parses too.
240 // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
241 $parseTimestamp = $newRev->getTimestamp();
242 // Parse the old rev and get the categories. Do not use link tables as that
243 // assumes these updates are perfectly FIFO and that link tables are always
244 // up to date, neither of which are true.
245 $oldCategories = $oldRev
246 ?
$this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
248 // Parse the new revision and get the categories
249 $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
251 $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
252 $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
254 return [ $categoryInserts, $categoryDeletes ];
258 * @param WikiPage $page
259 * @param Revision $rev
260 * @param string $parseTimestamp TS_MW
262 * @return string[] category names
264 private function getCategoriesAtRev( WikiPage
$page, Revision
$rev, $parseTimestamp ) {
265 $renderer = MediaWikiServices
::getInstance()->getRevisionRenderer();
266 $options = $page->makeParserOptions( 'canonical' );
267 $options->setTimestamp( $parseTimestamp );
269 $output = $rev->isCurrent() ?
$this->parserCache
->get( $page, $options ) : null;
271 if ( !$output ||
$output->getCacheRevisionId() !== $rev->getId() ) {
272 $output = $renderer->getRenderedRevision( $rev->getRevisionRecord(), $options )
273 ->getRevisionParserOutput();
276 // array keys will cast numeric category names to ints
277 // so we need to cast them back to strings to avoid breaking things!
278 return array_map( 'strval', array_keys( $output->getCategories() ) );
281 public function getDeduplicationInfo() {
282 $info = parent
::getDeduplicationInfo();
283 unset( $info['params']['revTimestamp'] ); // first job wins