3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
19 use Wikimedia\Purtle\RdfWriter
;
20 use Wikimedia\Purtle\TurtleRdfWriter
;
21 use Wikimedia\Rdbms\IDatabase
;
23 require_once __DIR__
. '/Maintenance.php';
26 * Maintenance script to provide RDF representation of the recent changes in category tree.
28 * @ingroup Maintenance
31 class CategoryChangesAsRdf
extends Maintenance
{
35 const SPARQL_INSERT
= <<<SPARQL
45 const SPARQL_DELETE
= <<<SPARQLD
60 const SPARQL_DELETE_INSERT
= <<<SPARQLDI
79 * Categories RDF helper.
82 private $categoriesRdf;
88 * List of processed page IDs,
89 * so we don't try to process same thing twice
92 protected $processed = [];
94 public function __construct() {
95 parent
::__construct();
97 $this->addDescription( "Generate RDF dump of category changes in a wiki." );
99 $this->setBatchSize( 200 );
100 $this->addOption( 'output', "Output file (default is stdout). Will be overwritten.", false,
102 $this->addOption( 'start', 'Starting timestamp (inclusive), in ISO or Mediawiki format.',
104 $this->addOption( 'end', 'Ending timestamp (exclusive), in ISO or Mediawiki format.', true,
109 * Initialize external service classes.
111 public function initialize() {
112 // SPARQL Update syntax is close to Turtle format, so we can use Turtle writer.
113 $this->rdfWriter
= new TurtleRdfWriter();
114 $this->categoriesRdf
= new CategoriesRdf( $this->rdfWriter
);
117 public function execute() {
119 $startTS = new MWTimestamp( $this->getOption( "start" ) );
121 $endTS = new MWTimestamp( $this->getOption( "end" ) );
122 $now = new MWTimestamp();
123 $rcMaxAge = $this->getConfig()->get( 'RCMaxAge' );
125 if ( $now->getTimestamp() - $startTS->getTimestamp() > $rcMaxAge ) {
126 $this->error( "Start timestamp too old, maximum RC age is $rcMaxAge!" );
128 if ( $now->getTimestamp() - $endTS->getTimestamp() > $rcMaxAge ) {
129 $this->error( "End timestamp too old, maximum RC age is $rcMaxAge!" );
132 $this->startTS
= $startTS->getTimestamp();
133 $this->endTS
= $endTS->getTimestamp();
135 $outFile = $this->getOption( 'output', 'php://stdout' );
136 if ( $outFile === '-' ) {
137 $outFile = 'php://stdout';
140 $output = fopen( $outFile, 'wb' );
142 $this->categoriesRdf
->setupPrefixes();
143 $this->rdfWriter
->start();
145 $prefixes = $this->getRdf();
146 // We have to strip @ from prefix, since SPARQL UPDATE doesn't use them
147 // Also strip dot at the end.
148 $prefixes = preg_replace( [ '/^@/m', '/\s*[.]$/m' ], '', $prefixes );
149 fwrite( $output, $prefixes );
151 $dbr = $this->getDB( DB_REPLICA
, [ 'vslow' ] );
153 // Deletes go first because if the page was deleted, other changes
154 // do not matter. This only gets true deletes, i.e. not pages that were restored.
155 $this->handleDeletes( $dbr, $output );
156 // Moves go before additions because if category is moved, we should not process creation
157 // as it would produce wrong data - because create row has old title
158 $this->handleMoves( $dbr, $output );
159 // We need to handle restores too since delete may have happened in previous update.
160 $this->handleRestores( $dbr, $output );
161 // Process newly added pages
162 $this->handleAdds( $dbr, $output );
163 // Process page edits
164 $this->handleEdits( $dbr, $output );
165 // Process categorization changes
166 $this->handleCategorization( $dbr, $output );
169 fwrite( $output, $this->updateTS( $this->endTS
) );
173 * Get the text of SPARQL INSERT DATA clause
176 private function getInsertRdf() {
177 $rdfText = $this->getRdf();
181 return sprintf( self
::SPARQL_INSERT
, $rdfText );
185 * Get SPARQL for updating set of categories
186 * @param IDatabase $dbr
187 * @param string[] $deleteUrls List of URIs to be deleted, with <>
188 * @param string[] $pages List of categories: id => title
189 * @param string $mark Marks which operation requests the query
190 * @return string SPARQL query
192 private function getCategoriesUpdate( IDatabase
$dbr, $deleteUrls, $pages, $mark ) {
193 if ( empty( $deleteUrls ) ) {
197 if ( !empty( $pages ) ) {
198 $this->writeParentCategories( $dbr, $pages );
201 return "# $mark\n" . sprintf( self
::SPARQL_DELETE
, implode( ' ', $deleteUrls ) ) .
202 $this->getInsertRdf();
206 * Write parent data for a set of categories.
207 * The list has the child categories.
208 * @param IDatabase $dbr
209 * @param string[] $pages List of child categories: id => title
211 private function writeParentCategories( IDatabase
$dbr, $pages ) {
212 foreach ( $this->getCategoryLinksIterator( $dbr, array_keys( $pages ) ) as $row ) {
213 $this->categoriesRdf
->writeCategoryLinkData( $pages[$row->cl_from
], $row->cl_to
);
218 * Generate SPARQL Update code for updating dump timestamp
219 * @param string|int $timestamp Timestamp for last change
220 * @return string SPARQL Update query for timestamp.
222 public function updateTS( $timestamp ) {
223 $dumpUrl = '<' . $this->categoriesRdf
->getDumpURI() . '>';
224 $ts = wfTimestamp( TS_ISO_8601
, $timestamp );
227 $dumpUrl schema:dateModified ?o .
230 $dumpUrl schema:dateModified ?o .
233 $dumpUrl schema:dateModified "$ts"^^xsd:dateTime .
241 * Set up standard iterator for retrieving category changes.
242 * @param IDatabase $dbr
243 * @param string[] $columns List of additional fields to get
244 * @param string[] $extra_tables List of additional tables to join
245 * @return BatchRowIterator
247 private function setupChangesIterator(
250 array $extra_tables = []
252 $tables = [ 'recentchanges', 'page_props', 'category' ];
253 if ( $extra_tables ) {
254 $tables = array_merge( $tables, $extra_tables );
256 $it = new BatchRowIterator( $dbr,
261 $this->addTimestampConditions( $it, $dbr );
262 $it->addJoinConditions(
265 'LEFT JOIN', [ 'pp_propname' => 'hiddencat', 'pp_page = rc_cur_id' ]
268 'LEFT JOIN', [ 'cat_title = rc_title' ]
272 $it->setFetchColumns( array_merge( $columns, [
284 * Fetch newly created categories
285 * @param IDatabase $dbr
286 * @return BatchRowIterator
288 protected function getNewCatsIterator( IDatabase
$dbr ) {
289 $it = $this->setupChangesIterator( $dbr );
290 $it->addConditions( [
291 'rc_namespace' => NS_CATEGORY
,
298 * Fetch moved categories
299 * @param IDatabase $dbr
300 * @return BatchRowIterator
302 protected function getMovedCatsIterator( IDatabase
$dbr ) {
303 $it = $this->setupChangesIterator( $dbr, [ 'page_title', 'page_namespace' ], [ 'page' ] );
304 $it->addConditions( [
305 'rc_namespace' => NS_CATEGORY
,
307 'rc_log_type' => 'move',
310 $it->addJoinConditions( [
311 'page' => [ 'JOIN', 'rc_cur_id = page_id' ],
313 $this->addIndex( $it );
318 * Fetch deleted categories
319 * @param IDatabase $dbr
320 * @return BatchRowIterator
322 protected function getDeletedCatsIterator( IDatabase
$dbr ) {
323 $it = new BatchRowIterator( $dbr,
328 $this->addTimestampConditions( $it, $dbr );
329 $it->addConditions( [
330 'rc_namespace' => NS_CATEGORY
,
332 'rc_log_type' => 'delete',
333 'rc_log_action' => 'delete',
335 // We will fetch ones that do not have page record. If they do,
336 // this means they were restored, thus restoring handler will pick it up.
337 'NOT EXISTS (SELECT * FROM page WHERE page_id = rc_cur_id)',
339 $this->addIndex( $it );
340 $it->setFetchColumns( [ 'rc_cur_id', 'rc_title' ] );
345 * Fetch restored categories
346 * @param IDatabase $dbr
347 * @return BatchRowIterator
349 protected function getRestoredCatsIterator( IDatabase
$dbr ) {
350 $it = $this->setupChangesIterator( $dbr );
351 $it->addConditions( [
352 'rc_namespace' => NS_CATEGORY
,
354 'rc_log_type' => 'delete',
355 'rc_log_action' => 'restore',
357 // We will only fetch ones that have page record
358 'EXISTS (SELECT page_id FROM page WHERE page_id = rc_cur_id)',
360 $this->addIndex( $it );
365 * Fetch categorization changes or edits
366 * @param IDatabase $dbr
367 * @return BatchRowIterator
369 protected function getChangedCatsIterator( IDatabase
$dbr, $type ) {
371 $this->setupChangesIterator( $dbr );
372 $it->addConditions( [
373 'rc_namespace' => NS_CATEGORY
,
377 $this->addIndex( $it );
382 * Add timestamp limits to iterator
383 * @param BatchRowIterator $it Iterator
384 * @param IDatabase $dbr
386 private function addTimestampConditions( BatchRowIterator
$it, IDatabase
$dbr ) {
387 $it->addConditions( [
388 'rc_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $this->startTS
) ),
389 'rc_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $this->endTS
) ),
394 * Need to force index, somehow on terbium the optimizer chooses wrong one
395 * @param BatchRowIterator $it
397 private function addIndex( BatchRowIterator
$it ) {
399 'USE INDEX' => [ 'recentchanges' => 'new_name_timestamp' ]
404 * Get iterator for links for categories.
405 * @param IDatabase $dbr
406 * @param int[] $ids List of page IDs
407 * @return Traversable
409 protected function getCategoryLinksIterator( IDatabase
$dbr, array $ids ) {
410 $it = new BatchRowIterator(
413 [ 'cl_from', 'cl_to' ],
416 $it->addConditions( [
417 'cl_type' => 'subcat',
420 $it->setFetchColumns( [ 'cl_from', 'cl_to' ] );
421 return new RecursiveIteratorIterator( $it );
425 * Get accumulated RDF.
428 public function getRdf() {
429 return $this->rdfWriter
->drain();
433 * Handle category deletes.
434 * @param IDatabase $dbr
435 * @param resource $output File to write the output
437 public function handleDeletes( IDatabase
$dbr, $output ) {
438 // This only does "true" deletes - i.e. those that the page stays deleted
439 foreach ( $this->getDeletedCatsIterator( $dbr ) as $batch ) {
441 foreach ( $batch as $row ) {
442 // This can produce duplicates, we don't care
443 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
444 $this->processed
[$row->rc_cur_id
] = true;
446 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, [], "Deletes" ) );
451 * Write category data to RDF.
452 * @param stdclass $row Database row
454 private function writeCategoryData( $row ) {
455 $this->categoriesRdf
->writeCategoryData(
457 $row->pp_propname
=== 'hiddencat',
458 (int)$row->cat_pages
- (int)$row->cat_subcats
- (int)$row->cat_files
,
459 (int)$row->cat_subcats
464 * @param IDatabase $dbr
465 * @param resource $output
467 public function handleMoves( IDatabase
$dbr, $output ) {
468 foreach ( $this->getMovedCatsIterator( $dbr ) as $batch ) {
471 foreach ( $batch as $row ) {
472 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
474 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
475 // We already captured this one before
479 if ( $row->page_namespace
!= NS_CATEGORY
) {
480 // If page was moved out of Category:, we'll just delete
483 $row->rc_title
= $row->page_title
;
484 $this->writeCategoryData( $row );
485 $pages[$row->rc_cur_id
] = $row->page_title
;
486 $this->processed
[$row->rc_cur_id
] = true;
489 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages, "Moves" ) );
494 * @param IDatabase $dbr
495 * @param resource $output
497 public function handleRestores( IDatabase
$dbr, $output ) {
498 fwrite( $output, "# Restores\n" );
499 // This will only find those restores that were not deleted later.
500 foreach ( $this->getRestoredCatsIterator( $dbr ) as $batch ) {
502 foreach ( $batch as $row ) {
503 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
504 // We already captured this one before
507 $this->writeCategoryData( $row );
508 $pages[$row->rc_cur_id
] = $row->rc_title
;
509 $this->processed
[$row->rc_cur_id
] = true;
512 if ( empty( $pages ) ) {
516 $this->writeParentCategories( $dbr, $pages );
518 fwrite( $output, $this->getInsertRdf() );
523 * @param IDatabase $dbr
524 * @param resource $output
526 public function handleAdds( IDatabase
$dbr, $output ) {
527 fwrite( $output, "# Additions\n" );
528 foreach ( $this->getNewCatsIterator( $dbr ) as $batch ) {
530 foreach ( $batch as $row ) {
531 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
532 // We already captured this one before
535 $this->writeCategoryData( $row );
536 $pages[$row->rc_cur_id
] = $row->rc_title
;
537 $this->processed
[$row->rc_cur_id
] = true;
540 if ( empty( $pages ) ) {
544 $this->writeParentCategories( $dbr, $pages );
545 fwrite( $output, $this->getInsertRdf() );
550 * Handle edits for category texts
551 * @param IDatabase $dbr
552 * @param resource $output
554 public function handleEdits( IDatabase
$dbr, $output ) {
555 // Editing category can change hidden flag and add new parents.
556 // TODO: it's pretty expensive to update all edited categories, and most edits
557 // aren't actually interesting for us. Some way to know which are interesting?
558 // We can capture recategorization on the next step, but not change in hidden status.
559 foreach ( $this->getChangedCatsIterator( $dbr, RC_EDIT
) as $batch ) {
562 foreach ( $batch as $row ) {
563 // Note that on categorization event, cur_id points to
564 // the child page, not the parent category!
565 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
566 // We already captured this one before
569 $this->writeCategoryData( $row );
570 $pages[$row->rc_cur_id
] = $row->rc_title
;
571 $this->processed
[$row->rc_cur_id
] = true;
572 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
575 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages, "Edits" ) );
580 * Handles categorization changes
581 * @param IDatabase $dbr
582 * @param resource $output
584 public function handleCategorization( IDatabase
$dbr, $output ) {
585 $processedTitle = [];
586 // Categorization change can add new parents and change counts
587 // for the parent category.
588 foreach ( $this->getChangedCatsIterator( $dbr, RC_CATEGORIZE
) as $batch ) {
590 * Note that on categorization event, cur_id points to
591 * the child page, not the parent category!
592 * So we need to have a two-stage process, since we have ID from one
593 * category and title from another, and we need both for proper updates.
594 * TODO: For now, we do full update even though some data hasn't changed,
595 * e.g. parents for parent cat and counts for child cat.
599 foreach ( $batch as $row ) {
600 $childPages[$row->rc_cur_id
] = true;
601 $parentCats[$row->rc_title
] = true;
607 [ 'pp_propname' => 'hiddencat', 'pp_page = page_id' ],
611 [ 'cat_title = page_title' ],
619 // Load child rows by ID
620 $childRows = $dbr->select(
621 [ 'page', 'page_props', 'category' ],
624 'rc_title' => 'page_title',
630 [ 'page_namespace' => NS_CATEGORY
, 'page_id' => array_keys( $childPages ) ],
635 foreach ( $childRows as $row ) {
636 if ( isset( $this->processed
[$row->page_id
] ) ) {
637 // We already captured this one before
640 $this->writeCategoryData( $row );
641 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
642 $this->processed
[$row->page_id
] = true;
647 // Load parent rows by title
651 [ 'page_title = cat_title', 'page_namespace' => NS_CATEGORY
],
655 [ 'pp_propname' => 'hiddencat', 'pp_page = page_id' ],
659 $parentRows = $dbr->select(
660 [ 'category', 'page', 'page_props' ],
663 'rc_title' => 'cat_title',
669 [ 'cat_title' => array_keys( $parentCats ) ],
674 foreach ( $parentRows as $row ) {
675 if ( $row->page_id
&& isset( $this->processed
[$row->page_id
] ) ) {
676 // We already captured this one before
679 if ( isset( $processedTitle[$row->rc_title
] ) ) {
680 // We already captured this one before
683 $this->writeCategoryData( $row );
684 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
685 if ( $row->page_id
) {
686 $this->processed
[$row->page_id
] = true;
688 $processedTitle[$row->rc_title
] = true;
692 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages, "Changes" ) );
697 $maintClass = CategoryChangesAsRdf
::class;
698 require_once RUN_MAINTENANCE_IF_MAIN
;