3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
19 use Wikimedia\Purtle\RdfWriter
;
20 use Wikimedia\Purtle\TurtleRdfWriter
;
21 use Wikimedia\Rdbms\IDatabase
;
23 require_once __DIR__
. '/Maintenance.php';
26 * Maintenance script to provide RDF representation of the recent changes in category tree.
28 * @ingroup Maintenance
31 class CategoryChangesAsRdf
extends Maintenance
{
35 const SPARQL_INSERT
= <<<SPARQL
45 const SPARQL_DELETE_INSERT
= <<<SPARQLDI
63 * Categories RDF helper.
66 private $categoriesRdf;
72 * List of processed page IDs,
73 * so we don't try to process same thing twice
76 protected $processed = [];
78 public function __construct() {
79 parent
::__construct();
81 $this->addDescription( "Generate RDF dump of category changes in a wiki." );
83 $this->setBatchSize( 200 );
84 $this->addOption( 'output', "Output file (default is stdout). Will be overwritten.", false,
86 $this->addOption( 'start', 'Starting timestamp (inclusive), in ISO or Mediawiki format.',
88 $this->addOption( 'end', 'Ending timestamp (exclusive), in ISO or Mediawiki format.', true,
93 * Initialize external service classes.
95 public function initialize() {
96 // SPARQL Update syntax is close to Turtle format, so we can use Turtle writer.
97 $this->rdfWriter
= new TurtleRdfWriter();
98 $this->categoriesRdf
= new CategoriesRdf( $this->rdfWriter
);
101 public function execute() {
106 $startTS = new MWTimestamp( $this->getOption( "start" ) );
107 $endTS = new MWTimestamp( $this->getOption( "end" ) );
108 $now = new MWTimestamp();
110 if ( $now->getTimestamp() - $startTS->getTimestamp() > $wgRCMaxAge ) {
111 $this->error( "Start timestamp too old, maximum RC age is $wgRCMaxAge!" );
113 if ( $now->getTimestamp() - $endTS->getTimestamp() > $wgRCMaxAge ) {
114 $this->error( "End timestamp too old, maximum RC age is $wgRCMaxAge!" );
117 $this->startTS
= $startTS->getTimestamp();
118 $this->endTS
= $endTS->getTimestamp();
120 $outFile = $this->getOption( 'output', 'php://stdout' );
121 if ( $outFile === '-' ) {
122 $outFile = 'php://stdout';
125 $output = fopen( $outFile, 'wb' );
127 $this->categoriesRdf
->setupPrefixes();
128 $this->rdfWriter
->start();
130 $prefixes = $this->getRdf();
131 // We have to strip @ from prefix, since SPARQL UPDATE doesn't use them
132 // Also strip dot at the end.
133 $prefixes = preg_replace( [ '/^@/m', '/\s*[.]$/m' ], '', $prefixes );
134 fwrite( $output, $prefixes );
136 $dbr = $this->getDB( DB_REPLICA
, [ 'vslow' ] );
138 // Deletes go first because if the page was deleted, other changes
139 // do not matter. This only gets true deletes, i.e. not pages that were restored.
140 $this->handleDeletes( $dbr, $output );
141 // Moves go before additions because if category is moved, we should not process creation
142 // as it would produce wrong data - because create row has old title
143 $this->handleMoves( $dbr, $output );
144 // We need to handle restores too since delete may have happened in previous update.
145 $this->handleRestores( $dbr, $output );
146 $this->handleAdds( $dbr, $output );
147 $this->handleChanges( $dbr, $output );
150 fwrite( $output, $this->updateTS( $this->endTS
) );
154 * Get SPARQL for updating set of categories
155 * @param IDatabase $dbr
156 * @param string[] $deleteUrls List of URIs to be deleted, with <>
157 * @param string[] $pages List of categories: id => title
158 * @param string $mark Marks which operation requests the query
159 * @return string SPARQL query
161 private function getCategoriesUpdate( IDatabase
$dbr, $deleteUrls, $pages, $mark ) {
162 if ( empty( $deleteUrls ) ) {
166 if ( !empty( $pages ) ) {
167 $this->writeParentCategories( $dbr, $pages );
170 return "# $mark\n" . sprintf( self
::SPARQL_DELETE_INSERT
,
172 implode( ' ', $deleteUrls ) );
176 * Write data for a set of categories
177 * @param IDatabase $dbr
178 * @param string[] $pages List of categories: id => title
180 private function writeParentCategories( IDatabase
$dbr, $pages ) {
181 foreach ( $this->getCategoryLinksIterator( $dbr, array_keys( $pages ) ) as $row ) {
182 $this->categoriesRdf
->writeCategoryLinkData( $pages[$row->cl_from
], $row->cl_to
);
187 * Generate SPARQL Update code for updating dump timestamp
188 * @param string|int $timestamp Timestamp for last change
189 * @return string SPARQL Update query for timestamp.
191 public function updateTS( $timestamp ) {
192 $dumpUrl = '<' . $this->categoriesRdf
->getDumpURI() . '>';
193 $ts = wfTimestamp( TS_ISO_8601
, $timestamp );
196 $dumpUrl schema:dateModified ?o .
199 $dumpUrl schema:dateModified ?o .
202 $dumpUrl schema:dateModified "$ts"^^xsd:dateTime .
210 * Set up standard iterator for retrieving category changes.
211 * @param IDatabase $dbr
212 * @param string[] $columns List of additional fields to get
213 * @param string[] $extra_tables List of additional tables to join
214 * @return BatchRowIterator
216 private function setupChangesIterator(
219 array $extra_tables = []
221 $tables = [ 'recentchanges', 'page_props', 'category' ];
222 if ( $extra_tables ) {
223 $tables = array_merge( $tables, $extra_tables );
225 $it = new BatchRowIterator( $dbr,
230 $this->addTimestampConditions( $it, $dbr );
231 $it->addJoinConditions(
234 'LEFT JOIN', [ 'pp_propname' => 'hiddencat', 'pp_page = rc_cur_id' ]
237 'LEFT JOIN', [ 'cat_title = rc_title' ]
241 $it->setFetchColumns( array_merge( $columns, [
253 * Fetch newly created categories
254 * @param IDatabase $dbr
255 * @return BatchRowIterator
257 protected function getNewCatsIterator( IDatabase
$dbr ) {
258 $it = $this->setupChangesIterator( $dbr );
259 $it->addConditions( [
260 'rc_namespace' => NS_CATEGORY
,
267 * Fetch moved categories
268 * @param IDatabase $dbr
269 * @return BatchRowIterator
271 protected function getMovedCatsIterator( IDatabase
$dbr ) {
272 $it = $this->setupChangesIterator( $dbr, [ 'page_title', 'page_namespace' ], [ 'page' ] );
273 $it->addConditions( [
274 'rc_namespace' => NS_CATEGORY
,
276 'rc_log_type' => 'move',
279 $it->addJoinConditions( [
280 'page' => [ 'INNER JOIN', 'rc_cur_id = page_id' ],
282 $this->addIndex( $it );
287 * Fetch deleted categories
288 * @param IDatabase $dbr
289 * @return BatchRowIterator
291 protected function getDeletedCatsIterator( IDatabase
$dbr ) {
292 $it = new BatchRowIterator( $dbr,
297 $this->addTimestampConditions( $it, $dbr );
298 $it->addConditions( [
299 'rc_namespace' => NS_CATEGORY
,
301 'rc_log_type' => 'delete',
302 'rc_log_action' => 'delete',
304 // We will fetch ones that do not have page record. If they do,
305 // this means they were restored, thus restoring handler will pick it up.
306 'NOT EXISTS (SELECT * FROM page WHERE page_id = rc_cur_id)',
308 $this->addIndex( $it );
309 $it->setFetchColumns( [ 'rc_cur_id', 'rc_title' ] );
314 * Fetch restored categories
315 * @param IDatabase $dbr
316 * @return BatchRowIterator
318 protected function getRestoredCatsIterator( IDatabase
$dbr ) {
319 $it = $this->setupChangesIterator( $dbr );
320 $it->addConditions( [
321 'rc_namespace' => NS_CATEGORY
,
323 'rc_log_type' => 'delete',
324 'rc_log_action' => 'restore',
326 // We will only fetch ones that have page record
327 'EXISTS (SELECT page_id FROM page WHERE page_id = rc_cur_id)',
329 $this->addIndex( $it );
334 * Fetch categorization changes
335 * @param IDatabase $dbr
336 * @return BatchRowIterator
338 protected function getChangedCatsIterator( IDatabase
$dbr ) {
339 $it = $this->setupChangesIterator( $dbr );
340 $it->addConditions( [
341 'rc_namespace' => NS_CATEGORY
,
343 'rc_type' => [ RC_EDIT
, RC_CATEGORIZE
],
345 $this->addIndex( $it );
350 * Add timestamp limits to iterator
351 * @param BatchRowIterator $it Iterator
352 * @param IDatabase $dbr
354 private function addTimestampConditions( BatchRowIterator
$it, IDatabase
$dbr ) {
355 $it->addConditions( [
356 'rc_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $this->startTS
) ),
357 'rc_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $this->endTS
) ),
362 * Need to force index, somehow on terbium the optimizer chooses wrong one
363 * @param BatchRowIterator $it
365 private function addIndex( BatchRowIterator
$it ) {
367 'USE INDEX' => [ 'recentchanges' => 'new_name_timestamp' ]
372 * Get iterator for links for categories.
373 * @param IDatabase $dbr
374 * @param array $ids List of page IDs
375 * @return Traversable
377 protected function getCategoryLinksIterator( IDatabase
$dbr, array $ids ) {
378 $it = new BatchRowIterator(
381 [ 'cl_from', 'cl_to' ],
384 $it->addConditions( [
385 'cl_type' => 'subcat',
388 $it->setFetchColumns( [ 'cl_from', 'cl_to' ] );
389 return new RecursiveIteratorIterator( $it );
393 * Get accumulated RDF.
396 public function getRdf() {
397 return $this->rdfWriter
->drain();
401 * Handle category deletes.
402 * @param IDatabase $dbr
403 * @param resource $output File to write the output
405 public function handleDeletes( IDatabase
$dbr, $output ) {
406 // This only does "true" deletes - i.e. those that the page stays deleted
407 foreach ( $this->getDeletedCatsIterator( $dbr ) as $batch ) {
409 foreach ( $batch as $row ) {
410 // This can produce duplicates, we don't care
411 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
412 $this->processed
[$row->rc_cur_id
] = true;
414 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, [], "Deletes" ) );
419 * Write category data to RDF.
420 * @param stdclass $row Database row
422 private function writeCategoryData( $row ) {
423 $this->categoriesRdf
->writeCategoryData(
425 $row->pp_propname
=== 'hiddencat',
426 (int)$row->cat_pages
- (int)$row->cat_subcats
- (int)$row->cat_files
,
427 (int)$row->cat_subcats
432 * @param IDatabase $dbr
433 * @param resource $output
435 public function handleMoves( IDatabase
$dbr, $output ) {
436 foreach ( $this->getMovedCatsIterator( $dbr ) as $batch ) {
439 foreach ( $batch as $row ) {
440 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
442 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
443 // We already captured this one before
447 if ( $row->page_namespace
!= NS_CATEGORY
) {
448 // If page was moved out of Category:, we'll just delete
451 $row->rc_title
= $row->page_title
;
452 $this->writeCategoryData( $row );
453 $pages[$row->rc_cur_id
] = $row->page_title
;
454 $this->processed
[$row->rc_cur_id
] = true;
457 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages, "Moves" ) );
462 * @param IDatabase $dbr
463 * @param resource $output
465 public function handleRestores( IDatabase
$dbr, $output ) {
466 fwrite( $output, "# Restores\n" );
467 // This will only find those restores that were not deleted later.
468 foreach ( $this->getRestoredCatsIterator( $dbr ) as $batch ) {
470 foreach ( $batch as $row ) {
471 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
472 // We already captured this one before
475 $this->writeCategoryData( $row );
476 $pages[$row->rc_cur_id
] = $row->rc_title
;
477 $this->processed
[$row->rc_cur_id
] = true;
480 if ( empty( $pages ) ) {
484 $this->writeParentCategories( $dbr, $pages );
486 fwrite( $output, sprintf( self
::SPARQL_INSERT
, $this->getRdf() ) );
491 * @param IDatabase $dbr
492 * @param resource $output
494 public function handleAdds( IDatabase
$dbr, $output ) {
495 fwrite( $output, "# Additions\n" );
496 foreach ( $this->getNewCatsIterator( $dbr ) as $batch ) {
498 foreach ( $batch as $row ) {
499 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
500 // We already captured this one before
503 $this->writeCategoryData( $row );
504 $pages[$row->rc_cur_id
] = $row->rc_title
;
505 $this->processed
[$row->rc_cur_id
] = true;
508 if ( empty( $pages ) ) {
512 $this->writeParentCategories( $dbr, $pages );
513 fwrite( $output, sprintf( self
::SPARQL_INSERT
, $this->getRdf() ) );
518 * @param IDatabase $dbr
519 * @param resource $output
521 public function handleChanges( IDatabase
$dbr, $output ) {
522 foreach ( $this->getChangedCatsIterator( $dbr ) as $batch ) {
525 foreach ( $batch as $row ) {
526 if ( isset( $this->processed
[$row->rc_cur_id
] ) ) {
527 // We already captured this one before
530 $this->writeCategoryData( $row );
531 $pages[$row->rc_cur_id
] = $row->rc_title
;
532 $this->processed
[$row->rc_cur_id
] = true;
533 $deleteUrls[] = '<' . $this->categoriesRdf
->labelToUrl( $row->rc_title
) . '>';
536 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages, "Changes" ) );
541 $maintClass = CategoryChangesAsRdf
::class;
542 require_once RUN_MAINTENANCE_IF_MAIN
;