3 * Base class for exporting
5 * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
30 use MediaWiki\MediaWikiServices
as MediaWikiServicesAlias
;
31 use MediaWiki\Revision\RevisionRecord
;
32 use Wikimedia\Rdbms\IResultWrapper
;
33 use Wikimedia\Rdbms\IDatabase
;
36 * @ingroup SpecialPage Dump
39 /** @var bool Return distinct author list (when not returning full history) */
40 public $list_authors = false;
43 public $dumpUploads = false;
46 public $dumpUploadFileContents = false;
49 public $author_list = "";
53 const STABLE
= 4; // extension defined
57 const TEXT
= XmlDumpWriter
::WRITE_CONTENT
;
58 const STUB
= XmlDumpWriter
::WRITE_STUB
;
60 const BATCH_SIZE
= 50000;
65 /** @var DumpOutput */
68 /** @var XmlDumpWriter */
77 /** @var array|null */
78 protected $limitNamespaces;
81 * Returns the default export schema version, as defined by $wgXmlDumpSchemaVersion.
84 public static function schemaVersion() {
85 global $wgXmlDumpSchemaVersion;
86 return $wgXmlDumpSchemaVersion;
90 * @param IDatabase $db
91 * @param int|array $history One of WikiExporter::FULL, WikiExporter::CURRENT,
92 * WikiExporter::RANGE or WikiExporter::STABLE, or an associative array:
93 * - offset: non-inclusive offset at which to start the query
94 * - limit: maximum number of rows to return
95 * - dir: "asc" or "desc" timestamp order
96 * @param int $text One of WikiExporter::TEXT or WikiExporter::STUB
97 * @param null|array $limitNamespaces Comma-separated list of namespace numbers
100 function __construct(
102 $history = self
::CURRENT
,
104 $limitNamespaces = null
107 $this->history
= $history;
108 $this->writer
= new XmlDumpWriter( $text, self
::schemaVersion() );
109 $this->sink
= new DumpOutput();
111 $this->limitNamespaces
= $limitNamespaces;
115 * @param string $schemaVersion which schema version the generated XML should comply to.
116 * One of the values from self::$supportedSchemas, using the XML_DUMP_SCHEMA_VERSION_XX
119 public function setSchemaVersion( $schemaVersion ) {
120 $this->writer
= new XmlDumpWriter( $this->text
, $schemaVersion );
124 * Set the DumpOutput or DumpFilter object which will receive
125 * various row objects and XML output for filtering. Filters
126 * can be chained or used as callbacks.
128 * @param DumpOutput|DumpFilter &$sink
130 public function setOutputSink( &$sink ) {
131 $this->sink
=& $sink;
134 public function openStream() {
135 $output = $this->writer
->openStream();
136 $this->sink
->writeOpenStream( $output );
139 public function closeStream() {
140 $output = $this->writer
->closeStream();
141 $this->sink
->writeCloseStream( $output );
145 * Dumps a series of page and revision records for all pages
146 * in the database, either including complete history or only
147 * the most recent version.
149 public function allPages() {
150 $this->dumpFrom( '' );
154 * Dumps a series of page and revision records for those pages
155 * in the database falling within the page_id range given.
156 * @param int $start Inclusive lower limit (this id is included)
157 * @param int $end Exclusive upper limit (this id is not included)
158 * If 0, no upper limit.
159 * @param bool $orderRevs order revisions within pages in ascending order
161 public function pagesByRange( $start, $end, $orderRevs ) {
163 $condition = 'rev_page >= ' . intval( $start );
165 $condition .= ' AND rev_page < ' . intval( $end );
168 $condition = 'page_id >= ' . intval( $start );
170 $condition .= ' AND page_id < ' . intval( $end );
173 $this->dumpFrom( $condition, $orderRevs );
177 * Dumps a series of page and revision records for those pages
178 * in the database with revisions falling within the rev_id range given.
179 * @param int $start Inclusive lower limit (this id is included)
180 * @param int $end Exclusive upper limit (this id is not included)
181 * If 0, no upper limit.
183 public function revsByRange( $start, $end ) {
184 $condition = 'rev_id >= ' . intval( $start );
186 $condition .= ' AND rev_id < ' . intval( $end );
188 $this->dumpFrom( $condition );
192 * @param Title $title
194 public function pageByTitle( $title ) {
196 'page_namespace=' . $title->getNamespace() .
197 ' AND page_title=' . $this->db
->addQuotes( $title->getDBkey() ) );
201 * @param string $name
202 * @throws MWException
204 public function pageByName( $name ) {
205 $title = Title
::newFromText( $name );
206 if ( is_null( $title ) ) {
207 throw new MWException( "Can't export invalid title" );
209 $this->pageByTitle( $title );
214 * @param array $names
216 public function pagesByName( $names ) {
217 foreach ( $names as $name ) {
218 $this->pageByName( $name );
222 public function allLogs() {
223 $this->dumpFrom( '' );
230 public function logsByRange( $start, $end ) {
231 $condition = 'log_id >= ' . intval( $start );
233 $condition .= ' AND log_id < ' . intval( $end );
235 $this->dumpFrom( $condition );
239 * Generates the distinct list of authors of an article
240 * Not called by default (depends on $this->list_authors)
241 * Can be set by Special:Export when not exporting whole history
243 * @param string $cond
245 protected function do_list_authors( $cond ) {
246 $this->author_list
= "<contributors>";
249 $revQuery = Revision
::getQueryInfo( [ 'page' ] );
250 $res = $this->db
->select(
253 'rev_user_text' => $revQuery['fields']['rev_user_text'],
254 'rev_user' => $revQuery['fields']['rev_user'],
257 $this->db
->bitAnd( 'rev_deleted', RevisionRecord
::DELETED_USER
) . ' = 0',
265 foreach ( $res as $row ) {
266 $this->author_list
.= "<contributor>" .
268 htmlspecialchars( $row->rev_user_text
) .
271 ( (int)$row->rev_user
) .
275 $this->author_list
.= "</contributors>";
279 * @param string $cond
280 * @param bool $orderRevs
281 * @throws MWException
284 protected function dumpFrom( $cond = '', $orderRevs = false ) {
285 if ( $this->history
& self
::LOGS
) {
286 $this->dumpLogs( $cond );
288 $this->dumpPages( $cond, $orderRevs );
293 * @param string $cond
296 protected function dumpLogs( $cond ) {
299 $hideLogs = LogEventsList
::getExcludeClause( $this->db
);
301 $where[] = $hideLogs;
303 # Add on any caller specified conditions
307 $result = null; // Assuring $result is not undefined, if exception occurs early
309 $commentQuery = CommentStore
::getStore()->getJoin( 'log_comment' );
310 $actorQuery = ActorMigration
::newMigration()->getJoin( 'log_user' );
312 $tables = array_merge(
313 [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ]
316 'log_id', 'log_type', 'log_action', 'log_timestamp', 'log_namespace',
317 'log_title', 'log_params', 'log_deleted', 'user_name'
318 ] +
$commentQuery['fields'] +
$actorQuery['fields'];
320 'ORDER BY' => 'log_id',
321 'USE INDEX' => [ 'logging' => 'PRIMARY' ],
322 'LIMIT' => self
::BATCH_SIZE
,
325 'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
326 ] +
$commentQuery['joins'] +
$actorQuery['joins'];
330 $result = $this->db
->select(
333 array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
339 if ( !$result->numRows() ) {
343 $lastLogId = $this->outputLogStream( $result );
348 * @param string $cond
349 * @param bool $orderRevs
350 * @throws MWException
353 protected function dumpPages( $cond, $orderRevs ) {
354 global $wgMultiContentRevisionSchemaMigrationStage;
355 if ( !( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_OLD
) ) {
356 // TODO: Make XmlDumpWriter use a RevisionStore! (see T198706 and T174031)
357 throw new MWException(
358 'Cannot use WikiExporter with SCHEMA_COMPAT_WRITE_OLD mode disabled!'
359 . ' Support for dumping from the new schema is not implemented yet!'
363 $revQuery = MediaWikiServicesAlias
::getInstance()->getRevisionStore()->getQueryInfo(
366 $slotQuery = MediaWikiServicesAlias
::getInstance()->getRevisionStore()->getSlotsQueryInfo(
370 // We want page primary rather than revision.
371 // We also want to join in the slots and content tables.
372 // NOTE: This means we may get multiple rows per revision, and more rows
373 // than the batch size! Should be ok, since the max number of slots is
374 // fixed and low (dozens at worst).
375 $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
376 $tables = array_merge( $tables, array_diff( $slotQuery['tables'], $tables ) );
377 $join = $revQuery['joins'] +
[
378 'revision' => $revQuery['joins']['page'],
379 'slots' => [ 'JOIN', [ 'slot_revision_id = rev_id' ] ],
380 'content' => [ 'JOIN', [ 'content_id = slot_content_id' ] ],
382 unset( $join['page'] );
384 $fields = array_merge( $revQuery['fields'], $slotQuery['fields'] );
385 $fields[] = 'page_restrictions';
387 if ( $this->text
!= self
::STUB
) {
388 $fields['_load_content'] = '1';
392 if ( $cond !== '' ) {
395 $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
396 $opts['USE INDEX'] = [];
399 if ( is_array( $this->history
) ) {
400 # Time offset/limit for all pages/history...
402 if ( $this->history
['dir'] == 'asc' ) {
403 $opts['ORDER BY'] = 'rev_timestamp ASC';
406 $opts['ORDER BY'] = 'rev_timestamp DESC';
409 if ( !empty( $this->history
['offset'] ) ) {
410 $conds[] = "rev_timestamp $op " .
411 $this->db
->addQuotes( $this->db
->timestamp( $this->history
['offset'] ) );
414 if ( !empty( $this->history
['limit'] ) ) {
415 $maxRowCount = intval( $this->history
['limit'] );
417 } elseif ( $this->history
& self
::FULL
) {
418 # Full history dumps...
419 # query optimization for history stub dumps
420 if ( $this->text
== self
::STUB
) {
421 $opts[] = 'STRAIGHT_JOIN';
422 $opts['USE INDEX']['revision'] = 'rev_page_id';
423 unset( $join['revision'] );
424 $join['page'] = [ 'JOIN', 'rev_page=page_id' ];
426 } elseif ( $this->history
& self
::CURRENT
) {
427 # Latest revision dumps...
428 if ( $this->list_authors
&& $cond != '' ) { // List authors, if so desired
429 $this->do_list_authors( $cond );
431 $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
432 $opts[ 'ORDER BY' ] = [ 'page_id ASC' ];
433 } elseif ( $this->history
& self
::STABLE
) {
434 # "Stable" revision dumps...
435 # Default JOIN, to be overridden...
436 $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
437 # One, and only one hook should set this, and return false
438 if ( Hooks
::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
439 throw new MWException( __METHOD__
. " given invalid history dump type." );
441 } elseif ( $this->history
& self
::RANGE
) {
442 # Dump of revisions within a specified range. Condition already set in revsByRange().
444 # Unknown history specification parameter?
445 throw new MWException( __METHOD__
. " given invalid history dump type." );
448 $result = null; // Assuring $result is not undefined, if exception occurs early
455 $opts['LIMIT'] = self
::BATCH_SIZE
;
457 Hooks
::run( 'ModifyExportQuery',
458 [ $this->db
, &$tables, &$cond, &$opts, &$join ] );
461 // If necessary, impose the overall maximum and stop looping after this iteration.
462 if ( !empty( $maxRowCount ) && $rowCount + self
::BATCH_SIZE
> $maxRowCount ) {
463 $opts['LIMIT'] = $maxRowCount - $rowCount;
467 $queryConds = $conds;
468 $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
469 intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
471 # Do the query and process any results, remembering max ids for the next iteration.
472 $result = $this->db
->select(
480 if ( $result->numRows() > 0 ) {
481 $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
482 $rowCount +
= $result->numRows();
483 $revPage = $lastRow->rev_page
;
484 $revId = $lastRow->rev_id
;
489 // If we are finished, close off final page element (if any).
490 if ( $done && $lastRow ) {
491 $this->finishPageStreamOutput( $lastRow );
497 * Runs through a query result set dumping page, revision, and slot records.
498 * The result set should join the page, revision, slots, and content tables,
499 * and be sorted/grouped by page and revision to avoid duplicate page records in the output.
501 * @param IResultWrapper $results
502 * @param object $lastRow the last row output from the previous call (or null if none)
503 * @return object the last row processed
505 protected function outputPageStreamBatch( $results, $lastRow ) {
508 $slotRows = $this->getSlotRowBatch( $results, $rowCarry );
514 // All revision info is present in all slot rows.
515 // Use the first slot row as the revision row.
516 $revRow = $slotRows[0];
518 if ( $this->limitNamespaces
&&
519 !in_array( $revRow->page_namespace
, $this->limitNamespaces
) ) {
524 if ( $lastRow === null ||
525 $lastRow->page_namespace
!== $revRow->page_namespace ||
526 $lastRow->page_title
!== $revRow->page_title
) {
527 if ( $lastRow !== null ) {
529 if ( $this->dumpUploads
) {
530 $output .= $this->writer
->writeUploads( $lastRow, $this->dumpUploadFileContents
);
532 $output .= $this->writer
->closePage();
533 $this->sink
->writeClosePage( $output );
535 $output = $this->writer
->openPage( $revRow );
536 $this->sink
->writeOpenPage( $revRow, $output );
538 $output = $this->writer
->writeRevision( $revRow, $slotRows );
539 $this->sink
->writeRevision( $revRow, $output );
544 throw new LogicException( 'Error while processing a stream of slot rows' );
551 * Returns all slot rows for a revision.
552 * Takes and returns a carry row from the last batch;
554 * @param IResultWrapper|array $results
555 * @param null|object &$carry A row carried over from the last call to getSlotRowBatch()
559 protected function getSlotRowBatch( $results, &$carry = null ) {
564 $slotRows[] = $carry;
569 while ( $row = $results->fetchObject() ) {
570 if ( $prev && $prev->rev_id
!== $row->rev_id
) {
582 * Final page stream output, after all batches are complete
584 * @param object $lastRow the last row output from the last batch (or null if none)
586 protected function finishPageStreamOutput( $lastRow ) {
588 if ( $this->dumpUploads
) {
589 $output .= $this->writer
->writeUploads( $lastRow, $this->dumpUploadFileContents
);
591 $output .= $this->author_list
;
592 $output .= $this->writer
->closePage();
593 $this->sink
->writeClosePage( $output );
597 * @param IResultWrapper $resultset
598 * @return int the log_id value of the last item output, or null if none
600 protected function outputLogStream( $resultset ) {
601 foreach ( $resultset as $row ) {
602 $output = $this->writer
->writeLogItem( $row );
603 $this->sink
->writeLogItem( $row, $output );
605 return isset( $row ) ?
$row->log_id
: null;