3 * Base class for exporting
5 * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
30 use Wikimedia\Rdbms\ResultWrapper
;
31 use Wikimedia\Rdbms\IDatabase
;
34 * @ingroup SpecialPage Dump
37 /** @var bool Return distinct author list (when not returning full history) */
38 public $list_authors = false;
41 public $dumpUploads = false;
44 public $dumpUploadFileContents = false;
47 public $author_list = "";
51 const STABLE
= 4; // extension defined
58 const BATCH_SIZE
= 50000;
63 /** @var DumpOutput */
67 * Returns the export schema version.
70 public static function schemaVersion() {
75 * @param IDatabase $db
76 * @param int|array $history One of WikiExporter::FULL, WikiExporter::CURRENT,
77 * WikiExporter::RANGE or WikiExporter::STABLE, or an associative array:
78 * - offset: non-inclusive offset at which to start the query
79 * - limit: maximum number of rows to return
80 * - dir: "asc" or "desc" timestamp order
81 * @param int $text One of WikiExporter::TEXT or WikiExporter::STUB
83 function __construct( $db, $history = self
::CURRENT
, $text = self
::TEXT
) {
85 $this->history
= $history;
86 $this->writer
= new XmlDumpWriter();
87 $this->sink
= new DumpOutput();
92 * Set the DumpOutput or DumpFilter object which will receive
93 * various row objects and XML output for filtering. Filters
94 * can be chained or used as callbacks.
96 * @param DumpOutput &$sink
98 public function setOutputSink( &$sink ) {
102 public function openStream() {
103 $output = $this->writer
->openStream();
104 $this->sink
->writeOpenStream( $output );
107 public function closeStream() {
108 $output = $this->writer
->closeStream();
109 $this->sink
->writeCloseStream( $output );
113 * Dumps a series of page and revision records for all pages
114 * in the database, either including complete history or only
115 * the most recent version.
117 public function allPages() {
118 $this->dumpFrom( '' );
122 * Dumps a series of page and revision records for those pages
123 * in the database falling within the page_id range given.
124 * @param int $start Inclusive lower limit (this id is included)
125 * @param int $end Exclusive upper limit (this id is not included)
126 * If 0, no upper limit.
127 * @param bool $orderRevs order revisions within pages in ascending order
129 public function pagesByRange( $start, $end, $orderRevs ) {
131 $condition = 'rev_page >= ' . intval( $start );
133 $condition .= ' AND rev_page < ' . intval( $end );
136 $condition = 'page_id >= ' . intval( $start );
138 $condition .= ' AND page_id < ' . intval( $end );
141 $this->dumpFrom( $condition, $orderRevs );
145 * Dumps a series of page and revision records for those pages
146 * in the database with revisions falling within the rev_id range given.
147 * @param int $start Inclusive lower limit (this id is included)
148 * @param int $end Exclusive upper limit (this id is not included)
149 * If 0, no upper limit.
151 public function revsByRange( $start, $end ) {
152 $condition = 'rev_id >= ' . intval( $start );
154 $condition .= ' AND rev_id < ' . intval( $end );
156 $this->dumpFrom( $condition );
160 * @param Title $title
162 public function pageByTitle( $title ) {
164 'page_namespace=' . $title->getNamespace() .
165 ' AND page_title=' . $this->db
->addQuotes( $title->getDBkey() ) );
169 * @param string $name
170 * @throws MWException
172 public function pageByName( $name ) {
173 $title = Title
::newFromText( $name );
174 if ( is_null( $title ) ) {
175 throw new MWException( "Can't export invalid title" );
177 $this->pageByTitle( $title );
182 * @param array $names
184 public function pagesByName( $names ) {
185 foreach ( $names as $name ) {
186 $this->pageByName( $name );
190 public function allLogs() {
191 $this->dumpFrom( '' );
198 public function logsByRange( $start, $end ) {
199 $condition = 'log_id >= ' . intval( $start );
201 $condition .= ' AND log_id < ' . intval( $end );
203 $this->dumpFrom( $condition );
207 * Generates the distinct list of authors of an article
208 * Not called by default (depends on $this->list_authors)
209 * Can be set by Special:Export when not exporting whole history
213 protected function do_list_authors( $cond ) {
214 $this->author_list
= "<contributors>";
217 $revQuery = Revision
::getQueryInfo( [ 'page' ] );
218 $res = $this->db
->select(
221 'rev_user_text' => $revQuery['fields']['rev_user_text'],
222 'rev_user' => $revQuery['fields']['rev_user'],
225 $this->db
->bitAnd( 'rev_deleted', Revision
::DELETED_USER
) . ' = 0',
233 foreach ( $res as $row ) {
234 $this->author_list
.= "<contributor>" .
236 htmlspecialchars( $row->rev_user_text
) .
239 ( (int)$row->rev_user
) .
243 $this->author_list
.= "</contributors>";
247 * @param string $cond
248 * @param bool $orderRevs
249 * @throws MWException
252 protected function dumpFrom( $cond = '', $orderRevs = false ) {
253 if ( $this->history
& self
::LOGS
) {
254 $this->dumpLogs( $cond );
256 $this->dumpPages( $cond, $orderRevs );
261 * @param string $cond
264 protected function dumpLogs( $cond ) {
267 $hideLogs = LogEventsList
::getExcludeClause( $this->db
);
269 $where[] = $hideLogs;
271 # Add on any caller specified conditions
275 # Get logging table name for logging.* clause
276 $logging = $this->db
->tableName( 'logging' );
278 $result = null; // Assuring $result is not undefined, if exception occurs early
280 $commentQuery = CommentStore
::getStore()->getJoin( 'log_comment' );
281 $actorQuery = ActorMigration
::newMigration()->getJoin( 'log_user' );
285 $result = $this->db
->select(
286 array_merge( [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ] ),
287 [ "{$logging}.*", 'user_name' ] +
$commentQuery['fields'] +
$actorQuery['fields'],
288 array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
291 'ORDER BY' => 'log_id',
292 'USE INDEX' => [ 'logging' => 'PRIMARY' ],
293 'LIMIT' => self
::BATCH_SIZE
,
296 'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
297 ] +
$commentQuery['joins'] +
$actorQuery['joins']
300 if ( !$result->numRows() ) {
304 $lastLogId = $this->outputLogStream( $result );
309 * @param string $cond
310 * @param bool $orderRevs
311 * @throws MWException
314 protected function dumpPages( $cond, $orderRevs ) {
315 global $wgMultiContentRevisionSchemaMigrationStage;
316 if ( !( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_OLD
) ) {
317 // TODO: Make XmlDumpWriter use a RevisionStore! (see T198706 and T174031)
318 throw new MWException(
319 'Cannot use WikiExporter with SCHEMA_COMPAT_WRITE_OLD mode disabled!'
320 . ' Support for dumping from the new schema is not implemented yet!'
324 $revOpts = [ 'page' ];
325 if ( $this->text
!= self
::STUB
) {
326 // TODO: remove the text and make XmlDumpWriter use a RevisionStore instead! (T198706)
329 $revQuery = Revision
::getQueryInfo( $revOpts );
331 // We want page primary rather than revision
332 $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
333 $join = $revQuery['joins'] +
[
334 'revision' => $revQuery['joins']['page']
336 unset( $join['page'] );
338 // TODO: remove rev_text_id and make XmlDumpWriter use a RevisionStore instead! (T198706)
339 $fields = array_merge( $revQuery['fields'], [ 'page_restrictions, rev_text_id' ] );
342 if ( $cond !== '' ) {
345 $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
346 $opts['USE INDEX'] = [];
349 if ( is_array( $this->history
) ) {
350 # Time offset/limit for all pages/history...
352 if ( $this->history
['dir'] == 'asc' ) {
353 $opts['ORDER BY'] = 'rev_timestamp ASC';
356 $opts['ORDER BY'] = 'rev_timestamp DESC';
359 if ( !empty( $this->history
['offset'] ) ) {
360 $conds[] = "rev_timestamp $op " .
361 $this->db
->addQuotes( $this->db
->timestamp( $this->history
['offset'] ) );
364 if ( !empty( $this->history
['limit'] ) ) {
365 $maxRowCount = intval( $this->history
['limit'] );
367 } elseif ( $this->history
& self
::FULL
) {
368 # Full history dumps...
369 # query optimization for history stub dumps
370 if ( $this->text
== self
::STUB
) {
371 $tables = $revQuery['tables'];
372 $opts[] = 'STRAIGHT_JOIN';
373 $opts['USE INDEX']['revision'] = 'rev_page_id';
374 unset( $join['revision'] );
375 $join['page'] = [ 'INNER JOIN', 'rev_page=page_id' ];
377 } elseif ( $this->history
& self
::CURRENT
) {
378 # Latest revision dumps...
379 if ( $this->list_authors
&& $cond != '' ) { // List authors, if so desired
380 $this->do_list_authors( $cond );
382 $join['revision'] = [ 'INNER JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
383 } elseif ( $this->history
& self
::STABLE
) {
384 # "Stable" revision dumps...
385 # Default JOIN, to be overridden...
386 $join['revision'] = [ 'INNER JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
387 # One, and only one hook should set this, and return false
388 if ( Hooks
::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
389 throw new MWException( __METHOD__
. " given invalid history dump type." );
391 } elseif ( $this->history
& self
::RANGE
) {
392 # Dump of revisions within a specified range. Condition already set in revsByRange().
394 # Unknown history specification parameter?
395 throw new MWException( __METHOD__
. " given invalid history dump type." );
398 $result = null; // Assuring $result is not undefined, if exception occurs early
405 $opts['LIMIT'] = self
::BATCH_SIZE
;
407 Hooks
::run( 'ModifyExportQuery',
408 [ $this->db
, &$tables, &$cond, &$opts, &$join ] );
411 // If necessary, impose the overall maximum and stop looping after this iteration.
412 if ( !empty( $maxRowCount ) && $rowCount + self
::BATCH_SIZE
> $maxRowCount ) {
413 $opts['LIMIT'] = $maxRowCount - $rowCount;
417 $queryConds = $conds;
418 $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
419 intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
421 # Do the query and process any results, remembering max ids for the next iteration.
422 $result = $this->db
->select(
430 if ( $result->numRows() > 0 ) {
431 $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
432 $rowCount +
= $result->numRows();
433 $revPage = $lastRow->rev_page
;
434 $revId = $lastRow->rev_id
;
439 // If we are finished, close off final page element (if any).
440 if ( $done && $lastRow ) {
441 $this->finishPageStreamOutput( $lastRow );
447 * Runs through a query result set dumping page and revision records.
448 * The result set should be sorted/grouped by page to avoid duplicate
449 * page records in the output.
451 * @param ResultWrapper $results
452 * @param object $lastRow the last row output from the previous call (or null if none)
453 * @return object the last row processed
455 protected function outputPageStreamBatch( $results, $lastRow ) {
456 foreach ( $results as $row ) {
457 if ( $lastRow === null ||
458 $lastRow->page_namespace
!= $row->page_namespace ||
459 $lastRow->page_title
!= $row->page_title
) {
460 if ( $lastRow !== null ) {
462 if ( $this->dumpUploads
) {
463 $output .= $this->writer
->writeUploads( $lastRow, $this->dumpUploadFileContents
);
465 $output .= $this->writer
->closePage();
466 $this->sink
->writeClosePage( $output );
468 $output = $this->writer
->openPage( $row );
469 $this->sink
->writeOpenPage( $row, $output );
471 $output = $this->writer
->writeRevision( $row );
472 $this->sink
->writeRevision( $row, $output );
480 * Final page stream output, after all batches are complete
482 * @param object $lastRow the last row output from the last batch (or null if none)
484 protected function finishPageStreamOutput( $lastRow ) {
486 if ( $this->dumpUploads
) {
487 $output .= $this->writer
->writeUploads( $lastRow, $this->dumpUploadFileContents
);
489 $output .= $this->author_list
;
490 $output .= $this->writer
->closePage();
491 $this->sink
->writeClosePage( $output );
495 * @param ResultWrapper $resultset
496 * @return int the log_id value of the last item output, or null if none
498 protected function outputLogStream( $resultset ) {
499 foreach ( $resultset as $row ) {
500 $output = $this->writer
->writeLogItem( $row );
501 $this->sink
->writeLogItem( $row, $output );
503 return isset( $row ) ?
$row->log_id
: null;