5 * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
25 use MediaWiki\MediaWikiServices
;
26 use MediaWiki\Revision\RevisionRecord
;
27 use MediaWiki\Revision\RevisionStore
;
28 use MediaWiki\Revision\SlotRecord
;
29 use MediaWiki\Revision\SuppressedDataException
;
30 use MediaWiki\Storage\SqlBlobStore
;
31 use Wikimedia\Assert\Assert
;
38 /** Output serialized revision content. */
39 const WRITE_CONTENT
= 0;
41 /** Only output subs for revision content. */
45 * Only output subs for revision content, indicating that the content has been
46 * deleted/suppressed. For internal use only.
48 const WRITE_STUB_DELETED
= 2;
51 * @var string[] the schema versions supported for output
54 public static $supportedSchemas = [
55 XML_DUMP_SCHEMA_VERSION_10
,
56 XML_DUMP_SCHEMA_VERSION_11
60 * @var string which schema version the generated XML should comply to.
61 * One of the values from self::$supportedSchemas, using the SCHEMA_VERSION_XX
64 private $schemaVersion;
67 * Title of the currently processed page
71 private $currentTitle = null;
74 * @var int Whether to output revision content or just stubs. WRITE_CONTENT or WRITE_STUB.
79 * XmlDumpWriter constructor.
81 * @param int $contentMode WRITE_CONTENT or WRITE_STUB.
82 * @param string $schemaVersion which schema version the generated XML should comply to.
83 * One of the values from self::$supportedSchemas, using the XML_DUMP_SCHEMA_VERSION_XX
86 public function __construct(
87 $contentMode = self
::WRITE_CONTENT
,
88 $schemaVersion = XML_DUMP_SCHEMA_VERSION_11
91 in_array( $contentMode, [ self
::WRITE_CONTENT
, self
::WRITE_STUB
] ),
93 'must be one of the following constants: WRITE_CONTENT or WRITE_STUB.'
97 in_array( $schemaVersion, self
::$supportedSchemas ),
99 'must be one of the following schema versions: '
100 . implode( ',', self
::$supportedSchemas )
103 $this->contentMode
= $contentMode;
104 $this->schemaVersion
= $schemaVersion;
108 * Opens the XML output stream's root "<mediawiki>" element.
109 * This does not include an xml directive, so is safe to include
110 * as a subelement in a larger XML stream. Namespace and XML Schema
111 * references are included.
113 * Output will be encoded in UTF-8.
117 function openStream() {
118 $ver = $this->schemaVersion
;
119 return Xml
::element( 'mediawiki', [
120 'xmlns' => "http://www.mediawiki.org/xml/export-$ver/",
121 'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
123 * When a new version of the schema is created, it needs staging on mediawiki.org.
124 * This requires a change in the operations/mediawiki-config git repo.
126 * Create a changeset like https://gerrit.wikimedia.org/r/#/c/149643/ in which
127 * you copy in the new xsd file.
129 * After it is reviewed, merged and deployed (sync-docroot), the index.html needs purging.
130 * echo "https://www.mediawiki.org/xml/index.html" | mwscript purgeList.php --wiki=aawiki
132 'xsi:schemaLocation' => "http://www.mediawiki.org/xml/export-$ver/ " .
133 "http://www.mediawiki.org/xml/export-$ver.xsd",
135 'xml:lang' => MediaWikiServices
::getInstance()->getContentLanguage()->getHtmlCode() ],
144 function siteInfo() {
150 $this->caseSetting(),
151 $this->namespaces() ];
152 return " <siteinfo>\n " .
153 implode( "\n ", $info ) .
160 function sitename() {
162 return Xml
::element( 'sitename', [], $wgSitename );
170 return Xml
::element( 'dbname', [], $wgDBname );
176 function generator() {
178 return Xml
::element( 'generator', [], "MediaWiki $wgVersion" );
184 function homelink() {
185 return Xml
::element( 'base', [], Title
::newMainPage()->getCanonicalURL() );
191 function caseSetting() {
192 global $wgCapitalLinks;
193 // "case-insensitive" option is reserved for future
194 $sensitivity = $wgCapitalLinks ?
'first-letter' : 'case-sensitive';
195 return Xml
::element( 'case', [], $sensitivity );
201 function namespaces() {
202 $spaces = "<namespaces>\n";
203 $nsInfo = MediaWikiServices
::getInstance()->getNamespaceInfo();
205 MediaWikiServices
::getInstance()->getContentLanguage()->getFormattedNamespaces()
209 Xml
::element( 'namespace',
212 'case' => $nsInfo->isCapitalized( $ns )
213 ?
'first-letter' : 'case-sensitive',
216 $spaces .= " </namespaces>";
221 * Closes the output stream with the closing root element.
222 * Call when finished dumping things.
226 function closeStream() {
227 return "</mediawiki>\n";
231 * Opens a "<page>" section on the output stream, with data
232 * from the given database row.
237 public function openPage( $row ) {
239 $this->currentTitle
= Title
::newFromRow( $row );
240 $canonicalTitle = self
::canonicalTitle( $this->currentTitle
);
241 $out .= ' ' . Xml
::elementClean( 'title', [], $canonicalTitle ) . "\n";
242 $out .= ' ' . Xml
::element( 'ns', [], strval( $row->page_namespace
) ) . "\n";
243 $out .= ' ' . Xml
::element( 'id', [], strval( $row->page_id
) ) . "\n";
244 if ( $row->page_is_redirect
) {
245 $page = WikiPage
::factory( $this->currentTitle
);
246 $redirect = $page->getRedirectTarget();
247 if ( $redirect instanceof Title
&& $redirect->isValidRedirectTarget() ) {
249 $out .= Xml
::element( 'redirect', [ 'title' => self
::canonicalTitle( $redirect ) ] );
254 if ( $row->page_restrictions
!= '' ) {
255 $out .= ' ' . Xml
::element( 'restrictions', [],
256 strval( $row->page_restrictions
) ) . "\n";
259 Hooks
::run( 'XmlDumpWriterOpenPage', [ $this, &$out, $row, $this->currentTitle
] );
265 * Closes a "<page>" section on the output stream.
270 function closePage() {
271 if ( $this->currentTitle
!== null ) {
272 $linkCache = MediaWikiServices
::getInstance()->getLinkCache();
273 // In rare cases, link cache has the same key for some pages which
274 // might be read as part of the same batch. T220424 and T220316
275 $linkCache->clearLink( $this->currentTitle
);
281 * @return RevisionStore
283 private function getRevisionStore() {
284 return MediaWikiServices
::getInstance()->getRevisionStore();
288 * @return SqlBlobStore
290 private function getBlobStore() {
291 return MediaWikiServices
::getInstance()->getBlobStore();
295 * Invokes the given method on the given object, catching and logging any storage related
299 * @param string $method
301 * @param string $warning The warning to output in case of a storage related exception.
303 * @return mixed Returns the method's return value,
304 * or null in case of a storage related exception.
307 private function invokeLenient( $obj, $method, $args = [], $warning ) {
309 return call_user_func_array( [ $obj, $method ], $args );
310 } catch ( SuppressedDataException
$ex ) {
312 } catch ( Exception
$ex ) {
313 if ( $ex instanceof MWException ||
$ex instanceof RuntimeException ||
314 $ex instanceof InvalidArgumentException
) {
315 MWDebug
::warning( $warning . ': ' . $ex->getMessage() );
324 * Dumps a "<revision>" section on the output stream, with
325 * data filled in from the given database row.
328 * @param null|object[] $slotRows
332 * @throws MWException
335 function writeRevision( $row, $slotRows = null ) {
336 $rev = $this->getRevisionStore()->newRevisionFromRowAndSlots(
343 $out = " <revision>\n";
344 $out .= " " . Xml
::element( 'id', null, strval( $rev->getId() ) ) . "\n";
346 if ( $rev->getParentId() ) {
347 $out .= " " . Xml
::element( 'parentid', null, strval( $rev->getParentId() ) ) . "\n";
350 $out .= $this->writeTimestamp( $rev->getTimestamp() );
352 if ( $rev->isDeleted( RevisionRecord
::DELETED_USER
) ) {
353 $out .= " " . Xml
::element( 'contributor', [ 'deleted' => 'deleted' ] ) . "\n";
355 // empty values get written out as uid 0, see T224221
356 $user = $rev->getUser();
357 $out .= $this->writeContributor(
358 $user ?
$user->getId() : 0,
359 $user ?
$user->getName() : ''
363 if ( $rev->isMinor() ) {
364 $out .= " <minor/>\n";
366 if ( $rev->isDeleted( RevisionRecord
::DELETED_COMMENT
) ) {
367 $out .= " " . Xml
::element( 'comment', [ 'deleted' => 'deleted' ] ) . "\n";
369 if ( $rev->getComment()->text
!= '' ) {
371 . Xml
::elementClean( 'comment', [], strval( $rev->getComment()->text
) )
376 $contentMode = $rev->isDeleted( RevisionRecord
::DELETED_TEXT
) ? self
::WRITE_STUB_DELETED
377 : $this->contentMode
;
379 foreach ( $rev->getSlots()->getSlots() as $slot ) {
380 $out .= $this->writeSlot( $slot, $contentMode );
383 if ( $rev->isDeleted( RevisionRecord
::DELETED_TEXT
) ) {
384 $out .= " <sha1/>\n";
386 $sha1 = $this->invokeLenient(
390 'failed to determine sha1 for revision ' . $rev->getId()
392 $out .= " " . Xml
::element( 'sha1', null, strval( $sha1 ) ) . "\n";
395 // Avoid PHP 7.1 warning from passing $this by reference
398 if ( $contentMode === self
::WRITE_CONTENT
) {
399 /** @var Content $content */
400 $content = $this->invokeLenient(
403 [ SlotRecord
::MAIN
, RevisionRecord
::RAW
],
404 'Failed to load main slot content of revision ' . $rev->getId()
407 $text = $content ?
$content->serialize() : '';
409 Hooks
::run( 'XmlDumpWriterWriteRevision', [ &$writer, &$out, $row, $text, $rev ] );
411 $out .= " </revision>\n";
417 * @param SlotRecord $slot
418 * @param int $contentMode see the WRITE_XXX constants
422 private function writeSlot( SlotRecord
$slot, $contentMode ) {
423 $isMain = $slot->getRole() === SlotRecord
::MAIN
;
424 $isV11 = $this->schemaVersion
>= XML_DUMP_SCHEMA_VERSION_11
;
426 if ( !$isV11 && !$isMain ) {
427 // ignore extra slots
435 // non-main slots are wrapped into an additional element.
436 $out .= ' ' . Xml
::openElement( 'content' ) . "\n";
438 $out .= $indent . Xml
::element( 'role', null, strval( $slot->getRole() ) ) . "\n";
442 $out .= $indent . Xml
::element( 'origin', null, strval( $slot->getOrigin() ) ) . "\n";
445 $contentModel = $slot->getModel();
446 $contentHandler = ContentHandler
::getForModelID( $contentModel );
447 $contentFormat = $contentHandler->getDefaultFormat();
449 // XXX: The content format is only relevant when actually outputting serialized content.
450 // It should probably be an attribute on the text tag.
451 $out .= $indent . Xml
::element( 'model', null, strval( $contentModel ) ) . "\n";
452 $out .= $indent . Xml
::element( 'format', null, strval( $contentFormat ) ) . "\n";
455 'xml:space' => 'preserve',
456 'bytes' => $this->invokeLenient(
460 'failed to determine size for slot ' . $slot->getRole() . ' of revision '
461 . $slot->getRevision()
466 $textAttributes['sha1'] = $this->invokeLenient(
470 'failed to determine sha1 for slot ' . $slot->getRole() . ' of revision '
471 . $slot->getRevision()
475 if ( $contentMode === self
::WRITE_CONTENT
) {
476 $content = $this->invokeLenient(
480 'failed to load content for slot ' . $slot->getRole() . ' of revision '
481 . $slot->getRevision()
484 if ( $content === null ) {
485 $out .= $indent . Xml
::element( 'text', $textAttributes ) . "\n";
487 $out .= $this->writeText( $content, $textAttributes, $indent );
489 } elseif ( $contentMode === self
::WRITE_STUB_DELETED
) {
490 // write <text> placeholder tag
491 $textAttributes['deleted'] = 'deleted';
492 $out .= $indent . Xml
::element( 'text', $textAttributes ) . "\n";
494 // write <text> stub tag
496 $textAttributes['location'] = $slot->getAddress();
499 // Output the numerical text ID if possible, for backwards compatibility.
500 // Note that this is currently the ONLY reason we have a BlobStore here at all.
501 // When removing this line, check whether the BlobStore has become unused.
503 // NOTE: this will only work for addresses of the form "tt:12345".
504 // If we want to support other kinds of addresses in the future,
505 // we will have to silently ignore failures here.
506 // For now, this fails for "tt:0", which is present in the WMF production
507 // database of of Juli 2019, due to data corruption.
508 $textId = $this->getBlobStore()->getTextIdFromAddress( $slot->getAddress() );
509 } catch ( InvalidArgumentException
$ex ) {
510 MWDebug
::warning( 'Bad content address for slot ' . $slot->getRole()
511 . ' of revision ' . $slot->getRevision() . ': ' . $ex->getMessage() );
516 $textAttributes['id'] = $textId;
519 $out .= $indent . Xml
::element( 'text', $textAttributes ) . "\n";
523 $out .= ' ' . Xml
::closeElement( 'content' ) . "\n";
530 * @param Content $content
531 * @param string[] $textAttributes
532 * @param string $indent
536 private function writeText( Content
$content, $textAttributes, $indent ) {
539 $contentHandler = $content->getContentHandler();
540 $contentFormat = $contentHandler->getDefaultFormat();
542 if ( $content instanceof TextContent
) {
543 // HACK: For text based models, bypass the serialization step. This allows extensions (like Flow)
544 // that use incompatible combinations of serialization format and content model.
545 $data = $content->getNativeData();
547 $data = $content->serialize( $contentFormat );
550 $data = $contentHandler->exportTransform( $data, $contentFormat );
551 $textAttributes['bytes'] = $size = strlen( $data ); // make sure to use the actual size
552 $out .= $indent . Xml
::elementClean( 'text', $textAttributes, strval( $data ) ) . "\n";
558 * Dumps a "<logitem>" section on the output stream, with
559 * data filled in from the given database row.
565 function writeLogItem( $row ) {
566 $out = " <logitem>\n";
567 $out .= " " . Xml
::element( 'id', null, strval( $row->log_id
) ) . "\n";
569 $out .= $this->writeTimestamp( $row->log_timestamp
, " " );
571 if ( $row->log_deleted
& LogPage
::DELETED_USER
) {
572 $out .= " " . Xml
::element( 'contributor', [ 'deleted' => 'deleted' ] ) . "\n";
574 $out .= $this->writeContributor( $row->log_user
, $row->user_name
, " " );
577 if ( $row->log_deleted
& LogPage
::DELETED_COMMENT
) {
578 $out .= " " . Xml
::element( 'comment', [ 'deleted' => 'deleted' ] ) . "\n";
580 $comment = CommentStore
::getStore()->getComment( 'log_comment', $row )->text
;
581 if ( $comment != '' ) {
582 $out .= " " . Xml
::elementClean( 'comment', null, strval( $comment ) ) . "\n";
586 $out .= " " . Xml
::element( 'type', null, strval( $row->log_type
) ) . "\n";
587 $out .= " " . Xml
::element( 'action', null, strval( $row->log_action
) ) . "\n";
589 if ( $row->log_deleted
& LogPage
::DELETED_ACTION
) {
590 $out .= " " . Xml
::element( 'text', [ 'deleted' => 'deleted' ] ) . "\n";
592 $title = Title
::makeTitle( $row->log_namespace
, $row->log_title
);
593 $out .= " " . Xml
::elementClean( 'logtitle', null, self
::canonicalTitle( $title ) ) . "\n";
594 $out .= " " . Xml
::elementClean( 'params',
595 [ 'xml:space' => 'preserve' ],
596 strval( $row->log_params
) ) . "\n";
599 $out .= " </logitem>\n";
605 * @param string $timestamp
606 * @param string $indent Default to six spaces
609 function writeTimestamp( $timestamp, $indent = " " ) {
610 $ts = wfTimestamp( TS_ISO_8601
, $timestamp );
611 return $indent . Xml
::element( 'timestamp', null, $ts ) . "\n";
616 * @param string $text
617 * @param string $indent Default to six spaces
620 function writeContributor( $id, $text, $indent = " " ) {
621 $out = $indent . "<contributor>\n";
622 if ( $id ||
!IP
::isValid( $text ) ) {
623 $out .= $indent . " " . Xml
::elementClean( 'username', null, strval( $text ) ) . "\n";
624 $out .= $indent . " " . Xml
::element( 'id', null, strval( $id ) ) . "\n";
626 $out .= $indent . " " . Xml
::elementClean( 'ip', null, strval( $text ) ) . "\n";
628 $out .= $indent . "</contributor>\n";
633 * Warning! This data is potentially inconsistent. :(
635 * @param bool $dumpContents
638 function writeUploads( $row, $dumpContents = false ) {
639 if ( $row->page_namespace
== NS_FILE
) {
640 $img = MediaWikiServices
::getInstance()->getRepoGroup()->getLocalRepo()
641 ->newFile( $row->page_title
);
642 if ( $img && $img->exists() ) {
644 foreach ( array_reverse( $img->getHistory() ) as $ver ) {
645 $out .= $this->writeUpload( $ver, $dumpContents );
647 $out .= $this->writeUpload( $img, $dumpContents );
656 * @param bool $dumpContents
659 function writeUpload( $file, $dumpContents = false ) {
660 if ( $file->isOld() ) {
662 Xml
::element( 'archivename', null, $file->getArchiveName() ) . "\n";
666 if ( $dumpContents ) {
667 $be = $file->getRepo()->getBackend();
668 # Dump file as base64
669 # Uses only XML-safe characters, so does not need escaping
670 # @todo Too bad this loads the contents into memory (script might swap)
671 $contents = ' <contents encoding="base64">' .
672 chunk_split( base64_encode(
673 $be->getFileContents( [ 'src' => $file->getPath() ] ) ) ) .
678 if ( $file->isDeleted( File
::DELETED_COMMENT
) ) {
679 $comment = Xml
::element( 'comment', [ 'deleted' => 'deleted' ] );
681 $comment = Xml
::elementClean( 'comment', null, strval( $file->getDescription() ) );
683 return " <upload>\n" .
684 $this->writeTimestamp( $file->getTimestamp() ) .
685 $this->writeContributor( $file->getUser( 'id' ), $file->getUser( 'text' ) ) .
686 " " . $comment . "\n" .
687 " " . Xml
::element( 'filename', null, $file->getName() ) . "\n" .
689 " " . Xml
::element( 'src', null, $file->getCanonicalUrl() ) . "\n" .
690 " " . Xml
::element( 'size', null, $file->getSize() ) . "\n" .
691 " " . Xml
::element( 'sha1base36', null, $file->getSha1() ) . "\n" .
692 " " . Xml
::element( 'rel', null, $file->getRel() ) . "\n" .
698 * Return prefixed text form of title, but using the content language's
699 * canonical namespace. This skips any special-casing such as gendered
700 * user namespaces -- which while useful, are not yet listed in the
701 * XML "<siteinfo>" data so are unsafe in export.
703 * @param Title $title
707 public static function canonicalTitle( Title
$title ) {
708 if ( $title->isExternal() ) {
709 return $title->getPrefixedText();
712 $prefix = MediaWikiServices
::getInstance()->getContentLanguage()->
713 getFormattedNsText( $title->getNamespace() );
715 // @todo Emit some kind of warning to the user if $title->getNamespace() !==
716 // NS_MAIN and $prefix === '' (viz. pages in an unregistered namespace)
718 if ( $prefix !== '' ) {
722 return $prefix . $title->getText();