3 * BackupDumper that postprocesses XML dumps from dumpBackup.php to add page text
5 * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
25 * @ingroup Maintenance
28 require_once __DIR__
. '/BackupDumper.php';
29 require_once __DIR__
. '/SevenZipStream.php';
30 require_once __DIR__
. '/../../includes/export/WikiExporter.php';
32 use MediaWiki\MediaWikiServices
;
33 use MediaWiki\Shell\Shell
;
34 use MediaWiki\Storage\BlobAccessException
;
35 use MediaWiki\Storage\SqlBlobStore
;
36 use Wikimedia\Rdbms\IMaintainableDatabase
;
39 * @ingroup Maintenance
41 class TextPassDumper
extends BackupDumper
{
43 public $prefetch = null;
44 /** @var string|bool */
46 /** @var string|bool */
49 // when we spend more than maxTimeAllowed seconds on this run, we continue
50 // processing until we write out the next complete page, then save output file(s),
51 // rename it/them and open new one(s)
52 public $maxTimeAllowed = 0; // 0 = no limit
54 protected $input = "php://stdin";
55 protected $history = WikiExporter
::FULL
;
56 protected $fetchCount = 0;
57 protected $prefetchCount = 0;
58 protected $prefetchCountLast = 0;
59 protected $fetchCountLast = 0;
61 protected $maxFailures = 5;
62 protected $maxConsecutiveFailedTextRetrievals = 200;
63 protected $failureTimeout = 5; // Seconds to sleep after db failure
65 protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
67 protected $php = "php";
68 protected $spawn = false;
73 protected $spawnProc = false;
78 protected $spawnWrite = false;
83 protected $spawnRead = false;
88 protected $spawnErr = false;
91 * @var bool|XmlDumpWriter
93 protected $xmlwriterobj = false;
95 protected $timeExceeded = false;
96 protected $firstPageWritten = false;
97 protected $lastPageWritten = false;
98 protected $checkpointJustWritten = false;
99 protected $checkpointFiles = [];
102 * @var IMaintainableDatabase
107 * @param array|null $args For backward compatibility
109 function __construct( $args = null ) {
110 parent
::__construct();
112 $this->addDescription( <<<TEXT
113 This script postprocesses XML dumps from dumpBackup.php to add
114 page text which was stubbed out (using --stub).
116 XML input is accepted on stdin.
117 XML output is sent to stdout; progress reports are sent to stderr.
120 $this->stderr
= fopen( "php://stderr", "wt" );
122 $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
123 'Specify as --stub=<type>:<file>.', false, true );
124 $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
125 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
127 $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
128 'out complete page, closing xml file properly, and opening new one' .
129 'with header). This option requires the checkpointfile option.', false, true );
130 $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
131 'first pageid written for the first %s (required) and the last pageid written for the ' .
132 'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
133 $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
134 $this->addOption( 'full', 'Dump all revisions of every page' );
135 $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
136 $this->addOption( 'spawn', 'Spawn a subprocess for loading text records, optionally specify ' .
137 'php[,mwscript] paths' );
138 $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
139 '(Default: 512KB, Minimum: 4KB)', false, true );
142 $this->loadWithArgv( $args );
143 $this->processOptions();
148 * @return SqlBlobStore
150 private function getBlobStore() {
151 return MediaWikiServices
::getInstance()->getBlobStore();
155 $this->processOptions();
159 function processOptions() {
160 parent
::processOptions();
162 if ( $this->hasOption( 'buffersize' ) ) {
163 $this->bufferSize
= max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
166 if ( $this->hasOption( 'prefetch' ) ) {
167 $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
168 $this->prefetch
= new BaseDump( $url );
171 if ( $this->hasOption( 'stub' ) ) {
172 $this->input
= $this->processFileOpt( $this->getOption( 'stub' ) );
175 if ( $this->hasOption( 'maxtime' ) ) {
176 $this->maxTimeAllowed
= intval( $this->getOption( 'maxtime' ) ) * 60;
179 if ( $this->hasOption( 'checkpointfile' ) ) {
180 $this->checkpointFiles
= $this->getOption( 'checkpointfile' );
183 if ( $this->hasOption( 'current' ) ) {
184 $this->history
= WikiExporter
::CURRENT
;
187 if ( $this->hasOption( 'full' ) ) {
188 $this->history
= WikiExporter
::FULL
;
191 if ( $this->hasOption( 'spawn' ) ) {
193 $val = $this->getOption( 'spawn' );
195 $this->php
= explode( ',', $val, 2 );
201 * Drop the database connection $this->db and try to get a new one.
203 * This function tries to get a /different/ connection if this is
204 * possible. Hence, (if this is possible) it switches to a different
205 * failover upon each call.
207 * This function resets $this->lb and closes all connections on it.
209 * @throws MWException
210 * @suppress PhanTypeObjectUnsetDeclaredProperty
212 function rotateDb() {
213 // Cleaning up old connections
214 if ( isset( $this->lb
) ) {
215 $this->lb
->closeAll();
219 if ( $this->forcedDb
!== null ) {
220 $this->db
= $this->forcedDb
;
225 if ( isset( $this->db
) && $this->db
->isOpen() ) {
226 throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
231 // Trying to set up new connection.
232 // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
233 // individually retrying at different layers of code.
236 $lbFactory = MediaWikiServices
::getInstance()->getDBLoadBalancerFactory();
237 $this->lb
= $lbFactory->newMainLB();
238 } catch ( Exception
$e ) {
239 throw new MWException( __METHOD__
240 . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
244 $this->db
= $this->lb
->getMaintenanceConnectionRef( DB_REPLICA
, 'dump' );
245 } catch ( Exception
$e ) {
246 throw new MWException( __METHOD__
247 . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
251 function initProgress( $history = WikiExporter
::FULL
) {
252 parent
::initProgress();
253 $this->timeOfCheckpoint
= $this->startTime
;
256 function dump( $history, $text = WikiExporter
::TEXT
) {
257 // Notice messages will foul up your XML output even if they're
258 // relatively harmless.
259 if ( ini_get( 'display_errors' ) ) {
260 ini_set( 'display_errors', 'stderr' );
263 $this->initProgress( $this->history
);
265 // We are trying to get an initial database connection to avoid that the
266 // first try of this request's first call to getText fails. However, if
267 // obtaining a good DB connection fails it's not a serious issue, as
268 // getText does retry upon failure and can start without having a working
272 } catch ( Exception
$e ) {
273 // We do not even count this as failure. Just let eventual
275 $this->progress( "Getting initial DB connection failed (" .
276 $e->getMessage() . ")" );
279 $this->egress
= new ExportProgressFilter( $this->sink
, $this );
281 // it would be nice to do it in the constructor, oh well. need egress set
282 $this->finalOptionCheck();
284 // we only want this so we know how to close a stream :-P
285 $this->xmlwriterobj
= new XmlDumpWriter( XmlDumpWriter
::WRITE_CONTENT
, $this->schemaVersion
);
287 $input = fopen( $this->input
, "rt" );
288 $this->readDump( $input );
290 if ( $this->spawnProc
) {
294 $this->report( true );
297 function processFileOpt( $opt ) {
298 $split = explode( ':', $opt, 2 );
301 if ( count( $split ) === 2 ) {
304 $fileURIs = explode( ';', $param );
305 foreach ( $fileURIs as $URI ) {
311 $newURI = "compress.zlib://$URI";
314 $newURI = "compress.bzip2://$URI";
317 $newURI = "mediawiki.compress.7z://$URI";
322 $newFileURIs[] = $newURI;
324 $val = implode( ';', $newFileURIs );
330 * Overridden to include prefetch ratio if enabled.
332 function showReport() {
333 if ( !$this->prefetch
) {
334 parent
::showReport();
339 if ( $this->reporting
) {
340 $now = wfTimestamp( TS_DB
);
341 $nowts = microtime( true );
342 $deltaAll = $nowts - $this->startTime
;
343 $deltaPart = $nowts - $this->lastTime
;
344 $this->pageCountPart
= $this->pageCount
- $this->pageCountLast
;
345 $this->revCountPart
= $this->revCount
- $this->revCountLast
;
348 $portion = $this->revCount
/ $this->maxCount
;
349 $eta = $this->startTime +
$deltaAll / $portion;
350 $etats = wfTimestamp( TS_DB
, intval( $eta ) );
351 if ( $this->fetchCount
) {
352 $fetchRate = 100.0 * $this->prefetchCount
/ $this->fetchCount
;
356 $pageRate = $this->pageCount
/ $deltaAll;
357 $revRate = $this->revCount
/ $deltaAll;
365 if ( $this->fetchCountLast
) {
366 $fetchRatePart = 100.0 * $this->prefetchCountLast
/ $this->fetchCountLast
;
368 $fetchRatePart = '-';
370 $pageRatePart = $this->pageCountPart
/ $deltaPart;
371 $revRatePart = $this->revCountPart
/ $deltaPart;
373 $fetchRatePart = '-';
378 $dbDomain = WikiMap
::getCurrentWikiDbDomain()->getId();
379 $this->progress( sprintf(
380 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
381 . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
382 . "prefetched (all|curr), ETA %s [max %d]",
383 $now, $dbDomain, $this->ID
, $this->pageCount
, $pageRate,
384 $pageRatePart, $this->revCount
, $revRate, $revRatePart,
385 $fetchRate, $fetchRatePart, $etats, $this->maxCount
387 $this->lastTime
= $nowts;
388 $this->revCountLast
= $this->revCount
;
389 $this->prefetchCountLast
= $this->prefetchCount
;
390 $this->fetchCountLast
= $this->fetchCount
;
394 function setTimeExceeded() {
395 $this->timeExceeded
= true;
398 function checkIfTimeExceeded() {
399 if ( $this->maxTimeAllowed
400 && ( $this->lastTime
- $this->timeOfCheckpoint
> $this->maxTimeAllowed
)
408 function finalOptionCheck() {
409 if ( ( $this->checkpointFiles
&& !$this->maxTimeAllowed
)
410 ||
( $this->maxTimeAllowed
&& !$this->checkpointFiles
)
412 throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
414 foreach ( $this->checkpointFiles
as $checkpointFile ) {
415 $count = substr_count( $checkpointFile, "%s" );
417 throw new MWException( "Option checkpointfile must contain two '%s' "
418 . "for substitution of first and last pageids, count is $count instead, "
419 . "file is $checkpointFile.\n" );
423 if ( $this->checkpointFiles
) {
424 $filenameList = (array)$this->egress
->getFilenames();
425 if ( count( $filenameList ) != count( $this->checkpointFiles
) ) {
426 throw new MWException( "One checkpointfile must be specified "
427 . "for each output option, if maxtime is used.\n" );
433 * @throws MWException Failure to parse XML input
434 * @param string $input
437 function readDump( $input ) {
439 $this->openElement
= false;
440 $this->atStart
= true;
442 $this->lastName
= "";
445 $this->thisRevModel
= null;
446 $this->thisRevFormat
= null;
448 $parser = xml_parser_create( "UTF-8" );
449 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING
, false );
451 xml_set_element_handler(
453 [ $this, 'startElement' ],
454 [ $this, 'endElement' ]
456 xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
458 $offset = 0; // for context extraction on error reporting
460 if ( $this->checkIfTimeExceeded() ) {
461 $this->setTimeExceeded();
463 $chunk = fread( $input, $this->bufferSize
);
464 if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
465 wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
467 $byte = xml_get_current_byte_index( $parser );
468 $msg = wfMessage( 'xml-error-string',
469 'XML import parse failure',
470 xml_get_current_line_number( $parser ),
471 xml_get_current_column_number( $parser ),
472 $byte . ( is_null( $chunk ) ?
null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
473 xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
475 xml_parser_free( $parser );
477 throw new MWException( $msg );
479 $offset +
= strlen( $chunk );
480 } while ( $chunk !== false && !feof( $input ) );
481 if ( $this->maxTimeAllowed
) {
482 $filenameList = (array)$this->egress
->getFilenames();
483 // we wrote some stuff after last checkpoint that needs renamed
484 if ( file_exists( $filenameList[0] ) ) {
486 # we might have just written the header and footer and had no
487 # pages or revisions written... perhaps they were all deleted
488 # there's no pageID 0 so we use that. the caller is responsible
489 # for deciding what to do with a file containing only the
490 # siteinfo information and the mw tags.
491 if ( !$this->firstPageWritten
) {
492 $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT
);
493 $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT
);
495 $firstPageID = str_pad( $this->firstPageWritten
, 9, "0", STR_PAD_LEFT
);
496 $lastPageID = str_pad( $this->lastPageWritten
, 9, "0", STR_PAD_LEFT
);
499 $filenameCount = count( $filenameList );
500 for ( $i = 0; $i < $filenameCount; $i++
) {
501 $checkpointNameFilledIn = sprintf( $this->checkpointFiles
[$i], $firstPageID, $lastPageID );
502 $fileinfo = pathinfo( $filenameList[$i] );
503 $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
505 $this->egress
->closeAndRename( $newFilenames );
508 xml_parser_free( $parser );
514 * Applies applicable export transformations to $text.
516 * @param string $text
517 * @param string $model
518 * @param string|null $format
522 private function exportTransform( $text, $model, $format = null ) {
524 $handler = ContentHandler
::getForModelID( $model );
525 $text = $handler->exportTransform( $text, $format );
527 catch ( MWException
$ex ) {
529 "Unable to apply export transformation for content model '$model': " .
538 * Tries to load revision text.
539 * Export transformations are applied if the content model is given or can be
540 * determined from the database.
542 * Upon errors, retries (Up to $this->maxFailures tries each call).
543 * If still no good revision could be found even after this retrying, "" is returned.
544 * If no good revision text could be returned for
545 * $this->maxConsecutiveFailedTextRetrievals consecutive calls to getText, MWException
548 * @param int|string $id Content address, or text row ID.
549 * @param string|bool|null $model The content model used to determine
550 * applicable export transformations.
551 * If $model is null, it will be determined from the database.
552 * @param string|null $format The content format used when applying export transformations.
554 * @throws MWException
555 * @return string The revision text for $id, or ""
557 function getText( $id, $model = null, $format = null ) {
558 global $wgContentHandlerUseDB;
560 $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
561 $text = false; // The candidate for a good text. false if no proper value.
562 $failures = 0; // The number of times, this invocation of getText already failed.
564 // The number of times getText failed without yielding a good text in between.
565 static $consecutiveFailedTextRetrievals = 0;
569 // To allow to simply return on success and do not have to worry about book keeping,
570 // we assume, this fetch works (possible after some retries). Nevertheless, we koop
571 // the old value, so we can restore it, if problems occur (See after the while loop).
572 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
573 $consecutiveFailedTextRetrievals = 0;
575 if ( $model === null && $wgContentHandlerUseDB ) {
576 // TODO: MCR: use content table
577 $row = $this->db
->selectRow(
579 [ 'rev_content_model', 'rev_content_format' ],
580 [ 'rev_id' => $this->thisRev
],
585 $model = $row->rev_content_model
;
586 $format = $row->rev_content_format
;
590 if ( $model === null ||
$model === '' ) {
594 while ( $failures < $this->maxFailures
) {
595 // As soon as we found a good text for the $id, we will return immediately.
596 // Hence, if we make it past the try catch block, we know that we did not
600 // Step 1: Get some text (or reuse from previous iteratuon if checking
601 // for plausibility failed)
603 // Trying to get prefetch, if it has not been tried before
604 if ( $text === false && isset( $this->prefetch
) && $prefetchNotTried ) {
605 $prefetchNotTried = false;
606 $tryIsPrefetch = true;
607 $text = $this->prefetch
->prefetch( (int)$this->thisPage
, (int)$this->thisRev
);
609 if ( $text === null ) {
613 if ( is_string( $text ) && $model !== false ) {
614 // Apply export transformation to text coming from an old dump.
615 // The purpose of this transformation is to convert up from legacy
616 // formats, which may still be used in the older dump that is used
617 // for pre-fetching. Applying the transformation again should not
618 // interfere with content that is already in the correct form.
619 $text = $this->exportTransform( $text, $model, $format );
623 if ( $text === false ) {
624 // Fallback to asking the database
625 $tryIsPrefetch = false;
626 if ( $this->spawn
) {
627 $text = $this->getTextSpawned( $id );
629 $text = $this->getTextDb( $id );
632 if ( $text !== false && $model !== false ) {
633 // Apply export transformation to text coming from the database.
634 // Prefetched text should already have transformations applied.
635 $text = $this->exportTransform( $text, $model, $format );
638 // No more checks for texts from DB for now.
639 // If we received something that is not false,
640 // We treat it as good text, regardless of whether it actually is or is not
641 if ( $text !== false ) {
646 if ( $text === false ) {
647 throw new MWException( "Generic error while obtaining text for id " . $id );
650 // We received a good candidate for the text of $id via some method
652 // Step 2: Checking for plausibility and return the text if it is
654 $revID = intval( $this->thisRev
);
655 if ( !isset( $this->db
) ) {
656 throw new MWException( "No database available" );
659 if ( $model !== CONTENT_MODEL_WIKITEXT
) {
660 $revLength = strlen( $text );
662 $revLength = $this->db
->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
665 if ( strlen( $text ) == $revLength ) {
666 if ( $tryIsPrefetch ) {
667 $this->prefetchCount++
;
674 throw new MWException( "Received text is unplausible for id " . $id );
675 } catch ( Exception
$e ) {
676 $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
677 if ( $failures +
1 < $this->maxFailures
) {
678 $msg .= " (Will retry " . ( $this->maxFailures
- $failures - 1 ) . " more times)";
680 $this->progress( $msg );
683 // Something went wrong; we did not a text that was plausible :(
686 // A failure in a prefetch hit does not warrant resetting db connection etc.
687 if ( !$tryIsPrefetch ) {
688 // After backing off for some time, we try to reboot the whole process as
689 // much as possible to not carry over failures from one part to the other
691 sleep( $this->failureTimeout
);
694 if ( $this->spawn
) {
698 } catch ( Exception
$e ) {
699 $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
700 " Trying to continue anyways" );
705 // Retirieving a good text for $id failed (at least) maxFailures times.
706 // We abort for this $id.
708 // Restoring the consecutive failures, and maybe aborting, if the dump
710 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals +
1;
711 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals
) {
712 throw new MWException( "Graceful storage failure" );
719 * Loads the serialized content from storage.
721 * @param int|string $id Content address, or text row ID.
722 * @return bool|string
724 private function getTextDb( $id ) {
725 $store = $this->getBlobStore();
726 $address = ( is_int( $id ) ||
strpos( $id, ':' ) === false )
727 ? SqlBlobStore
::makeAddressFromTextId( (int)$id )
731 $text = $store->getBlob( $address );
733 $stripped = str_replace( "\r", "", $text );
734 $normalized = MediaWikiServices
::getInstance()->getContentLanguage()
735 ->normalize( $stripped );
738 } catch ( BlobAccessException
$ex ) {
739 // XXX: log a warning?
745 * @param int|string $address Content address, or text row ID.
746 * @return bool|string
748 private function getTextSpawned( $address ) {
749 Wikimedia\
suppressWarnings();
750 if ( !$this->spawnProc
) {
754 $text = $this->getTextSpawnedOnce( $address );
755 Wikimedia\restoreWarnings
();
760 function openSpawn() {
763 $wiki = WikiMap
::getWikiIdFromDbDomain( WikiMap
::getCurrentWikiDbDomain() );
764 if ( count( $this->php
) == 2 ) {
765 $mwscriptpath = $this->php
[1];
767 $mwscriptpath = "$IP/../multiversion/MWScript.php";
769 if ( file_exists( $mwscriptpath ) ) {
771 array_map( [ Shell
::class, 'escape' ],
776 '--wiki', $wiki ] ) );
779 array_map( [ Shell
::class, 'escape' ],
782 "$IP/maintenance/fetchText.php",
783 '--wiki', $wiki ] ) );
786 0 => [ "pipe", "r" ],
787 1 => [ "pipe", "w" ],
788 2 => [ "file", "/dev/null", "a" ] ];
791 $this->progress( "Spawning database subprocess: $cmd" );
792 $this->spawnProc
= proc_open( $cmd, $spec, $pipes );
793 if ( !$this->spawnProc
) {
794 $this->progress( "Subprocess spawn failed." );
799 $this->spawnWrite
, // -> stdin
800 $this->spawnRead
, // <- stdout
806 private function closeSpawn() {
807 Wikimedia\
suppressWarnings();
808 if ( $this->spawnRead
) {
809 fclose( $this->spawnRead
);
811 $this->spawnRead
= false;
812 if ( $this->spawnWrite
) {
813 fclose( $this->spawnWrite
);
815 $this->spawnWrite
= false;
816 if ( $this->spawnErr
) {
817 fclose( $this->spawnErr
);
819 $this->spawnErr
= false;
820 if ( $this->spawnProc
) {
821 pclose( $this->spawnProc
);
823 $this->spawnProc
= false;
824 Wikimedia\restoreWarnings
();
828 * @param int|string $address Content address, or text row ID.
829 * @return bool|string
831 private function getTextSpawnedOnce( $address ) {
832 if ( is_int( $address ) ||
intval( $address ) ) {
833 $address = SqlBlobStore
::makeAddressFromTextId( (int)$address );
836 $ok = fwrite( $this->spawnWrite
, "$address\n" );
837 // $this->progress( ">> $id" );
842 $ok = fflush( $this->spawnWrite
);
843 // $this->progress( ">> [flush]" );
848 // check that the text address they are sending is the one we asked for
849 // this avoids out of sync revision text errors we have encountered in the past
850 $newAddress = fgets( $this->spawnRead
);
851 if ( $newAddress === false ) {
854 $newAddress = trim( $newAddress );
855 if ( strpos( $newAddress, ':' ) === false ) {
856 $newAddress = SqlBlobStore
::makeAddressFromTextId( intval( $newAddress ) );
859 if ( $newAddress !== $address ) {
863 $len = fgets( $this->spawnRead
);
864 // $this->progress( "<< " . trim( $len ) );
865 if ( $len === false ) {
869 $nbytes = intval( $len );
870 // actual error, not zero-length text
877 // Subprocess may not send everything at once, we have to loop.
878 while ( $nbytes > strlen( $text ) ) {
879 $buffer = fread( $this->spawnRead
, $nbytes - strlen( $text ) );
880 if ( $buffer === false ) {
886 $gotbytes = strlen( $text );
887 if ( $gotbytes != $nbytes ) {
888 $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
893 // Do normalization in the dump thread...
894 $stripped = str_replace( "\r", "", $text );
895 $normalized = MediaWikiServices
::getInstance()->getContentLanguage()->
896 normalize( $stripped );
901 function startElement( $parser, $name, $attribs ) {
902 $this->checkpointJustWritten
= false;
904 $this->clearOpenElement( null );
905 $this->lastName
= $name;
907 if ( $name == 'revision' ) {
908 $this->state
= $name;
909 $this->egress
->writeOpenPage( null, $this->buffer
);
911 } elseif ( $name == 'page' ) {
912 $this->state
= $name;
913 if ( $this->atStart
) {
914 $this->egress
->writeOpenStream( $this->buffer
);
916 $this->atStart
= false;
920 if ( $name == "text" && isset( $attribs['id'] ) ) {
921 $id = $attribs['id'];
922 $model = trim( $this->thisRevModel
);
923 $format = trim( $this->thisRevFormat
);
925 $model = $model === '' ?
null : $model;
926 $format = $format === '' ?
null : $format;
928 $text = $this->getText( $id, $model, $format );
929 $this->openElement
= [ $name, [ 'xml:space' => 'preserve' ] ];
930 if ( strlen( $text ) > 0 ) {
931 $this->characterData( $parser, $text );
934 $this->openElement
= [ $name, $attribs ];
938 function endElement( $parser, $name ) {
939 $this->checkpointJustWritten
= false;
941 if ( $this->openElement
) {
942 $this->clearOpenElement( "" );
944 $this->buffer
.= "</$name>";
947 if ( $name == 'revision' ) {
948 $this->egress
->writeRevision( null, $this->buffer
);
951 $this->thisRevModel
= null;
952 $this->thisRevFormat
= null;
953 } elseif ( $name == 'page' ) {
954 if ( !$this->firstPageWritten
) {
955 $this->firstPageWritten
= trim( $this->thisPage
);
957 $this->lastPageWritten
= trim( $this->thisPage
);
958 if ( $this->timeExceeded
) {
959 $this->egress
->writeClosePage( $this->buffer
);
960 // nasty hack, we can't just write the chardata after the
961 // page tag, it will include leading blanks from the next line
962 $this->egress
->sink
->write( "\n" );
964 $this->buffer
= $this->xmlwriterobj
->closeStream();
965 $this->egress
->writeCloseStream( $this->buffer
);
968 $this->thisPage
= "";
969 // this could be more than one file if we had more than one output arg
971 $filenameList = (array)$this->egress
->getFilenames();
973 $firstPageID = str_pad( $this->firstPageWritten
, 9, "0", STR_PAD_LEFT
);
974 $lastPageID = str_pad( $this->lastPageWritten
, 9, "0", STR_PAD_LEFT
);
975 $filenamesCount = count( $filenameList );
976 for ( $i = 0; $i < $filenamesCount; $i++
) {
977 $checkpointNameFilledIn = sprintf( $this->checkpointFiles
[$i], $firstPageID, $lastPageID );
978 $fileinfo = pathinfo( $filenameList[$i] );
979 $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
981 $this->egress
->closeRenameAndReopen( $newFilenames );
982 $this->buffer
= $this->xmlwriterobj
->openStream();
983 $this->timeExceeded
= false;
984 $this->timeOfCheckpoint
= $this->lastTime
;
985 $this->firstPageWritten
= false;
986 $this->checkpointJustWritten
= true;
988 $this->egress
->writeClosePage( $this->buffer
);
990 $this->thisPage
= "";
992 } elseif ( $name == 'mediawiki' ) {
993 $this->egress
->writeCloseStream( $this->buffer
);
998 function characterData( $parser, $data ) {
999 $this->clearOpenElement( null );
1000 if ( $this->lastName
== "id" ) {
1001 if ( $this->state
== "revision" ) {
1002 $this->thisRev
.= $data;
1003 } elseif ( $this->state
== "page" ) {
1004 $this->thisPage
.= $data;
1006 } elseif ( $this->lastName
== "model" ) {
1007 $this->thisRevModel
.= $data;
1008 } elseif ( $this->lastName
== "format" ) {
1009 $this->thisRevFormat
.= $data;
1012 // have to skip the newline left over from closepagetag line of
1013 // end of checkpoint files. nasty hack!!
1014 if ( $this->checkpointJustWritten
) {
1015 if ( $data[0] == "\n" ) {
1016 $data = substr( $data, 1 );
1018 $this->checkpointJustWritten
= false;
1020 $this->buffer
.= htmlspecialchars( $data );
1023 function clearOpenElement( $style ) {
1024 if ( $this->openElement
) {
1025 $this->buffer
.= Xml
::element( $this->openElement
[0], $this->openElement
[1], $style );
1026 $this->openElement
= false;