3 * Helper class for the --prefetch option of dumpTextPass.php
5 * Copyright © 2005 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
24 * @ingroup Maintenance
28 * Readahead helper for making large MediaWiki data dumps;
29 * reads in a previous XML dump to sequentially prefetch text
30 * records already normalized and decompressed.
32 * This can save load on the external database servers, hopefully.
34 * Assumes that dumps will be recorded in the canonical order:
35 * - ascending by page_id
36 * - ascending by rev_id within each page
37 * - text contents are immutable and should not change once
38 * recorded, so the previous dump is a reliable source
40 * @ingroup Maintenance
44 protected $reader = null;
45 protected $atEnd = false;
46 protected $atPageEnd = false;
47 protected $lastPage = 0;
48 protected $lastRev = 0;
49 protected $infiles = null;
51 public function __construct( $infile ) {
52 $this->infiles
= explode( ';', $infile );
53 $this->reader
= new XMLReader();
54 $infile = array_shift( $this->infiles
);
55 if ( defined( 'LIBXML_PARSEHUGE' ) ) {
56 $this->reader
->open( $infile, null, LIBXML_PARSEHUGE
);
58 $this->reader
->open( $infile );
63 * Attempts to fetch the text of a particular page revision
64 * from the dump stream. May return null if the page is
67 * @param int $page ID number of page to read
68 * @param int $rev ID number of revision to read
71 function prefetch( $page, $rev ) {
72 $page = intval( $page );
73 $rev = intval( $rev );
74 while ( $this->lastPage
< $page && !$this->atEnd
) {
75 $this->debug( "BaseDump::prefetch at page $this->lastPage, looking for $page" );
78 if ( $this->lastPage
> $page ||
$this->atEnd
) {
79 $this->debug( "BaseDump::prefetch already past page $page "
80 . "looking for rev $rev [$this->lastPage, $this->lastRev]" );
84 while ( $this->lastRev
< $rev && !$this->atEnd
&& !$this->atPageEnd
) {
85 $this->debug( "BaseDump::prefetch at page $this->lastPage, rev $this->lastRev, "
86 . "looking for $page, $rev" );
89 if ( $this->lastRev
== $rev && !$this->atEnd
) {
90 $this->debug( "BaseDump::prefetch hit on $page, $rev [$this->lastPage, $this->lastRev]" );
92 return $this->nextText();
94 $this->debug( "BaseDump::prefetch already past rev $rev on page $page "
95 . "[$this->lastPage, $this->lastRev]" );
101 function debug( $str ) {
102 wfDebug( $str . "\n" );
104 // $dumper->progress( $str );
110 function nextPage() {
111 if ( $this->skipTo( 'page', 'mediawiki' ) ) {
112 if ( $this->skipTo( 'id' ) ) {
113 $this->lastPage
= intval( $this->nodeContents() );
115 $this->atPageEnd
= false;
119 if ( count( $this->infiles
) ) {
120 $infile = array_shift( $this->infiles
);
121 $this->reader
->open( $infile );
122 $this->atEnd
= false;
131 if ( $this->skipTo( 'revision' ) ) {
132 if ( $this->skipTo( 'id' ) ) {
133 $this->lastRev
= intval( $this->nodeContents() );
136 $this->atPageEnd
= true;
144 function nextText() {
145 $this->skipTo( 'text' );
147 return strval( $this->nodeContents() );
152 * @param string $name
153 * @param string $parent
156 function skipTo( $name, $parent = 'page' ) {
157 if ( $this->atEnd
) {
160 while ( $this->reader
->read() ) {
161 if ( $this->reader
->nodeType
== XMLReader
::ELEMENT
162 && $this->reader
->name
== $name
166 if ( $this->reader
->nodeType
== XMLReader
::END_ELEMENT
167 && $this->reader
->name
== $parent
169 $this->debug( "BaseDump::skipTo found </$parent> searching for <$name>" );
175 return $this->close();
179 * Shouldn't something like this be built-in to XMLReader?
180 * Fetches text contents of the current element, assuming
181 * no sub-elements or such scary things.
186 function nodeContents() {
187 if ( $this->atEnd
) {
190 if ( $this->reader
->isEmptyElement
) {
194 while ( $this->reader
->read() ) {
195 switch ( $this->reader
->nodeType
) {
196 case XMLReader
::TEXT
:
197 // case XMLReader::WHITESPACE:
198 case XMLReader
::SIGNIFICANT_WHITESPACE
:
199 $buffer .= $this->reader
->value
;
201 case XMLReader
::END_ELEMENT
:
206 return $this->close();
214 $this->reader
->close();