3 * Helper class for the --prefetch option of dumpTextPass.php
5 * Copyright © 2005 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
24 * @ingroup Maintenance
28 * Readahead helper for making large MediaWiki data dumps;
29 * reads in a previous XML dump to sequentially prefetch text
30 * records already normalized and decompressed.
32 * This can save load on the external database servers, hopefully.
34 * Assumes that dumps will be recorded in the canonical order:
35 * - ascending by page_id
36 * - ascending by rev_id within each page
37 * - text contents are immutable and should not change once
38 * recorded, so the previous dump is a reliable source
40 * @ingroup Maintenance
44 protected $reader = null;
45 protected $atEnd = false;
46 protected $atPageEnd = false;
47 protected $lastPage = 0;
48 protected $lastRev = 0;
49 protected $infiles = null;
51 public function __construct( $infile ) {
52 $this->infiles
= explode( ';', $infile );
53 $this->reader
= new XMLReader();
54 $infile = array_shift( $this->infiles
);
55 $this->reader
->open( $infile, null, LIBXML_PARSEHUGE
);
59 * Attempts to fetch the text of a particular page revision
60 * from the dump stream. May return null if the page is
63 * @param int $page ID number of page to read
64 * @param int $rev ID number of revision to read
67 function prefetch( $page, $rev ) {
68 $page = intval( $page );
69 $rev = intval( $rev );
70 while ( $this->lastPage
< $page && !$this->atEnd
) {
71 $this->debug( "BaseDump::prefetch at page $this->lastPage, looking for $page" );
74 if ( $this->lastPage
> $page ||
$this->atEnd
) {
75 $this->debug( "BaseDump::prefetch already past page $page "
76 . "looking for rev $rev [$this->lastPage, $this->lastRev]" );
80 while ( $this->lastRev
< $rev && !$this->atEnd
&& !$this->atPageEnd
) {
81 $this->debug( "BaseDump::prefetch at page $this->lastPage, rev $this->lastRev, "
82 . "looking for $page, $rev" );
85 if ( $this->lastRev
== $rev && !$this->atEnd
) {
86 $this->debug( "BaseDump::prefetch hit on $page, $rev [$this->lastPage, $this->lastRev]" );
88 return $this->nextText();
90 $this->debug( "BaseDump::prefetch already past rev $rev on page $page "
91 . "[$this->lastPage, $this->lastRev]" );
97 function debug( $str ) {
98 wfDebug( $str . "\n" );
100 // $dumper->progress( $str );
106 function nextPage() {
107 if ( $this->skipTo( 'page', 'mediawiki' ) ) {
108 if ( $this->skipTo( 'id' ) ) {
109 $this->lastPage
= intval( $this->nodeContents() );
111 $this->atPageEnd
= false;
115 if ( count( $this->infiles
) ) {
116 $infile = array_shift( $this->infiles
);
117 $this->reader
->open( $infile );
118 $this->atEnd
= false;
127 if ( $this->skipTo( 'revision' ) ) {
128 if ( $this->skipTo( 'id' ) ) {
129 $this->lastRev
= intval( $this->nodeContents() );
132 $this->atPageEnd
= true;
140 function nextText() {
141 $this->skipTo( 'text' );
143 return strval( $this->nodeContents() );
148 * @param string $name
149 * @param string $parent
152 function skipTo( $name, $parent = 'page' ) {
153 if ( $this->atEnd
) {
156 while ( $this->reader
->read() ) {
157 if ( $this->reader
->nodeType
== XMLReader
::ELEMENT
158 && $this->reader
->name
== $name
162 if ( $this->reader
->nodeType
== XMLReader
::END_ELEMENT
163 && $this->reader
->name
== $parent
165 $this->debug( "BaseDump::skipTo found </$parent> searching for <$name>" );
171 return $this->close();
175 * Shouldn't something like this be built-in to XMLReader?
176 * Fetches text contents of the current element, assuming
177 * no sub-elements or such scary things.
182 function nodeContents() {
183 if ( $this->atEnd
) {
186 if ( $this->reader
->isEmptyElement
) {
190 while ( $this->reader
->read() ) {
191 switch ( $this->reader
->nodeType
) {
192 case XMLReader
::TEXT
:
193 // case XMLReader::WHITESPACE:
194 case XMLReader
::SIGNIFICANT_WHITESPACE
:
195 $buffer .= $this->reader
->value
;
197 case XMLReader
::END_ELEMENT
:
202 return $this->close();
210 $this->reader
->close();