4 * Readahead helper for making large MediaWiki data dumps;
5 * reads in a previous XML dump to sequentially prefetch text
6 * records already normalized and decompressed.
8 * This can save load on the external database servers, hopefully.
10 * Assumes that dumps will be recorded in the canonical order:
11 * - ascending by page_id
12 * - ascending by rev_id within each page
13 * - text contents are immutable and should not change once
14 * recorded, so the previous dump is a reliable source
16 * Requires PHP 5 and the XMLReader PECL extension.
24 function BaseDump( $infile ) {
25 $this->reader = new XMLReader();
26 $this->reader->open( $infile );
30 * Attempts to fetch the text of a particular page revision
31 * from the dump stream. May return null if the page is
34 * @param int $page ID number of page to read
35 * @param int $rev ID number of revision to read
36 * @return string or null
38 function prefetch( $page, $rev ) {
39 while( $this->lastPage < $page && !$this->atEnd ) {
42 if( $this->lastPage > $page || $this->atEnd ) {
43 $this->debug( "BaseDump::prefetch already past page $page looking for rev $rev\n" );
46 while( $this->lastRev < $rev && !$this->atEnd ) {
49 if( $this->lastRev == $rev ) {
50 $this->debug( "BaseDump::prefetch hit on $page, $rev\n" );
51 return $this->nextText();
53 $this->debug( "BaseDump::prefetch already past rev $rev on page $page\n" );
58 function debug( $str ) {
61 //$dumper->progress( $str );
68 $this->skipTo( 'page' );
69 $this->skipTo( 'id' );
70 $this->lastPage = intval( $this->nodeContents() );
78 $this->skipTo( 'revision' );
79 $this->skipTo( 'id' );
80 $this->lastRev = intval( $this->nodeContents() );
87 $this->skipTo( 'text' );
88 return strval( $this->nodeContents() );
94 function skipTo( $name ) {
95 while( $this->reader->read() ) {
96 if( $this->reader->nodeType == XMLREADER_ELEMENT &&
97 $this->reader->name == $name ) {
101 return $this->close();
105 * Shouldn't something like this be built-in to XMLReader?
106 * Fetches text contents of the current element, assuming
107 * no sub-elements or such scary things.
111 function nodeContents() {
112 if( $this->reader->isEmptyElement ) {
116 while( $this->reader->read() ) {
117 switch( $this->reader->nodeType ) {
119 // case XMLREADER_WHITESPACE:
120 case XMLREADER_SIGNIFICANT_WHITESPACE:
121 $buffer .= $this->reader->value;
123 case XMLREADER_END_ELEMENT:
127 return $this->close();
134 $this->reader->close();
135 $this->reader = null;